commit 0f3e483678aaf3cd7a3907843d004792a7cd27ad
parent d9e2356e47f3a064cf739013957338322d7cc731
Author: lash <dev@holbrook.no>
Date: Sat, 18 Jan 2025 08:21:34 +0000
Correct key outputs from dump in pg, gdbm
Diffstat:
6 files changed, 100 insertions(+), 39 deletions(-)
diff --git a/db/db.go b/db/db.go
@@ -78,6 +78,7 @@ type Db interface {
// Prefix returns the current active datatype prefix
Prefix() uint8
Dump(context.Context, []byte) (*Dumper, error)
+ DecodeKey(ctx context.Context, key []byte) ([]byte, error)
}
type LookupKey struct {
@@ -194,10 +195,10 @@ func(bd *DbBase) CheckPut() bool {
return bd.baseDb.pfx & bd.baseDb.lock == 0
}
-func ToSessionKey(pfx uint8, sessionId []byte, key []byte) []byte {
+func (bd *DbBase) ToSessionKey(pfx uint8, key []byte) []byte {
var b []byte
if (pfx > datatype_sessioned_threshold) {
- b = append([]byte(sessionId), key...)
+ b = append([]byte(bd.sid), key...)
} else {
b = key
}
@@ -209,7 +210,7 @@ func(bd *DbBase) FromSessionKey(key []byte) ([]byte, error) {
return key, nil
}
if !bytes.HasPrefix(key, bd.baseDb.sid) {
- return nil, fmt.Errorf("session id prefix %s does not match key", string(bd.baseDb.sid))
+ return nil, fmt.Errorf("session id prefix %s does not match key %x", string(bd.baseDb.sid), key)
}
return bytes.TrimPrefix(key, bd.baseDb.sid), nil
}
@@ -225,7 +226,8 @@ func(bd *DbBase) ToKey(ctx context.Context, key []byte) (LookupKey, error) {
if db.pfx == DATATYPE_UNKNOWN {
return lk, errors.New("datatype prefix cannot be UNKNOWN")
}
- b := ToSessionKey(db.pfx, db.sid, key)
+ //b := ToSessionKey(db.pfx, db.sid, key)
+ b := bd.ToSessionKey(db.pfx, key)
lk.Default = ToDbKey(db.pfx, b, nil)
if db.pfx & (DATATYPE_MENU | DATATYPE_TEMPLATE | DATATYPE_STATICLOAD) > 0 {
if db.lang != nil {
@@ -244,3 +246,18 @@ func(bd *DbBase) ToKey(ctx context.Context, key []byte) (LookupKey, error) {
logg.TraceCtxf(ctx, "made db lookup key", "lk", lk.Default, "pfx", db.pfx)
return lk, nil
}
+
+func(bd *DbBase) DecodeKey(ctx context.Context, key []byte) ([]byte, error) {
+ var err error
+ oldKey := key
+ key, err = FromDbKey(key)
+ if err != nil {
+ return []byte{}, err
+ }
+ key, err = bd.FromSessionKey(key)
+ if err != nil {
+ return []byte{}, err
+ }
+ logg.DebugCtxf(ctx, "decoded key", "key", key, "fromkey", oldKey)
+ return key, nil
+}
diff --git a/db/fs/fs.go b/db/fs/fs.go
@@ -72,18 +72,13 @@ func(fdb *fsDb) ToKey(ctx context.Context, key []byte) (db.LookupKey, error) {
}
func(fdb *fsDb) DecodeKey(ctx context.Context, key []byte) ([]byte, error) {
- var err error
- key, err = db.FromDbKey(key)
+ key, err := fdb.DbBase.DecodeKey(ctx, key)
if err != nil {
- return []byte{}, err
+ return nil, err
}
if !fdb.binary {
return key, nil
}
- key, err = fdb.DbBase.FromSessionKey(key)
- if err != nil {
- return []byte{}, err
- }
oldKey := key
key, err = base64.StdEncoding.DecodeString(string(key))
if err != nil {
diff --git a/db/gdbm/dump.go b/db/gdbm/dump.go
@@ -12,7 +12,14 @@ import (
// TODO: userdata is hardcoded here, should not be
func(gdb *gdbmDb) Dump(ctx context.Context, key []byte) (*db.Dumper, error) {
- key = append([]byte{db.DATATYPE_USERDATA}, key...)
+ gdb.SetPrefix(db.DATATYPE_USERDATA)
+ gdb.SetLanguage(nil)
+ lk, err := gdb.ToKey(ctx, key)
+ if err != nil {
+ return nil, err
+ }
+ key = lk.Default
+
gdb.it = gdb.conn.Iterator()
for true {
k, err := gdb.it()
@@ -23,18 +30,22 @@ func(gdb *gdbmDb) Dump(ctx context.Context, key []byte) (*db.Dumper, error) {
gdb.it = nil
return nil, err
}
- logg.TraceCtxf(ctx, "dump trace", "k", k, "key", key)
if !bytes.HasPrefix(k, key) {
continue
}
- gdb.SetPrefix(k[0])
- v, err := gdb.Get(ctx, k[1:])
+ //gdb.SetPrefix(k[0])
+ logg.TraceCtxf(ctx, "dump trace", "k", k, "key", key)
+ kk, err := gdb.DecodeKey(ctx, k)
+ if err != nil {
+ return nil, err
+ }
+ v, err := gdb.Get(ctx, kk)
if err != nil {
gdb.it = nil
return nil, err
}
gdb.itBase = key
- return db.NewDumper(gdb.dumpFunc).WithFirst(k[1:], v), nil
+ return db.NewDumper(gdb.dumpFunc).WithFirst(kk, v), nil
}
gdb.it = nil
return nil, db.NewErrNotFound(key)
@@ -61,12 +72,17 @@ func(gdb *gdbmDb) dumpFunc(ctx context.Context) ([]byte, []byte) {
gdb.it = nil
return nil, nil
}
- gdb.SetPrefix(k[0])
- v, err := gdb.Get(ctx, k[1:])
+ //gdb.SetPrefix(k[0])
+ logg.TraceCtxf(ctx, "gdbm dump func", "key", k)
+ kk, err := gdb.DecodeKey(ctx, k)
+ if err != nil {
+ return nil, nil
+ }
+ v, err := gdb.Get(ctx, kk)
if err != nil {
return nil, nil
}
- return k[1:], v
+ return kk, v
}
//func(gdb *gdbmDb) After(ctx context.Context, keyPart []byte) ([]byte, []byte) {
diff --git a/db/gdbm/dump_test.go b/db/gdbm/dump_test.go
@@ -45,14 +45,16 @@ func TestDumpGdbm(t *testing.T) {
t.Fatal(err)
}
k, v := o.Next(ctx)
- if !bytes.Equal(k, append([]byte{db.DATATYPE_USERDATA}, []byte("foobar")...)) {
- t.Fatalf("expected key 'foobar', got %s", k)
+ //if !bytes.Equal(k, append([]byte{db.DATATYPE_USERDATA}, []byte("foobar")...)) {
+ if !bytes.Equal(k, []byte("foobar")) {
+ t.Fatalf("expected key 'foobar', got '%s'", k)
}
if !bytes.Equal(v, []byte("pinky")) {
t.Fatalf("expected val 'pinky', got %s", v)
}
k, v = o.Next(ctx)
- if !bytes.Equal(k, append([]byte{db.DATATYPE_USERDATA}, []byte("foobarbaz")...)) {
+ //if !bytes.Equal(k, append([]byte{db.DATATYPE_USERDATA}, []byte("foobarbaz")...)) {
+ if !bytes.Equal(k, []byte("foobarbaz")) {
t.Fatalf("expected key 'foobarbaz', got %s", k)
}
if !bytes.Equal(v, []byte("blinky")) {
diff --git a/db/postgres/dump.go b/db/postgres/dump.go
@@ -13,7 +13,13 @@ func(pdb *pgDb) Dump(ctx context.Context, key []byte) (*db.Dumper, error) {
return nil, err
}
- k := append([]byte{db.DATATYPE_USERDATA}, key...)
+ pdb.SetPrefix(db.DATATYPE_USERDATA)
+ pdb.SetLanguage(nil)
+ lk, err := pdb.ToKey(ctx, key)
+ if err != nil {
+ return nil, err
+ }
+ k := lk.Default
query := fmt.Sprintf("SELECT key, value FROM %s.kv_vise WHERE key >= $1", pdb.schema)
logg.TraceCtxf(ctx, "getkey", "q", query, "key", k)
@@ -23,29 +29,56 @@ func(pdb *pgDb) Dump(ctx context.Context, key []byte) (*db.Dumper, error) {
tx.Rollback(ctx)
return nil, err
}
+ defer tx.Commit(ctx)
//defer rs.Close()
if rs.Next() {
- r := rs.RawValues()
+ var kk []byte
+ var vv []byte
+// r, err := rs.Values()
+// if err != nil {
+// return nil, err
+// }
+ err = rs.Scan(&kk, &vv)
+ if err != nil {
+ return nil, err
+ }
//tx.Rollback(ctx)
- tx.Commit(ctx)
+ //tx.Commit(ctx)
pdb.it = rs
pdb.itBase = k
- return db.NewDumper(pdb.dumpFunc).WithClose(pdb.closeFunc).WithFirst(r[0][1:], r[1]), nil
+ kk, err = pdb.DecodeKey(ctx, kk)
+ logg.Debugf("pg decode", "k", kk, "o", k, "err", err, "vv", vv)
+ if err != nil {
+ return nil, err
+ }
+ return db.NewDumper(pdb.dumpFunc).WithClose(pdb.closeFunc).WithFirst(kk, vv), nil
}
return nil, db.NewErrNotFound(k)
}
func(pdb *pgDb) dumpFunc(ctx context.Context) ([]byte, []byte) {
+ var kk []byte
+ var vv []byte
if !pdb.it.Next() {
logg.DebugCtxf(ctx, "no more data in pg iterator")
pdb.it = nil
pdb.itBase = nil
return nil, nil
}
- r := pdb.it.RawValues()
- return r[0][1:], r[1]
+ err := pdb.it.Scan(&kk, &vv)
+ if err != nil {
+ return nil, nil
+ }
+ //r := pdb.it.RawValues()
+ //k, err := pdb.DecodeKey(ctx, r[0])
+ k, err := pdb.DecodeKey(ctx, kk)
+ if err != nil {
+ return nil, nil
+ }
+ logg.Debugf("pg decode dump", "k", kk, "o", k, "err", err, "vv", vv)
+ return k, vv
}
func(pdb *pgDb) closeFunc() error {
diff --git a/db/postgres/dump_test.go b/db/postgres/dump_test.go
@@ -3,8 +3,6 @@ package postgres
import (
"bytes"
"context"
- "encoding/base64"
- "strings"
"testing"
pgxmock "github.com/pashagolub/pgxmock/v4"
@@ -47,7 +45,7 @@ func TestDumpPg(t *testing.T) {
// }
typMap := pgtype.NewMap()
- k := []byte("foo")
+ k := []byte("xyzzy.foo")
mockVfd := pgconn.FieldDescription{
Name: "value",
DataTypeOID: pgtype.ByteaOID,
@@ -60,8 +58,8 @@ func TestDumpPg(t *testing.T) {
}
rows := pgxmock.NewRowsWithColumnDefinition(mockKfd, mockVfd)
//rows = rows.AddRow([]byte("bar"), []byte("inky"))
- rows = rows.AddRow([]byte("foobar"), []byte("pinky"))
- rows = rows.AddRow([]byte("foobarbaz"), []byte("blinky"))
+ rows = rows.AddRow(append([]byte{db.DATATYPE_USERDATA}, []byte("xyzzy.foobar")...), []byte("pinky"))
+ rows = rows.AddRow(append([]byte{db.DATATYPE_USERDATA}, []byte("xyzzy.foobarbaz")...), []byte("blinky"))
//rows = rows.AddRow([]byte("xyzzy"), []byte("clyde"))
mock.ExpectBegin()
@@ -73,15 +71,15 @@ func TestDumpPg(t *testing.T) {
t.Fatal(err)
}
k, _ = o.Next(ctx)
- br, err := base64.StdEncoding.DecodeString(strings.Trim(string(k), "\""))
- if !bytes.Equal(br, []byte("foobar")) {
- t.Fatalf("expected key 'foobar', got %s", br)
+ //br, err := base64.StdEncoding.DecodeString(strings.Trim(string(k), "\""))
+ if !bytes.Equal(k, []byte("foobar")) {
+ t.Fatalf("expected key 'foobar', got %x", k)
}
k, _ = o.Next(ctx)
- br, err = base64.StdEncoding.DecodeString(strings.Trim(string(k), "\""))
- if !bytes.Equal(br, []byte("foobarbaz")) {
- t.Fatalf("expected key 'foobarbaz', got %s", br)
+ //br, err = base64.StdEncoding.DecodeString(strings.Trim(string(k), "\""))
+ if !bytes.Equal(k, []byte("foobarbaz")) {
+ t.Fatalf("expected key 'foobarbaz', got %x", k)
}
k, _ = o.Next(ctx)
