go-vise

commit 3b5fc85b650b75272d0410d4e35ed94c6de85e14
parent 927cb8b75a97c335f16894d3ba1ffe8cefe76e3c
Author: lash <dev@holbrook.no>
Date:   Tue, 14 Jan 2025 22:51:17 +0000

Handle binary keys in fs

Diffstat:
M
db/db.go
 | 
28
++++++++++++++++++++++++++++
M
db/fs/dump.go
 | 
34
+++++++++++++++++++++++++---------
M
db/fs/dump_test.go
 | 
102
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-
M
db/fs/fs.go
 | 
39
+++++++++++++++++++++++++++++++++++++++

4 files changed, 193 insertions(+), 10 deletions(-)
diff --git a/db/db.go b/db/db.go
@@ -1,8 +1,10 @@
 package db
 
 import (
+	"bytes"
 	"context"
 	"errors"
+	"fmt"
 
 	"git.defalsify.org/vise.git/lang"
 )
@@ -97,6 +99,22 @@ func ToDbKey(typ uint8, b []byte, l *lang.Language) []byte {
 	return append(k, b...)
 }
 
+func FromDbKey(b []byte) ([]byte, error) {
+	if len(b) < 2 {
+		return nil, fmt.Errorf("invalid db key")
+	}
+	typ := b[0]
+	b = b[1:]
+	if typ & (DATATYPE_MENU | DATATYPE_TEMPLATE | DATATYPE_STATICLOAD) > 0 {
+		if len(b) > 6 {
+			if b[len(b)-4] == '_' {
+				b = b[:len(b)-4]
+			}
+		}
+	}
+	return b, nil
+}
+
 // baseDb is a base class for all Db implementations.
 type baseDb struct {
 	pfx uint8
@@ -182,6 +200,16 @@ func ToSessionKey(pfx uint8, sessionId []byte, key []byte) []byte {
 	return b
 }
 
+func(bd *DbBase) FromSessionKey(key []byte) ([]byte, error) {
+	if len(bd.baseDb.sid) == 0 {
+		return key, nil
+	}
+	if !bytes.HasPrefix(key, bd.baseDb.sid) {
+		return nil, fmt.Errorf("session id prefix %s does not match key", string(bd.baseDb.sid))
+	}
+	return bytes.TrimPrefix(key, bd.baseDb.sid), nil
+}
+
 // ToKey creates a DbKey within the current session context.
 //
 // TODO: hard to read, clean up
diff --git a/db/fs/dump.go b/db/fs/dump.go
@@ -24,15 +24,21 @@ func(fdb *fsDb) Dump(ctx context.Context, key []byte) (*db.Dumper, error) {
 			s := v.Name()
 			k := []byte(s)
 			k[0] -= 0x30
-			vv, err := fdb.Get(ctx, k)
+			kk, err := fdb.DecodeKey(ctx, k)
 			if err != nil {
 				return nil, err
 			}
-			return db.NewDumper(fdb.dumpFunc).WithFirst(k, vv), nil
+			vv, err := fdb.Get(ctx, kk)
+			if err != nil {
+				return nil, err
+			}
+			kk = append([]byte{k[0]}, kk...)
+			return db.NewDumper(fdb.dumpFunc).WithFirst(kk, vv), nil
 		}
 	}
 	for len(fdb.elements) > 0 {
 		v := fdb.elements[0]
+		logg.TraceCtxf(ctx, "el", "v", v)
 		fdb.elements = fdb.elements[1:]
 		s := v.Name()
 		k := []byte(s)
@@ -40,12 +46,18 @@ func(fdb *fsDb) Dump(ctx context.Context, key []byte) (*db.Dumper, error) {
 			continue
 		}
 		k[0] -= 0x30
-		if bytes.HasPrefix(k, key) {
-			vv, err := fdb.Get(ctx, k[1:])
+		kk, err := fdb.DecodeKey(ctx, k)
+		if err != nil {
+			return nil, err
+		}
+		kkk := append([]byte{k[0]}, kk...)
+		if bytes.HasPrefix(kkk, key) {
+			vv, err := fdb.Get(ctx, kk)
 			if err != nil {
 				return nil, err
 			}
-			return db.NewDumper(fdb.dumpFunc).WithFirst(k, vv), nil
+			kk = append([]byte{k[0]}, kk...)
+			return db.NewDumper(fdb.dumpFunc).WithFirst(kk, vv), nil
 		}
 	}
 	return nil, db.NewErrNotFound(key)
@@ -60,13 +72,17 @@ func(fdb *fsDb) dumpFunc(ctx context.Context) ([]byte, []byte) {
 	s := v.Name()
 	k := []byte(s)
 	k[0] -= 0x30
-	if bytes.HasPrefix(k, fdb.matchPrefix) {
-		vv, err := fdb.Get(ctx, k[1:])
+	kk, err := fdb.DecodeKey(ctx, k)
+	if err != nil {
+		return nil, nil
+	}
+	kkk := append([]byte{k[0]}, kk...)
+	if bytes.HasPrefix(kkk, fdb.matchPrefix) {
+		vv, err := fdb.Get(ctx, kk)
 		if err != nil {
-			logg.ErrorCtxf(ctx, "failed to get entry", "key", k)
 			return nil, nil
 		}
-		return k, vv
+		return kkk, vv
 	}
 	return nil, nil
 }
diff --git a/db/fs/dump_test.go b/db/fs/dump_test.go
@@ -63,6 +63,106 @@ func TestDumpFs(t *testing.T) {
 		t.Fatalf("expected nil, got %s", k)
 	}
 }
-func TestDump(t *testing.T) {
 
+func TestDumpBinary(t *testing.T) {
+	ctx := context.Background()
+
+	store := NewFsDb()
+	store = store.WithBinary()
+	d, err := ioutil.TempDir("", "vise-db-fsbin-*")
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = store.Connect(ctx, d)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	store.SetPrefix(db.DATATYPE_USERDATA)
+	err = store.Put(ctx, []byte{0x01, 0x02, 0x03}, []byte("inky"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = store.Put(ctx, []byte{0x01, 0x02, 0x04}, []byte("pinky"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = store.Put(ctx, []byte{0x02, 0x03, 0x04}, []byte("blinky"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	o, err := store.Dump(ctx, []byte{0x01})
+	if err != nil {
+		t.Fatal(err)
+	}
+	k, v := o.Next(ctx)
+	if !bytes.Equal(k, append([]byte{db.DATATYPE_USERDATA}, []byte{0x01, 0x02, 0x03}...)) {
+		t.Fatalf("expected key '0x010203', got %x", k)
+	}
+	if !bytes.Equal(v, []byte("inky")) {
+		t.Fatalf("expected val 'inky', got %s", v)
+	}
+	k, v = o.Next(ctx)
+	if !bytes.Equal(k, append([]byte{db.DATATYPE_USERDATA}, []byte{0x01, 0x02, 0x04}...)) {
+		t.Fatalf("expected key '0x010204', got %x", k)
+	}
+	if !bytes.Equal(v, []byte("pinky")) {
+		t.Fatalf("expected val 'pinky', got %s", v)
+	}
+	k, v = o.Next(ctx)
+	if k != nil {
+		t.Fatalf("expected nil, got %s", k)
+	}
+}
+
+func TestDumpSessionBinary(t *testing.T) {
+	ctx := context.Background()
+
+	store := NewFsDb()
+	store = store.WithBinary()
+	store.SetSession("foobar")
+	d, err := ioutil.TempDir("", "vise-db-fsbin-*")
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = store.Connect(ctx, d)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	store.SetPrefix(db.DATATYPE_USERDATA)
+	err = store.Put(ctx, []byte{0x01, 0x02, 0x03}, []byte("inky"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = store.Put(ctx, []byte{0x01, 0x02, 0x04}, []byte("pinky"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = store.Put(ctx, []byte{0x02, 0x03, 0x04}, []byte("blinky"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	o, err := store.Dump(ctx, []byte{0x01})
+	if err != nil {
+		t.Fatal(err)
+	}
+	k, v := o.Next(ctx)
+	if !bytes.Equal(k, append([]byte{db.DATATYPE_USERDATA}, []byte{0x01, 0x02, 0x03}...)) {
+		t.Fatalf("expected key '0x010203', got %x", k)
+	}
+	if !bytes.Equal(v, []byte("inky")) {
+		t.Fatalf("expected val 'inky', got %s", v)
+	}
+	k, v = o.Next(ctx)
+	if !bytes.Equal(k, append([]byte{db.DATATYPE_USERDATA}, []byte{0x01, 0x02, 0x04}...)) {
+		t.Fatalf("expected key '0x010204', got %x", k)
+	}
+	if !bytes.Equal(v, []byte("pinky")) {
+		t.Fatalf("expected val 'pinky', got %s", v)
+	}
+	k, v = o.Next(ctx)
+	if k != nil {
+		t.Fatalf("expected nil, got %s", k)
+	}
 }
diff --git a/db/fs/fs.go b/db/fs/fs.go
@@ -3,6 +3,7 @@ package fs
 import (
 	"context"
 	"errors"
+	"encoding/base64"
 	"io/fs"
 	"io/ioutil"
 	"os"
@@ -23,6 +24,7 @@ type fsDb struct {
 	dir string
 	elements []os.DirEntry
 	matchPrefix []byte
+	binary bool
 }
 
 
@@ -34,6 +36,11 @@ func NewFsDb() *fsDb {
 	return db
 }
 
+func(fdb *fsDb) WithBinary() *fsDb {
+	fdb.binary = true
+	return fdb
+}
+
 // String implements the string interface.
 func(fdb *fsDb) String() string {
 	return "fsdb: " + fdb.dir
@@ -53,6 +60,38 @@ func(fdb *fsDb) Connect(ctx context.Context, connStr string) error {
 	return nil
 }
 
+// ToKey overrides the BaseDb implementation, creating a base64 string
+// if binary keys have been enabled
+func(fdb *fsDb) ToKey(ctx context.Context, key []byte) (db.LookupKey, error) {
+	if fdb.binary {
+		s := base64.StdEncoding.EncodeToString(key)
+		key = []byte(s)
+	}
+	return fdb.DbBase.ToKey(ctx, key)
+}
+
+func(fdb *fsDb) DecodeKey(ctx context.Context, key []byte) ([]byte, error) {
+	var err error
+	key, err = db.FromDbKey(key)
+	if err != nil {
+		return []byte{}, err
+	}
+	if !fdb.binary {
+		return key, nil
+	}
+	key, err = fdb.DbBase.FromSessionKey(key)
+	if err != nil {
+		return []byte{}, err
+	}
+	oldKey := key
+	key, err = base64.StdEncoding.DecodeString(string(key))
+	if err != nil {
+		return []byte{}, err
+	}
+	logg.TraceCtxf(ctx, "decoding base64 key", "base64", oldKey, "bin", key)
+	return key, nil
+}
+
 // Get implements the Db interface.
 func(fdb *fsDb) Get(ctx context.Context, key []byte) ([]byte, error) {
 	var f *os.File