commit b2eaf82eb98311e42ef88646db596a9846aa07af
parent 21449e347e37d58f5d42b7eea218e468512b552d
Author: lash <dev@holbrook.no>
Date: Mon, 7 Apr 2025 02:00:35 +0100
Add logdb
Diffstat:
8 files changed, 141 insertions(+), 13 deletions(-)
diff --git a/db/db.go b/db/db.go
@@ -127,6 +127,7 @@ type baseDb struct {
lang *lang.Language
seal bool
connStr string
+ known bool
logDb Db
}
@@ -140,12 +141,23 @@ type DbBase struct {
// NewDbBase instantiates a new DbBase.
func NewDbBase() *DbBase {
db := &DbBase{
- baseDb: &baseDb{},
+ baseDb: &baseDb{
+ known: true,
+ },
}
db.baseDb.defaultLock()
return db
}
+func (db *baseDb) AllowUnknownPrefix() bool {
+ known := db.known
+ if !known {
+ return false
+ }
+ db.known = false
+ return true
+}
+
// ensures default locking of read-only entries
func (db *baseDb) defaultLock() {
db.lock |= safeLock
@@ -229,7 +241,7 @@ func (bd *DbBase) ToKey(ctx context.Context, key []byte) (LookupKey, error) {
var lk LookupKey
//var b []byte
db := bd.baseDb
- if db.pfx == DATATYPE_UNKNOWN {
+ if db.known && db.pfx == DATATYPE_UNKNOWN {
return lk, errors.New("datatype prefix cannot be UNKNOWN")
}
//b := ToSessionKey(db.pfx, db.sid, key)
diff --git a/db/fs/fs.go b/db/fs/fs.go
@@ -37,7 +37,7 @@ func NewFsDb() *fsDb {
}
// Base implements Db
-func (fdb *fsDb) Base() *db.BaseDb {
+func (fdb *fsDb) Base() *db.DbBase {
return fdb.DbBase
}
diff --git a/db/gdbm/gdbm.go b/db/gdbm/gdbm.go
@@ -30,7 +30,7 @@ func NewGdbmDb() *gdbmDb {
}
// Base implements Db
-func (gdb *gdbmDb) Base() *db.BaseDb {
+func (gdb *gdbmDb) Base() *db.DbBase {
return gdb.DbBase
}
diff --git a/db/log/log.go b/db/log/log.go
@@ -19,10 +19,11 @@ type logDb struct {
logDb db.Db
}
-func NewLogDb(mainDb db.Db, db db.Db) db.Db {
+func NewLogDb(mainDb db.Db, subDb db.Db) db.Db {
+ subDb.Base().AllowUnknownPrefix()
return &logDb{
Db: mainDb,
- logDb: db,
+ logDb: subDb,
}
}
@@ -92,17 +93,20 @@ func (ldb *logDb) toLogDbEntry(ctx context.Context, key []byte, val []byte) ([]b
} else {
innerValKey = lk.Translation
}
+
+ l = make([]byte, 8)
binary.PutUvarint(l, uint64(len(innerValKey)))
innerValKey = append(l, innerValKey...)
- innerValKey = append(l, val...)
+ innerValKey = append(innerValKey, val...)
+ innerKey = make([]byte, 8)
t := time.Now().UnixNano()
binary.BigEndian.PutUint64(innerKey, uint64(t))
- innerKey = ldb.Base().ToSessionKey(db.DATATYPE_UNKNOWN, innerKey)
return innerKey, append(innerValKey, innerValVal...)
}
func (ldb *logDb) Put(ctx context.Context, key []byte, val []byte) error {
+ ldb.logDb.SetPrefix(db.DATATYPE_UNKNOWN)
err := ldb.Db.Put(ctx, key, val)
if err != nil {
return err
diff --git a/db/log/log_test.go b/db/log/log_test.go
@@ -1,11 +1,69 @@
package log
import (
+ "bytes"
"context"
+ "encoding/binary"
"testing"
+ "time"
+
+ "git.defalsify.org/vise.git/db"
+ "git.defalsify.org/vise.git/db/mem"
)
func TestLogDb(t *testing.T) {
+ sessionId := "xyzzy"
ctx := context.Background()
- _ = ctx
+ main := mem.NewMemDb()
+ sub := mem.NewMemDb()
+ store := NewLogDb(main, sub)
+ err := store.Connect(ctx, "main")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ k := []byte("foo")
+ v := []byte("bar")
+ tstart := time.Now()
+ store.SetPrefix(db.DATATYPE_USERDATA)
+ store.SetSession(sessionId)
+ err = store.Put(ctx, k, v)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ r, err := store.Get(ctx, k)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !bytes.Equal(r, v) {
+ t.Fatalf("Expected %x, got %x", v, r)
+ }
+
+ sub.SetPrefix(db.DATATYPE_UNKNOWN)
+ tend := time.Now()
+ dump, err := sub.Dump(ctx, append([]byte{db.DATATYPE_UNKNOWN}, []byte(sessionId)...))
+ if err != nil {
+ t.Fatal(err)
+ }
+ r, _ = dump.Next(ctx)
+ targetLen := len(sessionId) + 8 + 1 + 1
+ if len(r) != targetLen {
+ t.Fatalf("Unexpected length %d (%x), should be %d", len(r), r, targetLen)
+ }
+
+ k, err = sub.FromSessionKey(r[1:])
+ if err != nil {
+ t.Fatal(err)
+ }
+ tn := binary.BigEndian.Uint64(k)
+ tExpect := uint64(tstart.UnixNano())
+ if tn <= tExpect {
+ t.Fatalf("expected %d should be after %d", tn, tExpect)
+ }
+ tExpect = uint64(tend.UnixNano())
+ if tn >= tExpect {
+ t.Fatalf("expected %v should be before %v", tn, tExpect)
+ }
+
}
diff --git a/db/mem/dump.go b/db/mem/dump.go
@@ -1,12 +1,63 @@
package mem
import (
+ "bytes"
"context"
- "errors"
+ "encoding/hex"
+ "fmt"
+ "maps"
+ "slices"
"git.defalsify.org/vise.git/db"
)
func (mdb *memDb) Dump(ctx context.Context, key []byte) (*db.Dumper, error) {
- return nil, errors.New("unimplemented")
+ mdb.dumpKeys = slices.Sorted(maps.Keys(mdb.store))
+ mdb.dumpIdx = -1
+ for i := 0; i < len(mdb.dumpKeys); i++ {
+ s := mdb.dumpKeys[i]
+ k, err := hex.DecodeString(s)
+ if err != nil {
+ return nil, err
+ }
+ if bytes.HasPrefix(k, key) {
+ logg.DebugCtxf(ctx, "starting dump", "key", k)
+ mdb.dumpIdx = i
+ kk, err := mdb.Base().FromSessionKey(k[1:])
+ if err != nil {
+ return nil, fmt.Errorf("invalid dump key %x: %v", k, err)
+ }
+ v, err := mdb.Get(ctx, kk[:])
+ if err != nil {
+ return nil, fmt.Errorf("value err for key %x: %v", k, err)
+ }
+ return db.NewDumper(mdb.dumpFunc).WithFirst(k, v), nil
+ }
+ }
+ return nil, db.NewErrNotFound(key)
+}
+
+func (mdb *memDb) dumpFunc(ctx context.Context) ([]byte, []byte) {
+ if mdb.dumpIdx == -1 {
+ return nil, nil
+ }
+ if mdb.dumpIdx >= len(mdb.dumpKeys) {
+ mdb.dumpIdx = -1
+ return nil, nil
+ }
+ s := mdb.dumpKeys[mdb.dumpIdx]
+ k, err := hex.DecodeString(s)
+ if err != nil {
+ mdb.dumpIdx = -1
+ return nil, nil
+ }
+ kk, err := mdb.Base().FromSessionKey(k[1:])
+ if err != nil {
+ return nil, nil
+ }
+ v, err := mdb.Get(ctx, kk)
+ if err != nil {
+ return nil, nil
+ }
+ return k, v
}
diff --git a/db/mem/mem.go b/db/mem/mem.go
@@ -18,18 +18,21 @@ type memLookupKey struct {
type memDb struct {
*db.DbBase
store map[string][]byte
+ dumpIdx int
+ dumpKeys []string
}
// NewmemDb returns an in-process volatile Db implementation.
func NewMemDb() *memDb {
db := &memDb{
DbBase: db.NewDbBase(),
+ dumpIdx: -1,
}
return db
}
// Base implements Db
-func (mdb *memDb) Base() *db.BaseDb {
+func (mdb *memDb) Base() *db.DbBase {
return mdb.DbBase
}
diff --git a/db/postgres/pg.go b/db/postgres/pg.go
@@ -43,7 +43,7 @@ func NewPgDb() *pgDb {
}
// Base implements Db
-func (pdb *pgDb) Base() *db.BaseDb {
+func (pdb *pgDb) Base() *db.DbBase {
return pdb.DbBase
}
