go-vise

Constrained Size Output Virtual Machine
Info | Log | Files | Refs | README | LICENSE

commit 934a8ba440738d25567015423d72409fdf04495c
parent 61d1d00abf7794c1b7e77defbcc1d0c9d55d25cc
Author: lash <dev@holbrook.no>
Date:   Thu, 17 Oct 2024 12:21:30 +0100

Merge branch 'lash/postgres-kv-date'

Diffstat:
Adb/postgres/live_test.go | 67+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Mdb/postgres/pg.go | 64++++++++++++++++++++++++++++++++++++++--------------------------
Mdb/postgres/pg_test.go | 86+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++------------
Mgo.mod | 15+++++++++------
Mgo.sum | 30+++++++++++++++++++-----------
5 files changed, 206 insertions(+), 56 deletions(-)

diff --git a/db/postgres/live_test.go b/db/postgres/live_test.go @@ -0,0 +1,67 @@ +//go:build testlive +package postgres + +import ( + "bytes" + "context" + "testing" + + "git.defalsify.org/vise.git/db" + "git.defalsify.org/vise.git/db/dbtest" +) + +func TestLiveCasesPg(t *testing.T) { + ctx := context.Background() + + store := NewPgDb().WithSchema("vvise") + + err := store.Connect(ctx, "postgres://vise:esiv@localhost:5432/visedb") + if err != nil { + t.Fatal(err) + } + + err = dbtest.RunTests(t, ctx, store) + if err != nil { + t.Fatal(err) + } +} + +func TestLivePutGetPg(t *testing.T) { + var dbi db.Db + ses := "xyzzy" + store := NewPgDb().WithSchema("vvise") + store.SetPrefix(db.DATATYPE_USERDATA) + store.SetSession(ses) + ctx := context.Background() + + dbi = store + _ = dbi + + err := store.Connect(ctx, "postgres://vise:esiv@localhost:5432/visedb") + if err != nil { + t.Fatal(err) + } + err = store.Put(ctx, []byte("foo"), []byte("bar")) + if err != nil { + t.Fatal(err) + } + b, err := store.Get(ctx, []byte("foo")) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(b, []byte("bar")) { + t.Fatalf("expected 'bar', got %x", b) + } + err = store.Put(ctx, []byte("foo"), []byte("plugh")) + if err != nil { + t.Fatal(err) + } + b, err = store.Get(ctx, []byte("foo")) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(b, []byte("plugh")) { + t.Fatalf("expected 'plugh', got %x", b) + } + +} diff --git a/db/postgres/pg.go b/db/postgres/pg.go @@ -6,16 +6,26 @@ import ( "fmt" "github.com/jackc/pgx/v5/pgxpool" + pgx "github.com/jackc/pgx/v5" "git.defalsify.org/vise.git/db" ) +var ( + defaultTxOptions pgx.TxOptions +) + +type PgInterface interface { + BeginTx(context.Context, pgx.TxOptions) (pgx.Tx, error) +} + // pgDb is a Postgres backend implementation of the Db interface. type pgDb struct { *db.DbBase - conn *pgxpool.Pool + conn PgInterface schema string prefix uint8 + prepd bool } // NewpgDb creates a new Postgres backed Db implementation. @@ -33,6 +43,11 @@ func(pdb *pgDb) WithSchema(schema string) *pgDb { return pdb } +func(pdb *pgDb) WithConnection(pi PgInterface) *pgDb { + pdb.conn = pi + return pdb +} + // Connect implements Db. func(pdb *pgDb) Connect(ctx context.Context, connStr string) error { if pdb.conn != nil { @@ -45,7 +60,7 @@ func(pdb *pgDb) Connect(ctx context.Context, connStr string) error { return err } pdb.conn = conn - return pdb.prepare(ctx) + return pdb.Prepare(ctx) } // Put implements Db. @@ -59,15 +74,11 @@ func(pdb *pgDb) Put(ctx context.Context, key []byte, val []byte) error { return err } - tx, err := pdb.conn.Begin(ctx) + tx, err := pdb.conn.BeginTx(ctx, defaultTxOptions) if err != nil { return err } - defer tx.Rollback(ctx) - - query := fmt.Sprintf("INSERT INTO %s.kv_vise (key, value) VALUES ($1, $2) ON CONFLICT(key) DO UPDATE SET value = $2;", pdb.schema) - - // Use lk.Default if Translation is nil, otherwise use Translation + query := fmt.Sprintf("INSERT INTO %s.kv_vise (key, value, updated) VALUES ($1, $2, 'now') ON CONFLICT(key) DO UPDATE SET value = $2, updated = 'now';", pdb.schema) actualKey := lk.Default if lk.Translation != nil { actualKey = lk.Translation @@ -88,18 +99,16 @@ func (pdb *pgDb) Get(ctx context.Context, key []byte) ([]byte, error) { return nil, err } - query := fmt.Sprintf("SELECT value FROM %s.kv_vise WHERE key = $1", pdb.schema) + tx, err := pdb.conn.BeginTx(ctx, defaultTxOptions) + if err != nil { + return nil, err + } - // Try Translation key if it exists if lk.Translation != nil { - tx, err := pdb.conn.Begin(ctx) - if err != nil { - return nil, err - } - defer tx.Rollback(ctx) - + query := fmt.Sprintf("SELECT value FROM %s.kv_vise WHERE key = $1", pdb.schema) rs, err := tx.Query(ctx, query, lk.Translation) if err != nil { + tx.Rollback(ctx) return nil, err } defer rs.Close() @@ -107,24 +116,21 @@ func (pdb *pgDb) Get(ctx context.Context, key []byte) ([]byte, error) { if rs.Next() { r := rs.RawValues() tx.Commit(ctx) + tx.Rollback(ctx) return r[0], nil } } - // If Translation key doesn't exist or we didn't find a value, try Default key - tx, err := pdb.conn.Begin(ctx) - if err != nil { - return nil, err - } - defer tx.Rollback(ctx) - + query := fmt.Sprintf("SELECT value FROM %s.kv_vise WHERE key = $1", pdb.schema) rs, err := tx.Query(ctx, query, lk.Default) if err != nil { + tx.Rollback(ctx) return nil, err } defer rs.Close() if !rs.Next() { + tx.Rollback(ctx) return nil, db.NewErrNotFound(key) } @@ -140,8 +146,13 @@ func(pdb *pgDb) Close() error { } // set up table -func(pdb *pgDb) prepare(ctx context.Context) error { - tx, err := pdb.conn.Begin(ctx) +func(pdb *pgDb) Prepare(ctx context.Context) error { + if pdb.prepd { + logg.WarnCtxf(ctx, "Prepare called more than once") + return nil + } + pdb.prepd = true + tx, err := pdb.conn.BeginTx(ctx, defaultTxOptions) if err != nil { tx.Rollback(ctx) return err @@ -149,7 +160,8 @@ func(pdb *pgDb) prepare(ctx context.Context) error { query := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s.kv_vise ( id SERIAL NOT NULL, key BYTEA NOT NULL UNIQUE, - value BYTEA NOT NULL + value BYTEA NOT NULL, + updated TIMESTAMP NOT NULL ); `, pdb.schema) _, err = tx.Exec(ctx, query) diff --git a/db/postgres/pg_test.go b/db/postgres/pg_test.go @@ -3,8 +3,14 @@ package postgres import ( "bytes" "context" + "encoding/base64" + "strings" "testing" + pgxmock "github.com/pashagolub/pgxmock/v4" + "github.com/jackc/pgx/v5/pgtype" + "github.com/jackc/pgx/v5/pgconn" + "git.defalsify.org/vise.git/db" "git.defalsify.org/vise.git/db/dbtest" ) @@ -12,13 +18,15 @@ import ( func TestCasesPg(t *testing.T) { ctx := context.Background() - store := NewPgDb().WithSchema("public") - //t.Skip("need postgresql mock") + t.Skip("implement expects in all cases") - err := store.Connect(ctx, "postgres://postgres:rooter@localhost:5432/visedb") + mock, err := pgxmock.NewPool() if err != nil { t.Fatal(err) } + defer mock.Close() + + store := NewPgDb().WithConnection(mock).WithSchema("vvise") err = dbtest.RunTests(t, ctx, store) if err != nil { @@ -29,7 +37,14 @@ func TestCasesPg(t *testing.T) { func TestPutGetPg(t *testing.T) { var dbi db.Db ses := "xyzzy" - store := NewPgDb().WithSchema("public") + + mock, err := pgxmock.NewPool() + if err != nil { + t.Fatal(err) + } + defer mock.Close() + + store := NewPgDb().WithConnection(mock).WithSchema("vvise") store.SetPrefix(db.DATATYPE_USERDATA) store.SetSession(ses) ctx := context.Background() @@ -37,32 +52,77 @@ func TestPutGetPg(t *testing.T) { dbi = store _ = dbi - // t.Skip("need postgresql mock") - err := store.Connect(ctx, "postgres://postgres:rooter@localhost:5432/visedb") + typMap := pgtype.NewMap() + + k := []byte("foo") + ks := append([]byte{db.DATATYPE_USERDATA}, []byte("xyzzy.foo")...) + v := []byte("bar") + resInsert := pgxmock.NewResult("UPDATE", 1) + mock.ExpectBegin() + mock.ExpectExec("INSERT INTO vvise.kv_vise").WithArgs(ks, v).WillReturnResult(resInsert) + mock.ExpectCommit() + err = store.Put(ctx, k, v) if err != nil { t.Fatal(err) } - err = store.Put(ctx, []byte("foo"), []byte("bar")) + + mockVfd := pgconn.FieldDescription{ + Name: "value", + DataTypeOID: pgtype.ByteaOID, + Format: typMap.FormatCodeForOID(pgtype.ByteaOID), + } + row := pgxmock.NewRowsWithColumnDefinition(mockVfd) + row = row.AddRow(v) + mock.ExpectBegin() + mock.ExpectQuery("SELECT value FROM vvise.kv_vise").WithArgs(ks).WillReturnRows(row) + mock.ExpectCommit() + b, err := store.Get(ctx, k) if err != nil { t.Fatal(err) } - b, err := store.Get(ctx, []byte("foo")) + + // TODO: implement as pgtype map instead, and btw also ask why getting base64 here + br, err := base64.StdEncoding.DecodeString(strings.Trim(string(b), "\"")) if err != nil { t.Fatal(err) } - if !bytes.Equal(b, []byte("bar")) { + if !bytes.Equal(br, v) { t.Fatalf("expected 'bar', got %x", b) } - err = store.Put(ctx, []byte("foo"), []byte("plugh")) + + v = []byte("plugh") + mock.ExpectBegin() + mock.ExpectExec("INSERT INTO vvise.kv_vise").WithArgs(ks, v).WillReturnResult(resInsert) + mock.ExpectCommit() + err = store.Put(ctx, k, v) + if err != nil { + t.Fatal(err) + } + + mock.ExpectBegin() + mock.ExpectExec("INSERT INTO vvise.kv_vise").WithArgs(ks, v).WillReturnResult(resInsert) + mock.ExpectCommit() + err = store.Put(ctx, k, v) if err != nil { t.Fatal(err) } - b, err = store.Get(ctx, []byte("foo")) + + row = pgxmock.NewRowsWithColumnDefinition(mockVfd) + row = row.AddRow(v) + mock.ExpectBegin() + mock.ExpectQuery("SELECT value FROM vvise.kv_vise").WithArgs(ks).WillReturnRows(row) + mock.ExpectCommit() + b, err = store.Get(ctx, k) + if err != nil { + t.Fatal(err) + } + + br, err = base64.StdEncoding.DecodeString(strings.Trim(string(b), "\"")) if err != nil { t.Fatal(err) } - if !bytes.Equal(b, []byte("plugh")) { - t.Fatalf("expected 'plugh', got %x", b) + if !bytes.Equal(br, v) { + t.Fatalf("expected 'plugh', got %x", br) } } diff --git a/go.mod b/go.mod @@ -1,24 +1,27 @@ module git.defalsify.org/vise.git -go 1.20 +go 1.22.0 + +toolchain go1.23.1 require ( github.com/alecthomas/participle/v2 v2.0.0 github.com/barbashov/iso639-3 v0.0.0-20211020172741-1f4ffb2d8d1c github.com/fxamacker/cbor/v2 v2.4.0 github.com/graygnuorg/go-gdbm v0.0.0-20220711140707-71387d66dce4 - github.com/jackc/pgx/v5 v5.6.0 + github.com/jackc/pgx/v5 v5.7.0 + github.com/pashagolub/pgxmock/v4 v4.3.0 github.com/peteole/testdata-loader v0.3.0 gopkg.in/leonelquinteros/gotext.v1 v1.3.1 ) require ( github.com/jackc/pgpassfile v1.0.0 // indirect - github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.1 // indirect github.com/mattn/kinako v0.0.0-20170717041458-332c0a7e205a // indirect github.com/x448/float16 v0.8.4 // indirect - golang.org/x/crypto v0.17.0 // indirect - golang.org/x/sync v0.1.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/crypto v0.27.0 // indirect + golang.org/x/sync v0.8.0 // indirect + golang.org/x/text v0.18.0 // indirect ) diff --git a/go.sum b/go.sum @@ -1,26 +1,32 @@ github.com/alecthomas/assert/v2 v2.2.2 h1:Z/iVC0xZfWTaFNE6bA3z07T86hd45Xe2eLt6WVy2bbk= +github.com/alecthomas/assert/v2 v2.2.2/go.mod h1:pXcQ2Asjp247dahGEmsZ6ru0UVwnkhktn7S0bBDLxvQ= github.com/alecthomas/participle/v2 v2.0.0 h1:Fgrq+MbuSsJwIkw3fEj9h75vDP0Er5JzepJ0/HNHv0g= github.com/alecthomas/participle/v2 v2.0.0/go.mod h1:rAKZdJldHu8084ojcWevWAL8KmEU+AT+Olodb+WoN2Y= github.com/alecthomas/repr v0.2.0 h1:HAzS41CIzNW5syS8Mf9UwXhNH1J9aix/BvDRf1Ml2Yk= +github.com/alecthomas/repr v0.2.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= github.com/barbashov/iso639-3 v0.0.0-20211020172741-1f4ffb2d8d1c h1:H9Nm+I7Cg/YVPpEV1RzU3Wq2pjamPc/UtHDgItcb7lE= github.com/barbashov/iso639-3 v0.0.0-20211020172741-1f4ffb2d8d1c/go.mod h1:rGod7o6KPeJ+hyBpHfhi4v7blx9sf+QsHsA7KAsdN6U= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fxamacker/cbor/v2 v2.4.0 h1:ri0ArlOR+5XunOP8CRUowT0pSJOwhW098ZCUyskZD88= github.com/fxamacker/cbor/v2 v2.4.0/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo= github.com/graygnuorg/go-gdbm v0.0.0-20220711140707-71387d66dce4 h1:U4kkNYryi/qfbBF8gh7Vsbuz+cVmhf5kt6pE9bYYyLo= github.com/graygnuorg/go-gdbm v0.0.0-20220711140707-71387d66dce4/go.mod h1:zpZDgZFzeq9s0MIeB1P50NIEWDFFHSFBohI/NbaTD/Y= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= +github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= -github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= -github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.6.0 h1:SWJzexBzPL5jb0GEsrPMLIsi/3jOo7RHlzTjcAeDrPY= -github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.7.0 h1:FG6VLIdzvAPhnYqP14sQ2xhFLkiUQHCs6ySqO91kF4g= +github.com/jackc/pgx/v5 v5.7.0/go.mod h1:awP1KNnjylvpxHuHP63gzjhnGkI1iw+PMoIwvoleN/8= github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/mattn/kinako v0.0.0-20170717041458-332c0a7e205a h1:0Q3H0YXzMHiciXtRcM+j0jiCe8WKPQHoRgQiRTnfcLY= github.com/mattn/kinako v0.0.0-20170717041458-332c0a7e205a/go.mod h1:CdTTBOYzS5E4mWS1N8NWP6AHI19MP0A2B18n3hLzRMk= +github.com/pashagolub/pgxmock/v4 v4.3.0 h1:DqT7fk0OCK6H0GvqtcMsLpv8cIwWqdxWgfZNLeHCb/s= +github.com/pashagolub/pgxmock/v4 v4.3.0/go.mod h1:9VoVHXwS3XR/yPtKGzwQvwZX1kzGB9sM8SviDcHDa3A= github.com/peteole/testdata-loader v0.3.0 h1:8jckE9KcyNHgyv/VPoaljvKZE0Rqr8+dPVYH6rfNr9I= github.com/peteole/testdata-loader v0.3.0/go.mod h1:Mt0ZbRtb56u8SLJpNP+BnQbENljMorYBpqlvt3cS83U= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -28,17 +34,19 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= -golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= -golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= +golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/leonelquinteros/gotext.v1 v1.3.1 h1:8d9/fdTG0kn/B7NNGV1BsEyvektXFAbkMsTZS2sFSCc= gopkg.in/leonelquinteros/gotext.v1 v1.3.1/go.mod h1:X1WlGDeAFIYsW6GjgMm4VwUwZ2XjI7Zan2InxSUQWrU= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=