From 7b89c8fe18a20ac5e10eb645f6f80b8fafc9a88b Mon Sep 17 00:00:00 2001 From: Matt Kocubinski Date: Mon, 4 Nov 2024 11:58:11 -0600 Subject: [PATCH 01/20] feat(store/v2): iavl/v2 wrapper --- store/v2/commitment/iavlv2/tree.go | 106 ++++++++++++++++ store/v2/go.mod | 16 ++- store/v2/go.sum | 26 ++-- store/v2/storage/sqlite/batch.go | 33 ++--- store/v2/storage/sqlite/db.go | 190 ++++++++++++++++++---------- store/v2/storage/sqlite/db_test.go | 4 +- store/v2/storage/sqlite/iterator.go | 44 +++---- 7 files changed, 304 insertions(+), 115 deletions(-) create mode 100644 store/v2/commitment/iavlv2/tree.go diff --git a/store/v2/commitment/iavlv2/tree.go b/store/v2/commitment/iavlv2/tree.go new file mode 100644 index 000000000000..a88567e23759 --- /dev/null +++ b/store/v2/commitment/iavlv2/tree.go @@ -0,0 +1,106 @@ +package iavlv2 + +import ( + "fmt" + + "github.com/cosmos/iavl/v2" + ics23 "github.com/cosmos/ics23/go" + + "cosmossdk.io/store/v2" + "cosmossdk.io/store/v2/commitment" +) + +var ( + _ commitment.Tree = (*Tree)(nil) + _ store.PausablePruner = (*Tree)(nil) +) + +type Tree struct { + tree *iavl.Tree +} + +func NewTree(treeOptions iavl.TreeOptions, dbOptions iavl.SqliteDbOptions, pool *iavl.NodePool) (*Tree, error) { + sql, err := iavl.NewSqliteDb(pool, dbOptions) + if err != nil { + return nil, err + } + tree := iavl.NewTree(sql, pool, treeOptions) + return &Tree{tree: tree}, nil +} + +func (t Tree) Set(key, value []byte) error { + _, err := t.tree.Set(key, value) + return err +} + +func (t Tree) Remove(key []byte) error { + _, _, err := t.tree.Remove(key) + return err +} + +func (t Tree) GetLatestVersion() (uint64, error) { + return uint64(t.tree.Version()), nil +} + +func (t Tree) Hash() []byte { + return t.tree.Hash() +} + +func (t Tree) Version() uint64 { + return uint64(t.tree.Version()) +} + +func (t Tree) LoadVersion(version uint64) error { + // TODO fix this in iavl v2 + if version == 0 { + return nil + } + return t.tree.LoadVersion(int64(version)) +} + +func (t Tree) Commit() ([]byte, uint64, error) { + h, v, err := t.tree.SaveVersion() + return h, uint64(v), err +} + +func (t Tree) SetInitialVersion(version uint64) error { + t.tree.SetShouldCheckpoint() + return t.tree.SetInitialVersion(int64(version)) +} + +func (t Tree) GetProof(version uint64, key []byte) (*ics23.CommitmentProof, error) { + //TODO implement me + panic("implement me") +} + +func (t Tree) Get(version uint64, key []byte) ([]byte, error) { + if int64(version) != t.tree.Version() { + return nil, fmt.Errorf("loading past version not yet supported") + } + return t.tree.Get(key) +} + +func (t Tree) Export(version uint64) (commitment.Exporter, error) { + //TODO implement me + panic("implement me") +} + +func (t Tree) Import(version uint64) (commitment.Importer, error) { + //TODO implement me + panic("implement me") +} + +func (t Tree) Close() error { + return t.tree.Close() +} + +func (t Tree) Prune(version uint64) error { + return t.tree.DeleteVersionsTo(int64(version)) +} + +// PausePruning is unnecessary in IAVL v2 due to the advanced pruning mechanism +func (t Tree) PausePruning(bool) {} + +func (t *Tree) WorkingHash() []byte { + return t.tree.Hash() +} diff --git a/store/v2/go.mod b/store/v2/go.mod index f906dfb1c4b2..2ed84be60137 100644 --- a/store/v2/go.mod +++ b/store/v2/go.mod @@ -7,15 +7,16 @@ require ( cosmossdk.io/core/testing v0.0.0-20240923163230-04da382a9f29 cosmossdk.io/errors/v2 v2.0.0-20240731132947-df72853b3ca5 cosmossdk.io/log v1.4.1 + github.com/bvinc/go-sqlite-lite v0.6.1 github.com/cockroachdb/pebble v1.1.0 github.com/cosmos/cosmos-proto v1.0.0-beta.5 github.com/cosmos/gogoproto v1.7.0 - github.com/cosmos/iavl v1.0.0-beta.1.0.20240813194616-eb5078efcf9e + github.com/cosmos/iavl v1.3.0 + github.com/cosmos/iavl/v2 v2.0.0-20241029195051-d3ac274b6b58 github.com/cosmos/ics23/go v0.11.0 - github.com/google/btree v1.1.2 + github.com/google/btree v1.1.3 github.com/hashicorp/go-metrics v0.5.3 github.com/linxGnu/grocksdb v1.9.3 - github.com/mattn/go-sqlite3 v1.14.22 github.com/spf13/cast v1.7.0 github.com/stretchr/testify v1.9.0 github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d @@ -26,6 +27,7 @@ require ( require ( cosmossdk.io/schema v0.3.0 // indirect github.com/DataDog/zstd v1.5.5 // indirect + github.com/aybabtme/uniplot v0.0.0-20151203143629-039c559e5e7e // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cockroachdb/errors v1.11.1 // indirect @@ -33,6 +35,7 @@ require ( github.com/cockroachdb/redact v1.1.5 // indirect github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/emicklei/dot v1.6.2 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/getsentry/sentry-go v0.27.0 // indirect @@ -43,6 +46,7 @@ require ( github.com/hashicorp/go-uuid v1.0.1 // indirect github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/klauspost/compress v1.17.9 // indirect + github.com/kocubinski/costor-api v1.1.1 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect @@ -64,3 +68,9 @@ require ( google.golang.org/protobuf v1.35.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) + +replace ( + cosmossdk.io/core => ../../core + github.com/cosmos/iavl => ../../../../iavl/iavl-master + github.com/cosmos/iavl/v2 => ../../../../iavl/iavl-v2 +) diff --git a/store/v2/go.sum b/store/v2/go.sum index e0f9db427cea..592a144de2dc 100644 --- a/store/v2/go.sum +++ b/store/v2/go.sum @@ -1,5 +1,3 @@ -cosmossdk.io/core v1.0.0-alpha.5 h1:McjYXAQ6XcT20v2uHyH7PhoWH8V+mebzfVFqT3GinsI= -cosmossdk.io/core v1.0.0-alpha.5/go.mod h1:3u9cWq1FAVtiiCrDPpo4LhR+9V6k/ycSG4/Y/tREWCY= cosmossdk.io/core/testing v0.0.0-20240923163230-04da382a9f29 h1:NxxUo0GMJUbIuVg0R70e3cbn9eFTEuMr7ev1AFvypdY= cosmossdk.io/core/testing v0.0.0-20240923163230-04da382a9f29/go.mod h1:8s2tPeJtSiQuoyPmr2Ag7meikonISO4Fv4MoO8+ORrs= cosmossdk.io/errors/v2 v2.0.0-20240731132947-df72853b3ca5 h1:IQNdY2kB+k+1OM2DvqFG1+UgeU1JzZrWtwuWzI3ZfwA= @@ -15,10 +13,14 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/aybabtme/uniplot v0.0.0-20151203143629-039c559e5e7e h1:dSeuFcs4WAJJnswS8vXy7YY1+fdlbVPuEVmDAfqvFOQ= +github.com/aybabtme/uniplot v0.0.0-20151203143629-039c559e5e7e/go.mod h1:uh71c5Vc3VNIplXOFXsnDy21T1BepgT32c5X/YPrOyc= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bvinc/go-sqlite-lite v0.6.1 h1:JU8Rz5YAOZQiU3WEulKF084wfXpytRiqD2IaW2QjPz4= +github.com/bvinc/go-sqlite-lite v0.6.1/go.mod h1:2GiE60NUdb0aNhDdY+LXgrqAVDpi2Ijc6dB6ZMp9x6s= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -44,8 +46,8 @@ github.com/cosmos/cosmos-proto v1.0.0-beta.5 h1:eNcayDLpip+zVLRLYafhzLvQlSmyab+R github.com/cosmos/cosmos-proto v1.0.0-beta.5/go.mod h1:hQGLpiIUloJBMdQMMWb/4wRApmI9hjHH05nefC0Ojec= github.com/cosmos/gogoproto v1.7.0 h1:79USr0oyXAbxg3rspGh/m4SWNyoz/GLaAh0QlCe2fro= github.com/cosmos/gogoproto v1.7.0/go.mod h1:yWChEv5IUEYURQasfyBW5ffkMHR/90hiHgbNgrtp4j0= -github.com/cosmos/iavl v1.0.0-beta.1.0.20240813194616-eb5078efcf9e h1:LEii0v/FxtXa/F7mRn+tijZ0zaXBPn2ZkKwb6Qm4rqE= -github.com/cosmos/iavl v1.0.0-beta.1.0.20240813194616-eb5078efcf9e/go.mod h1:3ywr0wDnWeD7MUH6qu50wZ5bxuKH3LBrGG4/lZX8lVY= +github.com/cosmos/iavl-bench/bench v0.0.4 h1:J6zQPiBqF4CXMM3QBsLqZgQEBGY0taX85vLIZMhmAfQ= +github.com/cosmos/iavl-bench/bench v0.0.4/go.mod h1:j2rLae77EffacWcp7mmj3Uaa4AOAmZA7ymvhsuBQKKI= github.com/cosmos/ics23/go v0.11.0 h1:jk5skjT0TqX5e5QJbEnwXIS2yI2vnmLOgpQPeM5RtnU= github.com/cosmos/ics23/go v0.11.0/go.mod h1:A8OjxPE67hHST4Icw94hOxxFEJMBG031xIGF/JHNIY0= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= @@ -53,6 +55,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/emicklei/dot v1.6.2 h1:08GN+DD79cy/tzN6uLCT84+2Wk9u+wvqP+Hkx/dIR8A= github.com/emicklei/dot v1.6.2/go.mod h1:DeV7GvQtIw4h2u73RKBkkFdvVAz0D9fzeJrgPW6gy/s= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= @@ -91,8 +95,8 @@ github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= -github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= +github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -116,6 +120,8 @@ github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iP github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= @@ -123,6 +129,8 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/kocubinski/costor-api v1.1.1 h1:sgfJA7T/8IfZ59zxiMrED0xdjerAFuPNBTqyO90GiEE= +github.com/kocubinski/costor-api v1.1.1/go.mod h1:ESMBMDkKfN+9vvvhhNVdKLhbOmzI3O/i16iXvRM9Tuc= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -140,8 +148,6 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/ github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= -github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -206,6 +212,10 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w= github.com/spf13/cast v1.7.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= +github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= diff --git a/store/v2/storage/sqlite/batch.go b/store/v2/storage/sqlite/batch.go index 783b597e04af..ca4001200c97 100644 --- a/store/v2/storage/sqlite/batch.go +++ b/store/v2/storage/sqlite/batch.go @@ -1,8 +1,10 @@ package sqlite import ( - "database/sql" "fmt" + "sync" + + "github.com/bvinc/go-sqlite-lite/sqlite3" "cosmossdk.io/store/v2" ) @@ -23,24 +25,24 @@ type batchOp struct { } type Batch struct { - db *sql.DB - tx *sql.Tx + db *sqlite3.Conn + lock *sync.Mutex ops []batchOp size int - version uint64 + version int64 } -func NewBatch(db *sql.DB, version uint64) (*Batch, error) { - tx, err := db.Begin() +func NewBatch(db *sqlite3.Conn, writeLock *sync.Mutex, version uint64) (*Batch, error) { + err := db.Begin() if err != nil { return nil, fmt.Errorf("failed to create SQL transaction: %w", err) } return &Batch{ db: db, - tx: tx, + lock: writeLock, ops: make([]batchOp, 0), - version: version, + version: int64(version), }, nil } @@ -53,12 +55,11 @@ func (b *Batch) Reset() error { b.ops = make([]batchOp, 0) b.size = 0 - tx, err := b.db.Begin() + err := b.db.Begin() if err != nil { return err } - b.tx = tx return nil } @@ -75,7 +76,9 @@ func (b *Batch) Delete(storeKey, key []byte) error { } func (b *Batch) Write() error { - _, err := b.tx.Exec(reservedUpsertStmt, reservedStoreKey, keyLatestHeight, b.version, 0, b.version) + b.lock.Lock() + defer b.lock.Unlock() + err := b.db.Exec(reservedUpsertStmt, reservedStoreKey, keyLatestHeight, b.version, 0, b.version) if err != nil { return fmt.Errorf("failed to exec SQL statement: %w", err) } @@ -83,22 +86,22 @@ func (b *Batch) Write() error { for _, op := range b.ops { switch op.action { case batchActionSet: - _, err := b.tx.Exec(upsertStmt, op.storeKey, op.key, op.value, b.version, op.value) + err := b.db.Exec(upsertStmt, op.storeKey, op.key, op.value, b.version, op.value) if err != nil { return fmt.Errorf("failed to exec SQL statement: %w", err) } case batchActionDel: - _, err := b.tx.Exec(delStmt, b.version, op.storeKey, op.key, b.version) + err := b.db.Exec(delStmt, b.version, op.storeKey, op.key, b.version) if err != nil { return fmt.Errorf("failed to exec SQL statement: %w", err) } } } - if err := b.tx.Commit(); err != nil { + if err := b.db.Commit(); err != nil { return fmt.Errorf("failed to write SQL transaction: %w", err) } - return nil + return b.db.Close() } diff --git a/store/v2/storage/sqlite/db.go b/store/v2/storage/sqlite/db.go index 925648928601..f5f4db834ed7 100644 --- a/store/v2/storage/sqlite/db.go +++ b/store/v2/storage/sqlite/db.go @@ -2,13 +2,12 @@ package sqlite import ( "bytes" - "database/sql" - "errors" "fmt" "path/filepath" "strings" + "sync" - _ "github.com/mattn/go-sqlite3" + "github.com/bvinc/go-sqlite-lite/sqlite3" corestore "cosmossdk.io/core/store" "cosmossdk.io/store/v2" @@ -18,7 +17,7 @@ import ( const ( driverName = "sqlite3" - dbName = "ss.db?cache=shared&mode=rwc&_journal_mode=WAL" + dbName = "ss.db" reservedStoreKey = "_RESERVED_" keyLatestHeight = "latest_height" keyPruneHeight = "prune_height" @@ -50,18 +49,24 @@ var ( ) type Database struct { - storage *sql.DB - + storage *sqlite3.Conn + connStr string + writeLock *sync.Mutex // earliestVersion defines the earliest version set in the database, which is // only updated when the database is pruned. earliestVersion uint64 } func New(dataDir string) (*Database, error) { - storage, err := sql.Open(driverName, filepath.Join(dataDir, dbName)) + connStr := fmt.Sprintf("file:%s", filepath.Join(dataDir, dbName)) + db, err := sqlite3.Open(connStr) if err != nil { return nil, fmt.Errorf("failed to open sqlite DB: %w", err) } + err = db.Exec("PRAGMA journal_mode=WAL;") + if err != nil { + return nil, fmt.Errorf("failed to set journal mode: %w", err) + } stmt := ` CREATE TABLE IF NOT EXISTS state_storage ( @@ -76,18 +81,20 @@ func New(dataDir string) (*Database, error) { CREATE UNIQUE INDEX IF NOT EXISTS idx_store_key_version ON state_storage (store_key, key, version); ` - _, err = storage.Exec(stmt) + err = db.Exec(stmt) if err != nil { return nil, fmt.Errorf("failed to exec SQL statement: %w", err) } - pruneHeight, err := getPruneHeight(storage) + pruneHeight, err := getPruneHeight(db) if err != nil { return nil, fmt.Errorf("failed to get prune height: %w", err) } return &Database{ - storage: storage, + storage: db, + connStr: connStr, + writeLock: new(sync.Mutex), earliestVersion: pruneHeight, }, nil } @@ -99,10 +106,15 @@ func (db *Database) Close() error { } func (db *Database) NewBatch(version uint64) (store.Batch, error) { - return NewBatch(db.storage, version) + conn, err := sqlite3.Open(db.connStr) + if err != nil { + return nil, fmt.Errorf("failed to open sqlite DB: %w", err) + } + + return NewBatch(conn, db.writeLock, version) } -func (db *Database) GetLatestVersion() (uint64, error) { +func (db *Database) GetLatestVersion() (version uint64, err error) { stmt, err := db.storage.Prepare(` SELECT value FROM state_storage @@ -112,19 +124,32 @@ func (db *Database) GetLatestVersion() (uint64, error) { return 0, fmt.Errorf("failed to prepare SQL statement: %w", err) } - defer stmt.Close() - - var latestHeight uint64 - if err := stmt.QueryRow(reservedStoreKey, keyLatestHeight).Scan(&latestHeight); err != nil { - if errors.Is(err, sql.ErrNoRows) { - // in case of a fresh database - return 0, nil + defer func(stmt *sqlite3.Stmt) { + cErr := stmt.Close() + if cErr != nil { + err = fmt.Errorf("failed to close GetLatestVersion statement: %w", cErr) } + }(stmt) - return 0, fmt.Errorf("failed to query row: %w", err) + err = stmt.Bind(reservedStoreKey, keyLatestHeight) + if err != nil { + return 0, fmt.Errorf("failed to bind GetLatestVersion statement: %w", err) } - - return latestHeight, nil + hasRow, err := stmt.Step() + if err != nil { + return 0, fmt.Errorf("failed to step through GetLatestVersion rows: %w", err) + } + if !hasRow { + // in case of a fresh database + return 0, nil + } + var v int64 + err = stmt.Scan(&v) + if err != nil { + return 0, fmt.Errorf("failed to scan GetLatestVersion row: %w", err) + } + version = uint64(v) + return version, nil } func (db *Database) VersionExists(v uint64) (bool, error) { @@ -137,7 +162,9 @@ func (db *Database) VersionExists(v uint64) (bool, error) { } func (db *Database) SetLatestVersion(version uint64) error { - _, err := db.storage.Exec(reservedUpsertStmt, reservedStoreKey, keyLatestHeight, version, 0, version) + db.writeLock.Lock() + defer db.writeLock.Unlock() + err := db.storage.Exec(reservedUpsertStmt, reservedStoreKey, keyLatestHeight, int64(version), 0, int64(version)) if err != nil { return fmt.Errorf("failed to exec SQL statement: %w", err) } @@ -172,19 +199,27 @@ func (db *Database) Get(storeKey []byte, targetVersion uint64, key []byte) ([]by var ( value []byte - tomb uint64 + tomb int64 ) - if err := stmt.QueryRow(storeKey, key, targetVersion).Scan(&value, &tomb); err != nil { - if errors.Is(err, sql.ErrNoRows) { - return nil, nil - } - - return nil, fmt.Errorf("failed to query row: %w", err) + err = stmt.Bind(storeKey, key, int64(targetVersion)) + if err != nil { + return nil, fmt.Errorf("failed to bind SQL statement: %w", err) + } + hasRow, err := stmt.Step() + if err != nil { + return nil, fmt.Errorf("failed to step through SQL rows: %w", err) + } + if !hasRow { + return nil, nil + } + err = stmt.Scan(&value, &tomb) + if err != nil { + return nil, fmt.Errorf("failed to scan row: %w", err) } // A tombstone of zero or a target version that is less than the tombstone // version means the key is not deleted at the target version. - if tomb == 0 || targetVersion < tomb { + if tomb == 0 || targetVersion < uint64(tomb) { return value, nil } @@ -199,13 +234,16 @@ func (db *Database) Get(storeKey []byte, targetVersion uint64, key []byte) ([]by // We perform the prune by deleting all versions of a key, excluding reserved keys, // that are <= the given version, except for the latest version of the key. func (db *Database) Prune(version uint64) error { - tx, err := db.storage.Begin() + v := int64(version) + db.writeLock.Lock() + defer db.writeLock.Unlock() + err := db.storage.Begin() if err != nil { return fmt.Errorf("failed to create SQL transaction: %w", err) } defer func() { if err != nil { - err = tx.Rollback() + err = db.storage.Rollback() } }() @@ -218,8 +256,8 @@ func (db *Database) Prune(version uint64) error { t2.version <= ? ) AND store_key != ?; ` - if _, err := tx.Exec(pruneStmt, version, reservedStoreKey); err != nil { - return fmt.Errorf("failed to exec SQL statement: %w", err) + if err := db.storage.Exec(pruneStmt, v, reservedStoreKey); err != nil { + return fmt.Errorf("failed to exec prune keys statement: %w", err) } // prune removed stores @@ -235,22 +273,23 @@ func (db *Database) Prune(version uint64) error { WHERE s.store_key = t.key AND s.version <= t.max_version LIMIT 1 ); ` - if _, err := tx.Exec(pruneRemovedStoreKeysStmt, reservedStoreKey, valueRemovedStore, version, version); err != nil { - return fmt.Errorf("failed to exec SQL statement: %w", err) + if err := db.storage.Exec(pruneRemovedStoreKeysStmt, reservedStoreKey, valueRemovedStore, v); err != nil { + return fmt.Errorf("failed to exec prune store keys statement: %w", err) } // delete the removedKeys - if _, err := tx.Exec("DELETE FROM state_storage WHERE store_key = ? AND value = ? AND version <= ?", reservedStoreKey, valueRemovedStore, version); err != nil { - return fmt.Errorf("failed to exec SQL statement: %w", err) + if err := db.storage.Exec("DELETE FROM state_storage WHERE store_key = ? AND value = ? AND version <= ?", + reservedStoreKey, valueRemovedStore, v); err != nil { + return fmt.Errorf("failed to exec remove keys statement: %w", err) } // set the prune height so we can return for queries below this height - if _, err := tx.Exec(reservedUpsertStmt, reservedStoreKey, keyPruneHeight, version, 0, version); err != nil { - return fmt.Errorf("failed to exec SQL statement: %w", err) + if err := db.storage.Exec(reservedUpsertStmt, reservedStoreKey, keyPruneHeight, v, 0, v); err != nil { + return fmt.Errorf("failed to exec set prune height statement: %w", err) } - if err := tx.Commit(); err != nil { - return fmt.Errorf("failed to write SQL transaction: %w", err) + if err := db.storage.Commit(); err != nil { + return fmt.Errorf("failed to commit prune transaction: %w", err) } db.earliestVersion = version + 1 @@ -282,13 +321,15 @@ func (db *Database) ReverseIterator(storeKey []byte, version uint64, start, end } func (db *Database) PruneStoreKeys(storeKeys []string, version uint64) (err error) { - tx, err := db.storage.Begin() + db.writeLock.Lock() + defer db.writeLock.Unlock() + err = db.storage.Begin() if err != nil { return fmt.Errorf("failed to create SQL transaction: %w", err) } defer func() { if err != nil { - err = tx.Rollback() + err = db.storage.Rollback() } }() @@ -296,12 +337,12 @@ func (db *Database) PruneStoreKeys(storeKeys []string, version uint64) (err erro flushRemovedStoreKeyStmt := `INSERT INTO state_storage(store_key, key, value, version) VALUES (?, ?, ?, ?)` for _, storeKey := range storeKeys { - if _, err := tx.Exec(flushRemovedStoreKeyStmt, reservedStoreKey, []byte(storeKey), valueRemovedStore, version); err != nil { + if err := db.storage.Exec(flushRemovedStoreKeyStmt, reservedStoreKey, []byte(storeKey), valueRemovedStore, version); err != nil { return fmt.Errorf("failed to exec SQL statement: %w", err) } } - return tx.Commit() + return db.storage.Commit() } func (db *Database) PrintRowsDebug() { @@ -312,49 +353,66 @@ func (db *Database) PrintRowsDebug() { defer stmt.Close() - rows, err := stmt.Query() + err = stmt.Exec() if err != nil { panic(fmt.Errorf("failed to execute SQL query: %w", err)) } - var sb strings.Builder - for rows.Next() { + var ( + sb strings.Builder + ) + for { + hasRow, err := stmt.Step() + if err != nil { + panic(fmt.Errorf("failed to step through SQL rows: %w", err)) + } + if !hasRow { + break + } var ( storeKey []byte key []byte value []byte - version uint64 - tomb uint64 + version int64 + tomb int64 ) - if err := rows.Scan(&storeKey, &key, &value, &version, &tomb); err != nil { + if err := stmt.Scan(&storeKey, &key, &value, &version, &tomb); err != nil { panic(fmt.Sprintf("failed to scan row: %s", err)) } sb.WriteString(fmt.Sprintf("STORE_KEY: %s, KEY: %s, VALUE: %s, VERSION: %d, TOMBSTONE: %d\n", storeKey, key, value, version, tomb)) } - if err := rows.Err(); err != nil { - panic(fmt.Errorf("received unexpected error: %w", err)) - } fmt.Println(strings.TrimSpace(sb.String())) } -func getPruneHeight(storage *sql.DB) (uint64, error) { +func getPruneHeight(storage *sqlite3.Conn) (height uint64, err error) { stmt, err := storage.Prepare(`SELECT value FROM state_storage WHERE store_key = ? AND key = ?`) if err != nil { return 0, fmt.Errorf("failed to prepare SQL statement: %w", err) } - defer stmt.Close() - - var value uint64 - if err := stmt.QueryRow(reservedStoreKey, keyPruneHeight).Scan(&value); err != nil { - if errors.Is(err, sql.ErrNoRows) { - return 0, nil + defer func(stmt *sqlite3.Stmt) { + cErr := stmt.Close() + if cErr != nil { + err = fmt.Errorf("failed to close SQL statement: %w", cErr) } + }(stmt) - return 0, fmt.Errorf("failed to query row: %w", err) + if err = stmt.Bind(reservedStoreKey, keyPruneHeight); err != nil { + return 0, fmt.Errorf("failed to bind prune height SQL statement: %w", err) } - - return value, nil + hasRows, err := stmt.Step() + if err != nil { + return 0, fmt.Errorf("failed to step prune height SQL statement: %w", err) + } + if !hasRows { + return 0, nil + } + var h int64 + if err = stmt.Scan(&h); err != nil { + return 0, fmt.Errorf("failed to scan prune height SQL statement: %w", err) + } + height = uint64(h) + return height, nil } diff --git a/store/v2/storage/sqlite/db_test.go b/store/v2/storage/sqlite/db_test.go index fc4c27e6bfd9..b0666d4b12cd 100644 --- a/store/v2/storage/sqlite/db_test.go +++ b/store/v2/storage/sqlite/db_test.go @@ -88,7 +88,9 @@ func TestDatabase_ReverseIterator(t *testing.T) { } func TestParallelWrites(t *testing.T) { - db, err := New(t.TempDir()) + // tmpDir := t.TempDir() + tmpDir := "/tmp/sqlite" + db, err := New(tmpDir) require.NoError(t, err) defer db.Close() diff --git a/store/v2/storage/sqlite/iterator.go b/store/v2/storage/sqlite/iterator.go index daf7e073db3c..406ce1bdb352 100644 --- a/store/v2/storage/sqlite/iterator.go +++ b/store/v2/storage/sqlite/iterator.go @@ -7,22 +7,23 @@ import ( "slices" "strings" + "github.com/bvinc/go-sqlite-lite/sqlite3" + corestore "cosmossdk.io/core/store" ) var _ corestore.Iterator = (*iterator)(nil) type iterator struct { - statement *sql.Stmt - rows *sql.Rows + statement *sqlite3.Stmt key, val []byte start, end []byte valid bool err error } -func newIterator(db *Database, storeKey []byte, targetVersion uint64, start, end []byte, reverse bool) (*iterator, error) { - if targetVersion < db.earliestVersion { +func newIterator(db *Database, storeKey []byte, version uint64, start, end []byte, reverse bool) (*iterator, error) { + if version < db.earliestVersion { return &iterator{ start: start, end: end, @@ -31,8 +32,9 @@ func newIterator(db *Database, storeKey []byte, targetVersion uint64, start, end } var ( - keyClause = []string{"store_key = ?", "version <= ?"} - queryArgs []any + targetVersion = int64(version) + keyClause = []string{"store_key = ?", "version <= ?"} + queryArgs []any ) switch { @@ -72,18 +74,21 @@ func newIterator(db *Database, storeKey []byte, targetVersion uint64, start, end return nil, fmt.Errorf("failed to prepare SQL statement: %w", err) } - rows, err := stmt.Query(queryArgs...) + err = stmt.Bind(queryArgs...) if err != nil { _ = stmt.Close() - return nil, fmt.Errorf("failed to execute SQL query: %w", err) + return nil, fmt.Errorf("failed to bind SQL iterator query arguments: %w", err) } itr := &iterator{ statement: stmt, - rows: rows, start: start, end: end, - valid: rows.Next(), + } + itr.valid, err = itr.statement.Step() + if err != nil { + itr.err = fmt.Errorf("failed to step SQL iterator: %w", err) + return itr, nil } if !itr.valid { itr.err = fmt.Errorf("iterator invalid: %w", sql.ErrNoRows) @@ -106,7 +111,6 @@ func (itr *iterator) Close() (err error) { itr.valid = false itr.statement = nil - itr.rows = nil return err } @@ -128,8 +132,7 @@ func (itr *iterator) Value() []byte { } func (itr *iterator) Valid() bool { - if !itr.valid || itr.rows.Err() != nil { - itr.valid = false + if !itr.valid { return itr.valid } @@ -145,19 +148,16 @@ func (itr *iterator) Valid() bool { } func (itr *iterator) Next() { - if itr.rows.Next() { - itr.parseRow() + var hasRow bool + hasRow, itr.err = itr.statement.Step() + if itr.err != nil || !hasRow { + itr.valid = false return } - - itr.valid = false + itr.parseRow() } func (itr *iterator) Error() error { - if err := itr.rows.Err(); err != nil { - return err - } - return itr.err } @@ -166,7 +166,7 @@ func (itr *iterator) parseRow() { key []byte value []byte ) - if err := itr.rows.Scan(&key, &value); err != nil { + if err := itr.statement.Scan(&key, &value); err != nil { itr.err = fmt.Errorf("failed to scan row: %w", err) itr.valid = false return From a837b9b375d90a72699a47e32203cab3b8869141 Mon Sep 17 00:00:00 2001 From: Matt Kocubinski Date: Mon, 4 Nov 2024 11:58:55 -0600 Subject: [PATCH 02/20] go mod tidy all --- runtime/v2/go.mod | 4 ++-- runtime/v2/go.sum | 8 ++++---- server/v2/cometbft/go.mod | 2 +- server/v2/cometbft/go.sum | 4 ++-- server/v2/go.mod | 4 ++-- server/v2/go.sum | 8 ++++---- simapp/v2/go.mod | 2 +- simapp/v2/go.sum | 4 ++-- 8 files changed, 18 insertions(+), 18 deletions(-) diff --git a/runtime/v2/go.mod b/runtime/v2/go.mod index 78ad2b0fe974..2577167c3e59 100644 --- a/runtime/v2/go.mod +++ b/runtime/v2/go.mod @@ -34,6 +34,7 @@ require ( cosmossdk.io/errors/v2 v2.0.0-20240731132947-df72853b3ca5 // indirect github.com/DataDog/zstd v1.5.5 // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/bvinc/go-sqlite-lite v0.6.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cockroachdb/errors v1.11.1 // indirect github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect @@ -48,7 +49,7 @@ require ( github.com/getsentry/sentry-go v0.27.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/snappy v0.0.4 // indirect - github.com/google/btree v1.1.2 // indirect + github.com/google/btree v1.1.3 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/hashicorp/go-immutable-radix v1.3.1 // indirect github.com/hashicorp/go-metrics v0.5.3 // indirect @@ -59,7 +60,6 @@ require ( github.com/linxGnu/grocksdb v1.9.3 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/mattn/go-sqlite3 v1.14.22 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/onsi/gomega v1.28.1 // indirect github.com/pkg/errors v0.9.1 // indirect diff --git a/runtime/v2/go.sum b/runtime/v2/go.sum index cdf94ffd213b..d6815b38eff6 100644 --- a/runtime/v2/go.sum +++ b/runtime/v2/go.sum @@ -25,6 +25,8 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bvinc/go-sqlite-lite v0.6.1 h1:JU8Rz5YAOZQiU3WEulKF084wfXpytRiqD2IaW2QjPz4= +github.com/bvinc/go-sqlite-lite v0.6.1/go.mod h1:2GiE60NUdb0aNhDdY+LXgrqAVDpi2Ijc6dB6ZMp9x6s= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -100,8 +102,8 @@ github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= -github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= +github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -153,8 +155,6 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/ github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= -github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= diff --git a/server/v2/cometbft/go.mod b/server/v2/cometbft/go.mod index e582690cd789..bca3468afde7 100644 --- a/server/v2/cometbft/go.mod +++ b/server/v2/cometbft/go.mod @@ -60,6 +60,7 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/bgentry/speakeasy v0.2.0 // indirect github.com/btcsuite/btcd/btcec/v2 v2.3.4 // indirect + github.com/bvinc/go-sqlite-lite v0.6.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cockroachdb/errors v1.11.3 // indirect github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce // indirect @@ -128,7 +129,6 @@ require ( github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/mattn/go-sqlite3 v1.14.22 // indirect github.com/minio/highwayhash v1.0.3 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mtibben/percent v0.2.1 // indirect diff --git a/server/v2/cometbft/go.sum b/server/v2/cometbft/go.sum index 0fcabe91037e..eacb41ac69b9 100644 --- a/server/v2/cometbft/go.sum +++ b/server/v2/cometbft/go.sum @@ -64,6 +64,8 @@ github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOF github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= github.com/bufbuild/protocompile v0.4.0 h1:LbFKd2XowZvQ/kajzguUp2DC9UEIQhIq77fZZlaQsNA= github.com/bufbuild/protocompile v0.4.0/go.mod h1:3v93+mbWn/v3xzN+31nwkJfrEpAUwp+BagBSZWx+TP8= +github.com/bvinc/go-sqlite-lite v0.6.1 h1:JU8Rz5YAOZQiU3WEulKF084wfXpytRiqD2IaW2QjPz4= +github.com/bvinc/go-sqlite-lite v0.6.1/go.mod h1:2GiE60NUdb0aNhDdY+LXgrqAVDpi2Ijc6dB6ZMp9x6s= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -342,8 +344,6 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/ github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= -github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/minio/highwayhash v1.0.3 h1:kbnuUMoHYyVl7szWjSxJnxw11k2U709jqFPPmIUyD6Q= github.com/minio/highwayhash v1.0.3/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= diff --git a/server/v2/go.mod b/server/v2/go.mod index 1e2e76342465..db37d1f3351e 100644 --- a/server/v2/go.mod +++ b/server/v2/go.mod @@ -47,6 +47,7 @@ require ( github.com/DataDog/zstd v1.5.5 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/bvinc/go-sqlite-lite v0.6.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cockroachdb/errors v1.11.1 // indirect github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect @@ -63,7 +64,7 @@ require ( github.com/gogo/googleapis v1.4.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/snappy v0.0.4 // indirect - github.com/google/btree v1.1.2 // indirect + github.com/google/btree v1.1.3 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/hashicorp/go-immutable-radix v1.3.1 // indirect github.com/hashicorp/golang-lru v1.0.2 // indirect @@ -78,7 +79,6 @@ require ( github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/mattn/go-sqlite3 v1.14.22 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/oklog/run v1.1.0 // indirect github.com/onsi/gomega v1.28.1 // indirect diff --git a/server/v2/go.sum b/server/v2/go.sum index c35326ca5bbf..b63ddf06e8d7 100644 --- a/server/v2/go.sum +++ b/server/v2/go.sum @@ -29,6 +29,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bufbuild/protocompile v0.14.1 h1:iA73zAf/fyljNjQKwYzUHD6AD4R8KMasmwa/FBatYVw= github.com/bufbuild/protocompile v0.14.1/go.mod h1:ppVdAIhbr2H8asPk6k4pY7t9zB1OU5DoEw9xY/FUi1c= +github.com/bvinc/go-sqlite-lite v0.6.1 h1:JU8Rz5YAOZQiU3WEulKF084wfXpytRiqD2IaW2QjPz4= +github.com/bvinc/go-sqlite-lite v0.6.1/go.mod h1:2GiE60NUdb0aNhDdY+LXgrqAVDpi2Ijc6dB6ZMp9x6s= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= @@ -136,8 +138,8 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= -github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= +github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -210,8 +212,6 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/ github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= -github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= diff --git a/simapp/v2/go.mod b/simapp/v2/go.mod index 0e803ed0f86d..6e9a9329c733 100644 --- a/simapp/v2/go.mod +++ b/simapp/v2/go.mod @@ -79,6 +79,7 @@ require ( github.com/bgentry/speakeasy v0.2.0 // indirect github.com/bits-and-blooms/bitset v1.10.0 // indirect github.com/btcsuite/btcd/btcec/v2 v2.3.4 // indirect + github.com/bvinc/go-sqlite-lite v0.6.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/chzyer/readline v1.5.1 // indirect github.com/cockroachdb/apd/v2 v2.0.2 // indirect @@ -168,7 +169,6 @@ require ( github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.14 // indirect - github.com/mattn/go-sqlite3 v1.14.22 // indirect github.com/mdp/qrterminal/v3 v3.2.0 // indirect github.com/minio/highwayhash v1.0.3 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect diff --git a/simapp/v2/go.sum b/simapp/v2/go.sum index beee381cb4ec..babd652659f7 100644 --- a/simapp/v2/go.sum +++ b/simapp/v2/go.sum @@ -260,6 +260,8 @@ github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOF github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= github.com/bufbuild/protocompile v0.4.0 h1:LbFKd2XowZvQ/kajzguUp2DC9UEIQhIq77fZZlaQsNA= github.com/bufbuild/protocompile v0.4.0/go.mod h1:3v93+mbWn/v3xzN+31nwkJfrEpAUwp+BagBSZWx+TP8= +github.com/bvinc/go-sqlite-lite v0.6.1 h1:JU8Rz5YAOZQiU3WEulKF084wfXpytRiqD2IaW2QjPz4= +github.com/bvinc/go-sqlite-lite v0.6.1/go.mod h1:2GiE60NUdb0aNhDdY+LXgrqAVDpi2Ijc6dB6ZMp9x6s= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -657,8 +659,6 @@ github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= -github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mdp/qrterminal/v3 v3.2.0 h1:qteQMXO3oyTK4IHwj2mWsKYYRBOp1Pj2WRYFYYNTCdk= github.com/mdp/qrterminal/v3 v3.2.0/go.mod h1:XGGuua4Lefrl7TLEsSONiD+UEjQXJZ4mPzF+gWYIJkk= From 906144bf8aef9d5a19a2addb099f8c1ea219161d Mon Sep 17 00:00:00 2001 From: Matt Kocubinski Date: Mon, 4 Nov 2024 12:09:41 -0600 Subject: [PATCH 03/20] rm iavl v1 replace --- store/v2/go.mod | 1 - store/v2/go.sum | 4 ++++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/store/v2/go.mod b/store/v2/go.mod index 2ed84be60137..a3f0e8e6554a 100644 --- a/store/v2/go.mod +++ b/store/v2/go.mod @@ -71,6 +71,5 @@ require ( replace ( cosmossdk.io/core => ../../core - github.com/cosmos/iavl => ../../../../iavl/iavl-master github.com/cosmos/iavl/v2 => ../../../../iavl/iavl-v2 ) diff --git a/store/v2/go.sum b/store/v2/go.sum index 592a144de2dc..d0062be85bcc 100644 --- a/store/v2/go.sum +++ b/store/v2/go.sum @@ -46,6 +46,8 @@ github.com/cosmos/cosmos-proto v1.0.0-beta.5 h1:eNcayDLpip+zVLRLYafhzLvQlSmyab+R github.com/cosmos/cosmos-proto v1.0.0-beta.5/go.mod h1:hQGLpiIUloJBMdQMMWb/4wRApmI9hjHH05nefC0Ojec= github.com/cosmos/gogoproto v1.7.0 h1:79USr0oyXAbxg3rspGh/m4SWNyoz/GLaAh0QlCe2fro= github.com/cosmos/gogoproto v1.7.0/go.mod h1:yWChEv5IUEYURQasfyBW5ffkMHR/90hiHgbNgrtp4j0= +github.com/cosmos/iavl v1.3.0 h1:Ezaxt8aPA3kbkhsfyqwenChGLQwHDAIif3tG9x1FMV8= +github.com/cosmos/iavl v1.3.0/go.mod h1:T6SfBcyhulVIY2G/ZtAtQm/QiJvsuhIos52V4dWYk88= github.com/cosmos/iavl-bench/bench v0.0.4 h1:J6zQPiBqF4CXMM3QBsLqZgQEBGY0taX85vLIZMhmAfQ= github.com/cosmos/iavl-bench/bench v0.0.4/go.mod h1:j2rLae77EffacWcp7mmj3Uaa4AOAmZA7ymvhsuBQKKI= github.com/cosmos/ics23/go v0.11.0 h1:jk5skjT0TqX5e5QJbEnwXIS2yI2vnmLOgpQPeM5RtnU= @@ -80,6 +82,8 @@ github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5x github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= From ee21c4aca663c0197a475922f3f447a8138ebf1a Mon Sep 17 00:00:00 2001 From: Matt Kocubinski Date: Mon, 4 Nov 2024 15:44:27 -0600 Subject: [PATCH 04/20] go mod tidy all --- store/v2/commitment/iavlv2/tree.go | 10 ++++------ store/v2/go.mod | 5 +---- store/v2/go.sum | 2 ++ 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/store/v2/commitment/iavlv2/tree.go b/store/v2/commitment/iavlv2/tree.go index a88567e23759..2269a5a6eb14 100644 --- a/store/v2/commitment/iavlv2/tree.go +++ b/store/v2/commitment/iavlv2/tree.go @@ -1,6 +1,7 @@ package iavlv2 import ( + "errors" "fmt" "github.com/cosmos/iavl/v2" @@ -69,8 +70,7 @@ func (t Tree) SetInitialVersion(version uint64) error { } func (t Tree) GetProof(version uint64, key []byte) (*ics23.CommitmentProof, error) { - //TODO implement me - panic("implement me") + return t.tree.GetProof(int64(version), key) } func (t Tree) Get(version uint64, key []byte) ([]byte, error) { @@ -81,13 +81,11 @@ func (t Tree) Get(version uint64, key []byte) ([]byte, error) { } func (t Tree) Export(version uint64) (commitment.Exporter, error) { - //TODO implement me - panic("implement me") + return nil, errors.New("snapshot import/export not yet supported") } func (t Tree) Import(version uint64) (commitment.Importer, error) { - //TODO implement me - panic("implement me") + return nil, errors.New("snapshot import/export not yet supported") } func (t Tree) Close() error { diff --git a/store/v2/go.mod b/store/v2/go.mod index a3f0e8e6554a..b1184dad3d6d 100644 --- a/store/v2/go.mod +++ b/store/v2/go.mod @@ -69,7 +69,4 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect ) -replace ( - cosmossdk.io/core => ../../core - github.com/cosmos/iavl/v2 => ../../../../iavl/iavl-v2 -) +replace cosmossdk.io/core => ../../core diff --git a/store/v2/go.sum b/store/v2/go.sum index d0062be85bcc..3c443dfe7cbf 100644 --- a/store/v2/go.sum +++ b/store/v2/go.sum @@ -50,6 +50,8 @@ github.com/cosmos/iavl v1.3.0 h1:Ezaxt8aPA3kbkhsfyqwenChGLQwHDAIif3tG9x1FMV8= github.com/cosmos/iavl v1.3.0/go.mod h1:T6SfBcyhulVIY2G/ZtAtQm/QiJvsuhIos52V4dWYk88= github.com/cosmos/iavl-bench/bench v0.0.4 h1:J6zQPiBqF4CXMM3QBsLqZgQEBGY0taX85vLIZMhmAfQ= github.com/cosmos/iavl-bench/bench v0.0.4/go.mod h1:j2rLae77EffacWcp7mmj3Uaa4AOAmZA7ymvhsuBQKKI= +github.com/cosmos/iavl/v2 v2.0.0-20241029195051-d3ac274b6b58 h1:uQF/fzXtekKFxYN3eRVWiAmn4DdiYwqhq8CIuFTuOAE= +github.com/cosmos/iavl/v2 v2.0.0-20241029195051-d3ac274b6b58/go.mod h1:ZjSMMhZ6JqgzCRJO1oKJ5EtRksDFMZ5Djv+Rv0SX/uc= github.com/cosmos/ics23/go v0.11.0 h1:jk5skjT0TqX5e5QJbEnwXIS2yI2vnmLOgpQPeM5RtnU= github.com/cosmos/ics23/go v0.11.0/go.mod h1:A8OjxPE67hHST4Icw94hOxxFEJMBG031xIGF/JHNIY0= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= From 063a18fd2aa78b9a408dcb3d43a6194d62aa7e74 Mon Sep 17 00:00:00 2001 From: Matt Kocubinski Date: Mon, 4 Nov 2024 16:42:09 -0600 Subject: [PATCH 05/20] fix test and potential faulty uint64 conversion --- store/v2/storage/sqlite/batch.go | 3 +++ store/v2/storage/sqlite/db_test.go | 12 ++++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/store/v2/storage/sqlite/batch.go b/store/v2/storage/sqlite/batch.go index ca4001200c97..eba639c19d8b 100644 --- a/store/v2/storage/sqlite/batch.go +++ b/store/v2/storage/sqlite/batch.go @@ -33,6 +33,9 @@ type Batch struct { } func NewBatch(db *sqlite3.Conn, writeLock *sync.Mutex, version uint64) (*Batch, error) { + if version&(1<<63) != 0 { + return nil, fmt.Errorf("%d too large; uint64 with the highest bit set are not supported", version) + } err := db.Begin() if err != nil { return nil, fmt.Errorf("failed to create SQL transaction: %w", err) diff --git a/store/v2/storage/sqlite/db_test.go b/store/v2/storage/sqlite/db_test.go index b0666d4b12cd..7ec950bf93a0 100644 --- a/store/v2/storage/sqlite/db_test.go +++ b/store/v2/storage/sqlite/db_test.go @@ -2,6 +2,7 @@ package sqlite import ( "fmt" + "math" "sync" "testing" @@ -88,8 +89,7 @@ func TestDatabase_ReverseIterator(t *testing.T) { } func TestParallelWrites(t *testing.T) { - // tmpDir := t.TempDir() - tmpDir := "/tmp/sqlite" + tmpDir := t.TempDir() db, err := New(tmpDir) require.NoError(t, err) defer db.Close() @@ -200,3 +200,11 @@ func TestParallelWriteAndPruning(t *testing.T) { require.NoError(t, err) require.Equal(t, []byte(fmt.Sprintf("val-%d-%03d", version-1, 0)), val) } + +func TestUint64(t *testing.T) { + tmpDir := t.TempDir() + db, err := New(tmpDir) + require.NoError(t, err) + _, err = db.NewBatch(math.MaxUint64) + require.Error(t, err) +} From c3415dec69e25e9f8975b21506b8a3f89f161a8b Mon Sep 17 00:00:00 2001 From: Matt Kocubinski Date: Mon, 4 Nov 2024 16:49:18 -0600 Subject: [PATCH 06/20] fix iavl/v2 version and method receivers --- store/v2/commitment/iavlv2/tree.go | 31 +++++++++++++++--------------- store/v2/go.mod | 2 +- store/v2/go.sum | 4 ++-- store/v2/storage/sqlite/batch.go | 8 ++++++-- store/v2/storage/sqlite/db.go | 7 +------ 5 files changed, 25 insertions(+), 27 deletions(-) diff --git a/store/v2/commitment/iavlv2/tree.go b/store/v2/commitment/iavlv2/tree.go index 2269a5a6eb14..5a680eeba694 100644 --- a/store/v2/commitment/iavlv2/tree.go +++ b/store/v2/commitment/iavlv2/tree.go @@ -29,75 +29,74 @@ func NewTree(treeOptions iavl.TreeOptions, dbOptions iavl.SqliteDbOptions, pool return &Tree{tree: tree}, nil } -func (t Tree) Set(key, value []byte) error { +func (t *Tree) Set(key, value []byte) error { _, err := t.tree.Set(key, value) return err } -func (t Tree) Remove(key []byte) error { +func (t *Tree) Remove(key []byte) error { _, _, err := t.tree.Remove(key) return err } -func (t Tree) GetLatestVersion() (uint64, error) { +func (t *Tree) GetLatestVersion() (uint64, error) { return uint64(t.tree.Version()), nil } -func (t Tree) Hash() []byte { +func (t *Tree) Hash() []byte { return t.tree.Hash() } -func (t Tree) Version() uint64 { +func (t *Tree) Version() uint64 { return uint64(t.tree.Version()) } -func (t Tree) LoadVersion(version uint64) error { - // TODO fix this in iavl v2 +func (t *Tree) LoadVersion(version uint64) error { if version == 0 { return nil } return t.tree.LoadVersion(int64(version)) } -func (t Tree) Commit() ([]byte, uint64, error) { +func (t *Tree) Commit() ([]byte, uint64, error) { h, v, err := t.tree.SaveVersion() return h, uint64(v), err } -func (t Tree) SetInitialVersion(version uint64) error { +func (t *Tree) SetInitialVersion(version uint64) error { t.tree.SetShouldCheckpoint() return t.tree.SetInitialVersion(int64(version)) } -func (t Tree) GetProof(version uint64, key []byte) (*ics23.CommitmentProof, error) { +func (t *Tree) GetProof(version uint64, key []byte) (*ics23.CommitmentProof, error) { return t.tree.GetProof(int64(version), key) } -func (t Tree) Get(version uint64, key []byte) ([]byte, error) { +func (t *Tree) Get(version uint64, key []byte) ([]byte, error) { if int64(version) != t.tree.Version() { return nil, fmt.Errorf("loading past version not yet supported") } return t.tree.Get(key) } -func (t Tree) Export(version uint64) (commitment.Exporter, error) { +func (t *Tree) Export(version uint64) (commitment.Exporter, error) { return nil, errors.New("snapshot import/export not yet supported") } -func (t Tree) Import(version uint64) (commitment.Importer, error) { +func (t *Tree) Import(version uint64) (commitment.Importer, error) { return nil, errors.New("snapshot import/export not yet supported") } -func (t Tree) Close() error { +func (t *Tree) Close() error { return t.tree.Close() } -func (t Tree) Prune(version uint64) error { +func (t *Tree) Prune(version uint64) error { return t.tree.DeleteVersionsTo(int64(version)) } // PausePruning is unnecessary in IAVL v2 due to the advanced pruning mechanism -func (t Tree) PausePruning(bool) {} +func (t *Tree) PausePruning(bool) {} func (t *Tree) WorkingHash() []byte { return t.tree.Hash() diff --git a/store/v2/go.mod b/store/v2/go.mod index b1184dad3d6d..03d3f1e43756 100644 --- a/store/v2/go.mod +++ b/store/v2/go.mod @@ -12,7 +12,7 @@ require ( github.com/cosmos/cosmos-proto v1.0.0-beta.5 github.com/cosmos/gogoproto v1.7.0 github.com/cosmos/iavl v1.3.0 - github.com/cosmos/iavl/v2 v2.0.0-20241029195051-d3ac274b6b58 + github.com/cosmos/iavl/v2 v2.0.0-20241104195329-835276bdf831 github.com/cosmos/ics23/go v0.11.0 github.com/google/btree v1.1.3 github.com/hashicorp/go-metrics v0.5.3 diff --git a/store/v2/go.sum b/store/v2/go.sum index 3c443dfe7cbf..42ec36586cf7 100644 --- a/store/v2/go.sum +++ b/store/v2/go.sum @@ -50,8 +50,8 @@ github.com/cosmos/iavl v1.3.0 h1:Ezaxt8aPA3kbkhsfyqwenChGLQwHDAIif3tG9x1FMV8= github.com/cosmos/iavl v1.3.0/go.mod h1:T6SfBcyhulVIY2G/ZtAtQm/QiJvsuhIos52V4dWYk88= github.com/cosmos/iavl-bench/bench v0.0.4 h1:J6zQPiBqF4CXMM3QBsLqZgQEBGY0taX85vLIZMhmAfQ= github.com/cosmos/iavl-bench/bench v0.0.4/go.mod h1:j2rLae77EffacWcp7mmj3Uaa4AOAmZA7ymvhsuBQKKI= -github.com/cosmos/iavl/v2 v2.0.0-20241029195051-d3ac274b6b58 h1:uQF/fzXtekKFxYN3eRVWiAmn4DdiYwqhq8CIuFTuOAE= -github.com/cosmos/iavl/v2 v2.0.0-20241029195051-d3ac274b6b58/go.mod h1:ZjSMMhZ6JqgzCRJO1oKJ5EtRksDFMZ5Djv+Rv0SX/uc= +github.com/cosmos/iavl/v2 v2.0.0-20241104195329-835276bdf831 h1:K5TLZIz7vh/6k23zSbEj0nUYeNO8RuZB5xD7/f9tWeM= +github.com/cosmos/iavl/v2 v2.0.0-20241104195329-835276bdf831/go.mod h1:7RSm0aeApe3S1x4TrLffvUL6pjOtMYV4glYnpAhr2lw= github.com/cosmos/ics23/go v0.11.0 h1:jk5skjT0TqX5e5QJbEnwXIS2yI2vnmLOgpQPeM5RtnU= github.com/cosmos/ics23/go v0.11.0/go.mod h1:A8OjxPE67hHST4Icw94hOxxFEJMBG031xIGF/JHNIY0= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= diff --git a/store/v2/storage/sqlite/batch.go b/store/v2/storage/sqlite/batch.go index eba639c19d8b..4b4edfdd3815 100644 --- a/store/v2/storage/sqlite/batch.go +++ b/store/v2/storage/sqlite/batch.go @@ -32,11 +32,15 @@ type Batch struct { version int64 } -func NewBatch(db *sqlite3.Conn, writeLock *sync.Mutex, version uint64) (*Batch, error) { +func NewBatch(connStr string, writeLock *sync.Mutex, version uint64) (*Batch, error) { if version&(1<<63) != 0 { return nil, fmt.Errorf("%d too large; uint64 with the highest bit set are not supported", version) } - err := db.Begin() + db, err := sqlite3.Open(connStr) + if err != nil { + return nil, fmt.Errorf("failed to open SQL database: %w", err) + } + err = db.Begin() if err != nil { return nil, fmt.Errorf("failed to create SQL transaction: %w", err) } diff --git a/store/v2/storage/sqlite/db.go b/store/v2/storage/sqlite/db.go index f5f4db834ed7..9eda2e7418e6 100644 --- a/store/v2/storage/sqlite/db.go +++ b/store/v2/storage/sqlite/db.go @@ -106,12 +106,7 @@ func (db *Database) Close() error { } func (db *Database) NewBatch(version uint64) (store.Batch, error) { - conn, err := sqlite3.Open(db.connStr) - if err != nil { - return nil, fmt.Errorf("failed to open sqlite DB: %w", err) - } - - return NewBatch(conn, db.writeLock, version) + return NewBatch(db.connStr, db.writeLock, version) } func (db *Database) GetLatestVersion() (version uint64, err error) { From 56ea0e55b84c12ce4f77adbc0d4ca2af32a5914e Mon Sep 17 00:00:00 2001 From: Matt Kocubinski Date: Mon, 4 Nov 2024 17:08:57 -0600 Subject: [PATCH 07/20] some clean up --- store/v2/storage/sqlite/batch.go | 18 +++++++----------- store/v2/storage/sqlite/db.go | 17 ++++++++--------- 2 files changed, 15 insertions(+), 20 deletions(-) diff --git a/store/v2/storage/sqlite/batch.go b/store/v2/storage/sqlite/batch.go index 4b4edfdd3815..9aa621e4e2f3 100644 --- a/store/v2/storage/sqlite/batch.go +++ b/store/v2/storage/sqlite/batch.go @@ -32,18 +32,10 @@ type Batch struct { version int64 } -func NewBatch(connStr string, writeLock *sync.Mutex, version uint64) (*Batch, error) { +func NewBatch(db *sqlite3.Conn, writeLock *sync.Mutex, version uint64) (*Batch, error) { if version&(1<<63) != 0 { return nil, fmt.Errorf("%d too large; uint64 with the highest bit set are not supported", version) } - db, err := sqlite3.Open(connStr) - if err != nil { - return nil, fmt.Errorf("failed to open SQL database: %w", err) - } - err = db.Begin() - if err != nil { - return nil, fmt.Errorf("failed to create SQL transaction: %w", err) - } return &Batch{ db: db, @@ -85,7 +77,11 @@ func (b *Batch) Delete(storeKey, key []byte) error { func (b *Batch) Write() error { b.lock.Lock() defer b.lock.Unlock() - err := b.db.Exec(reservedUpsertStmt, reservedStoreKey, keyLatestHeight, b.version, 0, b.version) + err := b.db.Begin() + if err != nil { + return fmt.Errorf("failed to start SQL transaction: %w", err) + } + err = b.db.Exec(reservedUpsertStmt, reservedStoreKey, keyLatestHeight, b.version, 0, b.version) if err != nil { return fmt.Errorf("failed to exec SQL statement: %w", err) } @@ -110,5 +106,5 @@ func (b *Batch) Write() error { return fmt.Errorf("failed to write SQL transaction: %w", err) } - return b.db.Close() + return nil } diff --git a/store/v2/storage/sqlite/db.go b/store/v2/storage/sqlite/db.go index 9eda2e7418e6..c535ce935d51 100644 --- a/store/v2/storage/sqlite/db.go +++ b/store/v2/storage/sqlite/db.go @@ -2,6 +2,7 @@ package sqlite import ( "bytes" + "errors" "fmt" "path/filepath" "strings" @@ -51,7 +52,7 @@ var ( type Database struct { storage *sqlite3.Conn connStr string - writeLock *sync.Mutex + writeLock sync.Mutex // earliestVersion defines the earliest version set in the database, which is // only updated when the database is pruned. earliestVersion uint64 @@ -94,7 +95,7 @@ func New(dataDir string) (*Database, error) { return &Database{ storage: db, connStr: connStr, - writeLock: new(sync.Mutex), + writeLock: sync.Mutex{}, earliestVersion: pruneHeight, }, nil } @@ -106,7 +107,7 @@ func (db *Database) Close() error { } func (db *Database) NewBatch(version uint64) (store.Batch, error) { - return NewBatch(db.connStr, db.writeLock, version) + return NewBatch(db.storage, &db.writeLock, version) } func (db *Database) GetLatestVersion() (version uint64, err error) { @@ -228,17 +229,17 @@ func (db *Database) Get(storeKey []byte, targetVersion uint64, key []byte) ([]by // // We perform the prune by deleting all versions of a key, excluding reserved keys, // that are <= the given version, except for the latest version of the key. -func (db *Database) Prune(version uint64) error { +func (db *Database) Prune(version uint64) (err error) { v := int64(version) db.writeLock.Lock() defer db.writeLock.Unlock() - err := db.storage.Begin() + err = db.storage.Begin() if err != nil { return fmt.Errorf("failed to create SQL transaction: %w", err) } defer func() { if err != nil { - err = db.storage.Rollback() + err = errors.Join(db.storage.Rollback()) } }() @@ -353,9 +354,7 @@ func (db *Database) PrintRowsDebug() { panic(fmt.Errorf("failed to execute SQL query: %w", err)) } - var ( - sb strings.Builder - ) + var sb strings.Builder for { hasRow, err := stmt.Step() if err != nil { From 200f7dda320a8ded4eefddf988f5b1fbffe23639 Mon Sep 17 00:00:00 2001 From: Matt Kocubinski Date: Mon, 4 Nov 2024 17:12:53 -0600 Subject: [PATCH 08/20] clean up error handling in Batch.Write() --- store/v2/storage/sqlite/batch.go | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/store/v2/storage/sqlite/batch.go b/store/v2/storage/sqlite/batch.go index 9aa621e4e2f3..411fd78caedf 100644 --- a/store/v2/storage/sqlite/batch.go +++ b/store/v2/storage/sqlite/batch.go @@ -1,6 +1,7 @@ package sqlite import ( + "errors" "fmt" "sync" @@ -74,29 +75,31 @@ func (b *Batch) Delete(storeKey, key []byte) error { return nil } -func (b *Batch) Write() error { +func (b *Batch) Write() (err error) { b.lock.Lock() defer b.lock.Unlock() - err := b.db.Begin() + err = b.db.Begin() if err != nil { return fmt.Errorf("failed to start SQL transaction: %w", err) } - err = b.db.Exec(reservedUpsertStmt, reservedStoreKey, keyLatestHeight, b.version, 0, b.version) - if err != nil { + defer func() { + if err != nil { + err = errors.Join(err, b.db.Rollback()) + } + }() + if err := b.db.Exec(reservedUpsertStmt, reservedStoreKey, keyLatestHeight, b.version, 0, b.version); err != nil { return fmt.Errorf("failed to exec SQL statement: %w", err) } for _, op := range b.ops { switch op.action { case batchActionSet: - err := b.db.Exec(upsertStmt, op.storeKey, op.key, op.value, b.version, op.value) - if err != nil { + if err := b.db.Exec(upsertStmt, op.storeKey, op.key, op.value, b.version, op.value); err != nil { return fmt.Errorf("failed to exec SQL statement: %w", err) } case batchActionDel: - err := b.db.Exec(delStmt, b.version, op.storeKey, op.key, b.version) - if err != nil { + if err := b.db.Exec(delStmt, b.version, op.storeKey, op.key, b.version); err != nil { return fmt.Errorf("failed to exec SQL statement: %w", err) } } From e1f057482c4028cb02bfc2ae13bcceca6dc6759d Mon Sep 17 00:00:00 2001 From: Matt Kocubinski Date: Mon, 4 Nov 2024 17:24:38 -0600 Subject: [PATCH 09/20] reuse uint64 check --- store/v2/storage/sqlite/batch.go | 2 +- store/v2/storage/sqlite/db.go | 6 +++++- store/v2/storage/sqlite/iterator.go | 3 +++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/store/v2/storage/sqlite/batch.go b/store/v2/storage/sqlite/batch.go index 411fd78caedf..01252f2e0c7c 100644 --- a/store/v2/storage/sqlite/batch.go +++ b/store/v2/storage/sqlite/batch.go @@ -34,7 +34,7 @@ type Batch struct { } func NewBatch(db *sqlite3.Conn, writeLock *sync.Mutex, version uint64) (*Batch, error) { - if version&(1<<63) != 0 { + if isHighBitSet(version) { return nil, fmt.Errorf("%d too large; uint64 with the highest bit set are not supported", version) } diff --git a/store/v2/storage/sqlite/db.go b/store/v2/storage/sqlite/db.go index c535ce935d51..9f011864e33d 100644 --- a/store/v2/storage/sqlite/db.go +++ b/store/v2/storage/sqlite/db.go @@ -389,7 +389,7 @@ func getPruneHeight(storage *sqlite3.Conn) (height uint64, err error) { defer func(stmt *sqlite3.Stmt) { cErr := stmt.Close() if cErr != nil { - err = fmt.Errorf("failed to close SQL statement: %w", cErr) + err = errors.Join(err, fmt.Errorf("failed to close SQL statement: %w", cErr)) } }(stmt) @@ -410,3 +410,7 @@ func getPruneHeight(storage *sqlite3.Conn) (height uint64, err error) { height = uint64(h) return height, nil } + +func isHighBitSet(version uint64) bool { + return version&(1<<63) != 0 +} diff --git a/store/v2/storage/sqlite/iterator.go b/store/v2/storage/sqlite/iterator.go index 406ce1bdb352..dbe3339c0c25 100644 --- a/store/v2/storage/sqlite/iterator.go +++ b/store/v2/storage/sqlite/iterator.go @@ -23,6 +23,9 @@ type iterator struct { } func newIterator(db *Database, storeKey []byte, version uint64, start, end []byte, reverse bool) (*iterator, error) { + if isHighBitSet(version) { + return nil, fmt.Errorf("%d too large; uint64 with the highest bit set are not supported", version) + } if version < db.earliestVersion { return &iterator{ start: start, From bb2eeec3875d02b71170b8483959677ea446d604 Mon Sep 17 00:00:00 2001 From: Matt Kocubinski Date: Mon, 4 Nov 2024 17:35:37 -0600 Subject: [PATCH 10/20] add integer overflow guards in iavl v2 wrapper --- store/v2/commitment/iavlv2/tree.go | 24 ++++++++++++++++++++++++ store/v2/storage/sqlite/db.go | 4 ++-- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/store/v2/commitment/iavlv2/tree.go b/store/v2/commitment/iavlv2/tree.go index 5a680eeba694..a8cd7ef26024 100644 --- a/store/v2/commitment/iavlv2/tree.go +++ b/store/v2/commitment/iavlv2/tree.go @@ -52,6 +52,10 @@ func (t *Tree) Version() uint64 { } func (t *Tree) LoadVersion(version uint64) error { + if err := isHighBitSet(version); err != nil { + return err + } + if version == 0 { return nil } @@ -64,15 +68,24 @@ func (t *Tree) Commit() ([]byte, uint64, error) { } func (t *Tree) SetInitialVersion(version uint64) error { + if err := isHighBitSet(version); err != nil { + return err + } t.tree.SetShouldCheckpoint() return t.tree.SetInitialVersion(int64(version)) } func (t *Tree) GetProof(version uint64, key []byte) (*ics23.CommitmentProof, error) { + if err := isHighBitSet(version); err != nil { + return nil, err + } return t.tree.GetProof(int64(version), key) } func (t *Tree) Get(version uint64, key []byte) ([]byte, error) { + if err := isHighBitSet(version); err != nil { + return nil, err + } if int64(version) != t.tree.Version() { return nil, fmt.Errorf("loading past version not yet supported") } @@ -92,6 +105,10 @@ func (t *Tree) Close() error { } func (t *Tree) Prune(version uint64) error { + if err := isHighBitSet(version); err != nil { + return err + } + return t.tree.DeleteVersionsTo(int64(version)) } @@ -101,3 +118,10 @@ func (t *Tree) PausePruning(bool) {} func (t *Tree) WorkingHash() []byte { return t.tree.Hash() } + +func isHighBitSet(version uint64) error { + if version&(1<<63) != 0 { + return fmt.Errorf("%d too large; uint64 with the highest bit set are not supported", version) + } + return nil +} diff --git a/store/v2/storage/sqlite/db.go b/store/v2/storage/sqlite/db.go index 9f011864e33d..baf1ea7a7a8d 100644 --- a/store/v2/storage/sqlite/db.go +++ b/store/v2/storage/sqlite/db.go @@ -123,7 +123,7 @@ func (db *Database) GetLatestVersion() (version uint64, err error) { defer func(stmt *sqlite3.Stmt) { cErr := stmt.Close() if cErr != nil { - err = fmt.Errorf("failed to close GetLatestVersion statement: %w", cErr) + err = errors.Join(err, fmt.Errorf("failed to close GetLatestVersion statement: %w", cErr)) } }(stmt) @@ -325,7 +325,7 @@ func (db *Database) PruneStoreKeys(storeKeys []string, version uint64) (err erro } defer func() { if err != nil { - err = db.storage.Rollback() + err = errors.Join(err, db.storage.Rollback()) } }() From 2f7ef1ec1481f74682991847f91772c4d0585a29 Mon Sep 17 00:00:00 2001 From: Matt Kocubinski Date: Mon, 4 Nov 2024 17:37:57 -0600 Subject: [PATCH 11/20] fix prune rollback --- store/v2/storage/sqlite/db.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/store/v2/storage/sqlite/db.go b/store/v2/storage/sqlite/db.go index baf1ea7a7a8d..6cd826757b01 100644 --- a/store/v2/storage/sqlite/db.go +++ b/store/v2/storage/sqlite/db.go @@ -239,7 +239,7 @@ func (db *Database) Prune(version uint64) (err error) { } defer func() { if err != nil { - err = errors.Join(db.storage.Rollback()) + err = errors.Join(err, db.storage.Rollback()) } }() From 84e5b389eedd170b797c39e51d25f4914b55cfb8 Mon Sep 17 00:00:00 2001 From: Matt Kocubinski Date: Thu, 28 Nov 2024 14:37:41 -0600 Subject: [PATCH 12/20] fix upgrades test, it was confused --- store/v2/commitment/iavl/tree_test.go | 7 +++- store/v2/commitment/iavlv2/tree.go | 22 +++++++++++- store/v2/commitment/iavlv2/tree_test.go | 45 +++++++++++++++++++++++ store/v2/commitment/store_test_suite.go | 47 ++++++++++++++----------- store/v2/go.mod | 5 ++- 5 files changed, 103 insertions(+), 23 deletions(-) create mode 100644 store/v2/commitment/iavlv2/tree_test.go diff --git a/store/v2/commitment/iavl/tree_test.go b/store/v2/commitment/iavl/tree_test.go index 65503190b943..728347999a31 100644 --- a/store/v2/commitment/iavl/tree_test.go +++ b/store/v2/commitment/iavl/tree_test.go @@ -16,7 +16,12 @@ import ( func TestCommitterSuite(t *testing.T) { s := &commitment.CommitStoreTestSuite{ - NewStore: func(db corestore.KVStoreWithBatch, storeKeys, oldStoreKeys []string, logger corelog.Logger) (*commitment.CommitStore, error) { + NewStore: func( + db corestore.KVStoreWithBatch, + _ string, + storeKeys, oldStoreKeys []string, + logger corelog.Logger, + ) (*commitment.CommitStore, error) { multiTrees := make(map[string]commitment.Tree) cfg := DefaultConfig() mountTreeFn := func(storeKey string) (commitment.Tree, error) { diff --git a/store/v2/commitment/iavlv2/tree.go b/store/v2/commitment/iavlv2/tree.go index a8cd7ef26024..dddc83ce2ea9 100644 --- a/store/v2/commitment/iavlv2/tree.go +++ b/store/v2/commitment/iavlv2/tree.go @@ -1,6 +1,7 @@ package iavlv2 import ( + corestore "cosmossdk.io/core/store" "errors" "fmt" @@ -13,6 +14,7 @@ import ( var ( _ commitment.Tree = (*Tree)(nil) + _ commitment.Reader = (*Tree)(nil) _ store.PausablePruner = (*Tree)(nil) ) @@ -83,13 +85,31 @@ func (t *Tree) GetProof(version uint64, key []byte) (*ics23.CommitmentProof, err } func (t *Tree) Get(version uint64, key []byte) ([]byte, error) { + if err := isHighBitSet(version); err != nil { + return nil, err + } + if int64(version) != t.tree.Version() { + cloned, err := t.tree.ReadonlyClone() + if err != nil { + return nil, err + } + if err = cloned.LoadVersion(int64(version)); err != nil { + return nil, err + } + return cloned.Get(key) + } else { + return t.tree.Get(key) + } +} + +func (t *Tree) Iterator(version uint64, start, end []byte, ascending bool) (corestore.Iterator, error) { if err := isHighBitSet(version); err != nil { return nil, err } if int64(version) != t.tree.Version() { return nil, fmt.Errorf("loading past version not yet supported") } - return t.tree.Get(key) + return t.tree.Iterator(start, end, ascending) } func (t *Tree) Export(version uint64) (commitment.Exporter, error) { diff --git a/store/v2/commitment/iavlv2/tree_test.go b/store/v2/commitment/iavlv2/tree_test.go new file mode 100644 index 000000000000..16f82f19e824 --- /dev/null +++ b/store/v2/commitment/iavlv2/tree_test.go @@ -0,0 +1,45 @@ +package iavlv2 + +import ( + "fmt" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "testing" + + "github.com/cosmos/iavl/v2" + + corelog "cosmossdk.io/core/log" + corestore "cosmossdk.io/core/store" + "cosmossdk.io/store/v2/commitment" +) + +func TestCommitterSuite(t *testing.T) { + nodePool := iavl.NewNodePool() + s := &commitment.CommitStoreTestSuite{ + NewStore: func( + db corestore.KVStoreWithBatch, + dbDir string, + storeKeys, oldStoreKeys []string, + logger corelog.Logger, + ) (*commitment.CommitStore, error) { + multiTrees := make(map[string]commitment.Tree) + mountTreeFn := func(storeKey string) (commitment.Tree, error) { + path := fmt.Sprintf("%s/%s", dbDir, storeKey) + tree, err := NewTree(iavl.DefaultTreeOptions(), iavl.SqliteDbOptions{Path: path}, nodePool) + require.NoError(t, err) + return tree, nil + } + for _, storeKey := range storeKeys { + multiTrees[storeKey], _ = mountTreeFn(storeKey) + } + oldTrees := make(map[string]commitment.Tree) + for _, storeKey := range oldStoreKeys { + oldTrees[storeKey], _ = mountTreeFn(storeKey) + } + + return commitment.NewCommitStore(multiTrees, oldTrees, db, logger) + }, + } + + suite.Run(t, s) +} diff --git a/store/v2/commitment/store_test_suite.go b/store/v2/commitment/store_test_suite.go index afb468de8595..4a25cb89cab0 100644 --- a/store/v2/commitment/store_test_suite.go +++ b/store/v2/commitment/store_test_suite.go @@ -28,12 +28,12 @@ const ( type CommitStoreTestSuite struct { suite.Suite - NewStore func(db corestore.KVStoreWithBatch, storeKeys, oldStoreKeys []string, logger corelog.Logger) (*CommitStore, error) + NewStore func(db corestore.KVStoreWithBatch, dbDir string, storeKeys, oldStoreKeys []string, logger corelog.Logger) (*CommitStore, error) } func (s *CommitStoreTestSuite) TestStore_Snapshotter() { storeKeys := []string{storeKey1, storeKey2} - commitStore, err := s.NewStore(dbm.NewMemDB(), storeKeys, nil, coretesting.NewNopLogger()) + commitStore, err := s.NewStore(dbm.NewMemDB(), s.T().TempDir(), storeKeys, nil, coretesting.NewNopLogger()) s.Require().NoError(err) latestVersion := uint64(10) @@ -67,7 +67,7 @@ func (s *CommitStoreTestSuite) TestStore_Snapshotter() { }, } - targetStore, err := s.NewStore(dbm.NewMemDB(), storeKeys, nil, coretesting.NewNopLogger()) + targetStore, err := s.NewStore(dbm.NewMemDB(), s.T().TempDir(), storeKeys, nil, coretesting.NewNopLogger()) s.Require().NoError(err) chunks := make(chan io.ReadCloser, kvCount*int(latestVersion)) @@ -130,7 +130,8 @@ func (s *CommitStoreTestSuite) TestStore_Snapshotter() { func (s *CommitStoreTestSuite) TestStore_LoadVersion() { storeKeys := []string{storeKey1, storeKey2} mdb := dbm.NewMemDB() - commitStore, err := s.NewStore(mdb, storeKeys, nil, coretesting.NewNopLogger()) + dbDir := s.T().TempDir() + commitStore, err := s.NewStore(mdb, dbDir, storeKeys, nil, coretesting.NewNopLogger()) s.Require().NoError(err) latestVersion := uint64(10) @@ -151,7 +152,7 @@ func (s *CommitStoreTestSuite) TestStore_LoadVersion() { } // load the store with the latest version - targetStore, err := s.NewStore(mdb, storeKeys, nil, coretesting.NewNopLogger()) + targetStore, err := s.NewStore(mdb, dbDir, storeKeys, nil, coretesting.NewNopLogger()) s.Require().NoError(err) err = targetStore.LoadVersion(latestVersion) s.Require().NoError(err) @@ -164,7 +165,7 @@ func (s *CommitStoreTestSuite) TestStore_LoadVersion() { // rollback to a previous version rollbackVersion := uint64(5) - rollbackStore, err := s.NewStore(mdb, storeKeys, nil, coretesting.NewNopLogger()) + rollbackStore, err := s.NewStore(mdb, dbDir, storeKeys, nil, coretesting.NewNopLogger()) s.Require().NoError(err) err = rollbackStore.LoadVersion(rollbackVersion) s.Require().NoError(err) @@ -185,7 +186,7 @@ func (s *CommitStoreTestSuite) TestStore_LoadVersion() { func (s *CommitStoreTestSuite) TestStore_Pruning() { storeKeys := []string{storeKey1, storeKey2} pruneOpts := store.NewPruningOptionWithCustom(10, 5) - commitStore, err := s.NewStore(dbm.NewMemDB(), storeKeys, nil, coretesting.NewNopLogger()) + commitStore, err := s.NewStore(dbm.NewMemDB(), s.T().TempDir(), storeKeys, nil, coretesting.NewNopLogger()) s.Require().NoError(err) latestVersion := uint64(100) @@ -225,7 +226,7 @@ func (s *CommitStoreTestSuite) TestStore_Pruning() { func (s *CommitStoreTestSuite) TestStore_GetProof() { storeKeys := []string{storeKey1, storeKey2} - commitStore, err := s.NewStore(dbm.NewMemDB(), storeKeys, nil, coretesting.NewNopLogger()) + commitStore, err := s.NewStore(dbm.NewMemDB(), s.T().TempDir(), storeKeys, nil, coretesting.NewNopLogger()) s.Require().NoError(err) toVersion := uint64(10) @@ -268,7 +269,7 @@ func (s *CommitStoreTestSuite) TestStore_GetProof() { func (s *CommitStoreTestSuite) TestStore_Get() { storeKeys := []string{storeKey1, storeKey2} - commitStore, err := s.NewStore(dbm.NewMemDB(), storeKeys, nil, coretesting.NewNopLogger()) + commitStore, err := s.NewStore(dbm.NewMemDB(), s.T().TempDir(), storeKeys, nil, coretesting.NewNopLogger()) s.Require().NoError(err) toVersion := uint64(10) @@ -303,7 +304,8 @@ func (s *CommitStoreTestSuite) TestStore_Get() { func (s *CommitStoreTestSuite) TestStore_Upgrades() { storeKeys := []string{storeKey1, storeKey2, storeKey3} commitDB := dbm.NewMemDB() - commitStore, err := s.NewStore(commitDB, storeKeys, nil, coretesting.NewNopLogger()) + commitDir := s.T().TempDir() + commitStore, err := s.NewStore(commitDB, commitDir, storeKeys, nil, coretesting.NewNopLogger()) s.Require().NoError(err) latestVersion := uint64(10) @@ -330,14 +332,14 @@ func (s *CommitStoreTestSuite) TestStore_Upgrades() { } newStoreKeys := []string{storeKey1, storeKey2, storeKey3, "newStore1", "newStore2"} realStoreKeys := []string{storeKey1, storeKey2, "newStore1", "newStore2"} - oldStoreKeys := []string{storeKey1, storeKey3} - commitStore, err = s.NewStore(commitDB, newStoreKeys, oldStoreKeys, coretesting.NewNopLogger()) + oldStoreKeys := []string{storeKey3} + commitStore, err = s.NewStore(commitDB, commitDir, newStoreKeys, oldStoreKeys, coretesting.NewNopLogger()) s.Require().NoError(err) err = commitStore.LoadVersionAndUpgrade(latestVersion, upgrades) s.Require().NoError(err) // GetProof should work for the old stores - for _, storeKey := range []string{storeKey1, storeKey3} { + for _, storeKey := range []string{storeKey3} { for i := uint64(1); i <= latestVersion; i++ { for j := 0; j < kvCount; j++ { proof, err := commitStore.GetProof([]byte(storeKey), i, []byte(fmt.Sprintf("key-%d-%d", i, j))) @@ -391,20 +393,20 @@ func (s *CommitStoreTestSuite) TestStore_Upgrades() { // verify existing store for i := uint64(1); i < latestVersion*2; i++ { for j := 0; j < kvCount; j++ { - proof, err := commitStore.GetProof([]byte(storeKey2), i, []byte(fmt.Sprintf("key-%d-%d", i, j))) + prf, err := commitStore.GetProof([]byte(storeKey2), i, []byte(fmt.Sprintf("key-%d-%d", i, j))) s.Require().NoError(err) - s.Require().NotNil(proof) + s.Require().NotNil(prf) } } // create a new commitment store with one more upgrades upgrades = &corestore.StoreUpgrades{ - Added: []string{storeKey3}, Deleted: []string{storeKey2}, + Added: []string{"newStore3"}, } - newRealStoreKeys := []string{storeKey1, storeKey3, "newStore1", "newStore2"} + newRealStoreKeys := []string{storeKey1, "newStore1", "newStore2", "newStore3"} oldStoreKeys = []string{storeKey2, storeKey3} - commitStore, err = s.NewStore(commitDB, newStoreKeys, oldStoreKeys, coretesting.NewNopLogger()) + commitStore, err = s.NewStore(commitDB, commitDir, newRealStoreKeys, oldStoreKeys, coretesting.NewNopLogger()) s.Require().NoError(err) err = commitStore.LoadVersionAndUpgrade(2*latestVersion-1, upgrades) s.Require().NoError(err) @@ -420,7 +422,8 @@ func (s *CommitStoreTestSuite) TestStore_Upgrades() { kvPairs[storeKey] = append(kvPairs[storeKey], corestore.KVPair{Key: key, Value: value}) } } - s.Require().NoError(commitStore.WriteChangeset(corestore.NewChangesetWithPairs(i, kvPairs))) + err = commitStore.WriteChangeset(corestore.NewChangesetWithPairs(i, kvPairs)) + s.Require().NoError(err) commitInfo, err := commitStore.Commit(i) s.Require().NoError(err) s.Require().NotNil(commitInfo) @@ -432,6 +435,7 @@ func (s *CommitStoreTestSuite) TestStore_Upgrades() { // prune the old stores s.Require().NoError(commitStore.Prune(latestVersion)) + s.T().Logf("prune to version %d", latestVersion) // GetProof should fail for the old stores for _, storeKey := range []string{storeKey1, storeKey3} { for i := uint64(1); i <= latestVersion; i++ { @@ -441,6 +445,7 @@ func (s *CommitStoreTestSuite) TestStore_Upgrades() { } } } + s.T().Log("GetProof should work for the new stores") // GetProof should not fail for the newly removed store for i := latestVersion + 1; i < latestVersion*2; i++ { for j := 0; j < kvCount; j++ { @@ -450,6 +455,7 @@ func (s *CommitStoreTestSuite) TestStore_Upgrades() { } } + s.T().Logf("Prune to version %d", latestVersion*2) s.Require().NoError(commitStore.Prune(latestVersion * 2)) // GetProof should fail for the newly deleted stores for i := uint64(1); i < latestVersion*2; i++ { @@ -458,10 +464,11 @@ func (s *CommitStoreTestSuite) TestStore_Upgrades() { s.Require().Error(err) } } + s.T().Log("GetProof should work for the new added store") // GetProof should work for the new added store for i := latestVersion*2 + 1; i < latestVersion*3; i++ { for j := 0; j < kvCount; j++ { - proof, err := commitStore.GetProof([]byte(storeKey3), i, []byte(fmt.Sprintf("key-%d-%d", i, j))) + proof, err := commitStore.GetProof([]byte("newStore3"), i, []byte(fmt.Sprintf("key-%d-%d", i, j))) s.Require().NoError(err) s.Require().NotNil(proof) } diff --git a/store/v2/go.mod b/store/v2/go.mod index 3f22d31f48cf..f94f3d9019c9 100644 --- a/store/v2/go.mod +++ b/store/v2/go.mod @@ -76,4 +76,7 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect ) -replace cosmossdk.io/core => ../../core +replace ( + cosmossdk.io/core => ../../core + github.com/cosmos/iavl/v2 => ../../../../iavl/iavl-v2 +) From 37147bf9cdb69743eddb9164e14fbeb9ebab5b62 Mon Sep 17 00:00:00 2001 From: Matt Kocubinski Date: Thu, 28 Nov 2024 14:43:30 -0600 Subject: [PATCH 13/20] skip snapshot test for now --- store/v2/commitment/iavlv2/tree_test.go | 1 + store/v2/commitment/store_test_suite.go | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/store/v2/commitment/iavlv2/tree_test.go b/store/v2/commitment/iavlv2/tree_test.go index 16f82f19e824..0afc9f453795 100644 --- a/store/v2/commitment/iavlv2/tree_test.go +++ b/store/v2/commitment/iavlv2/tree_test.go @@ -16,6 +16,7 @@ import ( func TestCommitterSuite(t *testing.T) { nodePool := iavl.NewNodePool() s := &commitment.CommitStoreTestSuite{ + TreeType: "iavlv2", NewStore: func( db corestore.KVStoreWithBatch, dbDir string, diff --git a/store/v2/commitment/store_test_suite.go b/store/v2/commitment/store_test_suite.go index 4a25cb89cab0..c41a2540a070 100644 --- a/store/v2/commitment/store_test_suite.go +++ b/store/v2/commitment/store_test_suite.go @@ -29,9 +29,13 @@ type CommitStoreTestSuite struct { suite.Suite NewStore func(db corestore.KVStoreWithBatch, dbDir string, storeKeys, oldStoreKeys []string, logger corelog.Logger) (*CommitStore, error) + TreeType string } func (s *CommitStoreTestSuite) TestStore_Snapshotter() { + if s.TreeType == "iavlv2" { + s.T().Skip("FIXME: iavlv2 does not yet support snapshots") + } storeKeys := []string{storeKey1, storeKey2} commitStore, err := s.NewStore(dbm.NewMemDB(), s.T().TempDir(), storeKeys, nil, coretesting.NewNopLogger()) s.Require().NoError(err) From c815594aec3b2f40ea2becab3fb4e989250f76f0 Mon Sep 17 00:00:00 2001 From: Matt Kocubinski Date: Thu, 28 Nov 2024 14:52:45 -0600 Subject: [PATCH 14/20] go mod tidy --- server/v2/cometbft/go.mod | 1 - simapp/v2/go.mod | 1 - store/v2/go.mod | 7 ++----- store/v2/go.sum | 4 ++-- 4 files changed, 4 insertions(+), 9 deletions(-) diff --git a/server/v2/cometbft/go.mod b/server/v2/cometbft/go.mod index e1d55fb80a7e..e0a5cd741e1e 100644 --- a/server/v2/cometbft/go.mod +++ b/server/v2/cometbft/go.mod @@ -60,7 +60,6 @@ require ( github.com/Microsoft/go-winio v0.6.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bgentry/speakeasy v0.2.0 // indirect - github.com/btcsuite/btcd/btcec/v2 v2.3.4 // indirect github.com/bvinc/go-sqlite-lite v0.6.1 // indirect github.com/bytedance/sonic v1.12.4 // indirect github.com/bytedance/sonic/loader v0.2.1 // indirect diff --git a/simapp/v2/go.mod b/simapp/v2/go.mod index df15f64cc88d..dffbac74044b 100644 --- a/simapp/v2/go.mod +++ b/simapp/v2/go.mod @@ -78,7 +78,6 @@ require ( github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect github.com/bgentry/speakeasy v0.2.0 // indirect github.com/bits-and-blooms/bitset v1.10.0 // indirect - github.com/btcsuite/btcd/btcec/v2 v2.3.4 // indirect github.com/bvinc/go-sqlite-lite v0.6.1 // indirect github.com/bytedance/sonic v1.12.4 // indirect github.com/bytedance/sonic/loader v0.2.1 // indirect diff --git a/store/v2/go.mod b/store/v2/go.mod index f94f3d9019c9..a2351efc3b92 100644 --- a/store/v2/go.mod +++ b/store/v2/go.mod @@ -12,7 +12,7 @@ require ( github.com/cosmos/cosmos-proto v1.0.0-beta.5 github.com/cosmos/gogoproto v1.7.0 github.com/cosmos/iavl v1.3.1 - github.com/cosmos/iavl/v2 v2.0.0-20241104195329-835276bdf831 + github.com/cosmos/iavl/v2 v2.0.0-20241128205019-1b18c0edbbd9 github.com/cosmos/ics23/go v0.11.0 github.com/google/btree v1.1.3 github.com/hashicorp/go-metrics v0.5.3 @@ -76,7 +76,4 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect ) -replace ( - cosmossdk.io/core => ../../core - github.com/cosmos/iavl/v2 => ../../../../iavl/iavl-v2 -) +replace cosmossdk.io/core => ../../core diff --git a/store/v2/go.sum b/store/v2/go.sum index 95f4d82ffaf3..fa8b03c077ac 100644 --- a/store/v2/go.sum +++ b/store/v2/go.sum @@ -59,8 +59,8 @@ github.com/cosmos/iavl v1.3.1 h1:+W1G2uSUtJMqMGpwz/fKiwZxY2DDT/9/0hyNLm6Geu0= github.com/cosmos/iavl v1.3.1/go.mod h1:T6SfBcyhulVIY2G/ZtAtQm/QiJvsuhIos52V4dWYk88= github.com/cosmos/iavl-bench/bench v0.0.4 h1:J6zQPiBqF4CXMM3QBsLqZgQEBGY0taX85vLIZMhmAfQ= github.com/cosmos/iavl-bench/bench v0.0.4/go.mod h1:j2rLae77EffacWcp7mmj3Uaa4AOAmZA7ymvhsuBQKKI= -github.com/cosmos/iavl/v2 v2.0.0-20241104195329-835276bdf831 h1:K5TLZIz7vh/6k23zSbEj0nUYeNO8RuZB5xD7/f9tWeM= -github.com/cosmos/iavl/v2 v2.0.0-20241104195329-835276bdf831/go.mod h1:7RSm0aeApe3S1x4TrLffvUL6pjOtMYV4glYnpAhr2lw= +github.com/cosmos/iavl/v2 v2.0.0-20241128205019-1b18c0edbbd9 h1:H+ttW6HTzezz2l3Fp/hFNNHWA+a+7qZgNDE5OFySTiY= +github.com/cosmos/iavl/v2 v2.0.0-20241128205019-1b18c0edbbd9/go.mod h1:7RSm0aeApe3S1x4TrLffvUL6pjOtMYV4glYnpAhr2lw= github.com/cosmos/ics23/go v0.11.0 h1:jk5skjT0TqX5e5QJbEnwXIS2yI2vnmLOgpQPeM5RtnU= github.com/cosmos/ics23/go v0.11.0/go.mod h1:A8OjxPE67hHST4Icw94hOxxFEJMBG031xIGF/JHNIY0= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= From 850914e2df4814921548af065ef69c0be3fd598d Mon Sep 17 00:00:00 2001 From: Matt Kocubinski Date: Thu, 28 Nov 2024 14:56:59 -0600 Subject: [PATCH 15/20] lint fix --- store/v2/commitment/iavlv2/tree.go | 2 +- store/v2/commitment/iavlv2/tree_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/store/v2/commitment/iavlv2/tree.go b/store/v2/commitment/iavlv2/tree.go index dddc83ce2ea9..003c5f4f45e0 100644 --- a/store/v2/commitment/iavlv2/tree.go +++ b/store/v2/commitment/iavlv2/tree.go @@ -1,10 +1,10 @@ package iavlv2 import ( - corestore "cosmossdk.io/core/store" "errors" "fmt" + corestore "cosmossdk.io/core/store" "github.com/cosmos/iavl/v2" ics23 "github.com/cosmos/ics23/go" diff --git a/store/v2/commitment/iavlv2/tree_test.go b/store/v2/commitment/iavlv2/tree_test.go index 0afc9f453795..a173c453c976 100644 --- a/store/v2/commitment/iavlv2/tree_test.go +++ b/store/v2/commitment/iavlv2/tree_test.go @@ -2,11 +2,11 @@ package iavlv2 import ( "fmt" - "github.com/stretchr/testify/require" - "github.com/stretchr/testify/suite" "testing" "github.com/cosmos/iavl/v2" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" corelog "cosmossdk.io/core/log" corestore "cosmossdk.io/core/store" From b59fe7f5735d61196e839987f0b3b9cd9bca72da Mon Sep 17 00:00:00 2001 From: Matt Kocubinski Date: Fri, 29 Nov 2024 11:14:12 -0600 Subject: [PATCH 16/20] rm sqlite ss --- store/v2/pruning/manager_test.go | 6 +- store/v2/root/factory.go | 12 +- store/v2/root/migrate_test.go | 6 +- store/v2/root/store_test.go | 22 +- store/v2/root/upgrade_test.go | 6 +- store/v2/storage/pebbledb/batch.go | 1 + store/v2/storage/sqlite/batch.go | 113 -------- store/v2/storage/sqlite/db.go | 416 ---------------------------- store/v2/storage/sqlite/db_test.go | 210 -------------- store/v2/storage/sqlite/iterator.go | 186 ------------- store/v2/storage/store.go | 3 +- 11 files changed, 25 insertions(+), 956 deletions(-) delete mode 100644 store/v2/storage/sqlite/batch.go delete mode 100644 store/v2/storage/sqlite/db.go delete mode 100644 store/v2/storage/sqlite/db_test.go delete mode 100644 store/v2/storage/sqlite/iterator.go diff --git a/store/v2/pruning/manager_test.go b/store/v2/pruning/manager_test.go index 0a0333451ae5..3dd7ec5fd336 100644 --- a/store/v2/pruning/manager_test.go +++ b/store/v2/pruning/manager_test.go @@ -15,7 +15,7 @@ import ( "cosmossdk.io/store/v2/commitment/iavl" dbm "cosmossdk.io/store/v2/db" "cosmossdk.io/store/v2/storage" - "cosmossdk.io/store/v2/storage/sqlite" + "cosmossdk.io/store/v2/storage/pebbledb" ) var storeKeys = []string{"store1", "store2", "store3"} @@ -45,9 +45,9 @@ func (s *PruningManagerTestSuite) SetupTest() { s.sc, err = commitment.NewCommitStore(multiTrees, nil, mdb, nopLog) s.Require().NoError(err) - sqliteDB, err := sqlite.New(s.T().TempDir()) + pebbleDB, err := pebbledb.New(s.T().TempDir()) s.Require().NoError(err) - s.ss = storage.NewStorageStore(sqliteDB, nopLog) + s.ss = storage.NewStorageStore(pebbleDB, nopLog) scPruningOption := store.NewPruningOptionWithCustom(0, 1) // prune all ssPruningOption := store.NewPruningOptionWithCustom(5, 10) // prune some s.manager = NewManager(s.sc, s.ss, scPruningOption, ssPruningOption) diff --git a/store/v2/root/factory.go b/store/v2/root/factory.go index 2511a53b434e..2bd710182f26 100644 --- a/store/v2/root/factory.go +++ b/store/v2/root/factory.go @@ -17,7 +17,6 @@ import ( "cosmossdk.io/store/v2/storage" "cosmossdk.io/store/v2/storage/pebbledb" "cosmossdk.io/store/v2/storage/rocksdb" - "cosmossdk.io/store/v2/storage/sqlite" ) type ( @@ -26,7 +25,6 @@ type ( ) const ( - SSTypeSQLite SSType = "sqlite" SSTypePebble SSType = "pebble" SSTypeRocks SSType = "rocksdb" SCTypeIavl SCType = "iavl" @@ -35,7 +33,7 @@ const ( // Options are the options for creating a root store. type Options struct { - SSType SSType `mapstructure:"ss-type" toml:"ss-type" comment:"State storage database type. Currently we support: \"sqlite\", \"pebble\" and \"rocksdb\""` + SSType SSType `mapstructure:"ss-type" toml:"ss-type" comment:"State storage database type. Currently we support: \"pebble\" and \"rocksdb\""` SCType SCType `mapstructure:"sc-type" toml:"sc-type" comment:"State commitment database type. Currently we support: \"iavl\" and \"iavl-v2\""` SSPruningOption *store.PruningOption `mapstructure:"ss-pruning-option" toml:"ss-pruning-option" comment:"Pruning options for state storage"` SCPruningOption *store.PruningOption `mapstructure:"sc-pruning-option" toml:"sc-pruning-option" comment:"Pruning options for state commitment"` @@ -54,7 +52,7 @@ type FactoryOptions struct { // DefaultStoreOptions returns the default options for creating a root store. func DefaultStoreOptions() Options { return Options{ - SSType: SSTypeSQLite, + SSType: SSTypePebble, SCType: SCTypeIavl, SCPruningOption: &store.PruningOption{ KeepRecent: 2, @@ -91,12 +89,6 @@ func CreateRootStore(opts *FactoryOptions) (store.RootStore, error) { storeOpts := opts.Options switch storeOpts.SSType { - case SSTypeSQLite: - dir := fmt.Sprintf("%s/data/ss/sqlite", opts.RootDir) - if err = ensureDir(dir); err != nil { - return nil, err - } - ssDb, err = sqlite.New(dir) case SSTypePebble: dir := fmt.Sprintf("%s/data/ss/pebble", opts.RootDir) if err = ensureDir(dir); err != nil { diff --git a/store/v2/root/migrate_test.go b/store/v2/root/migrate_test.go index 2d83038b812e..82a53919d93f 100644 --- a/store/v2/root/migrate_test.go +++ b/store/v2/root/migrate_test.go @@ -18,7 +18,7 @@ import ( "cosmossdk.io/store/v2/pruning" "cosmossdk.io/store/v2/snapshots" "cosmossdk.io/store/v2/storage" - "cosmossdk.io/store/v2/storage/sqlite" + "cosmossdk.io/store/v2/storage/pebbledb" ) var storeKeys = []string{"store1", "store2", "store3"} @@ -62,9 +62,9 @@ func (s *MigrateStoreTestSuite) SetupTest() { } // create a new storage and commitment stores - sqliteDB, err := sqlite.New(s.T().TempDir()) + pebbleDB, err := pebbledb.New(s.T().TempDir()) s.Require().NoError(err) - ss := storage.NewStorageStore(sqliteDB, testLog) + ss := storage.NewStorageStore(pebbleDB, testLog) multiTrees1 := make(map[string]commitment.Tree) for _, storeKey := range storeKeys { diff --git a/store/v2/root/store_test.go b/store/v2/root/store_test.go index 10a3a049d3c0..59a490b11b00 100644 --- a/store/v2/root/store_test.go +++ b/store/v2/root/store_test.go @@ -17,7 +17,7 @@ import ( "cosmossdk.io/store/v2/proof" "cosmossdk.io/store/v2/pruning" "cosmossdk.io/store/v2/storage" - "cosmossdk.io/store/v2/storage/sqlite" + "cosmossdk.io/store/v2/storage/pebbledb" ) const ( @@ -47,9 +47,9 @@ func TestStorageTestSuite(t *testing.T) { func (s *RootStoreTestSuite) SetupTest() { noopLog := coretesting.NewNopLogger() - sqliteDB, err := sqlite.New(s.T().TempDir()) + pebbleDB, err := pebbledb.New(s.T().TempDir()) s.Require().NoError(err) - ss := storage.NewStorageStore(sqliteDB, noopLog) + ss := storage.NewStorageStore(pebbleDB, noopLog) tree := iavl.NewIavlTree(dbm.NewMemDB(), noopLog, iavl.DefaultConfig()) tree2 := iavl.NewIavlTree(dbm.NewMemDB(), noopLog, iavl.DefaultConfig()) @@ -67,9 +67,9 @@ func (s *RootStoreTestSuite) SetupTest() { func (s *RootStoreTestSuite) newStoreWithPruneConfig(config *store.PruningOption) { noopLog := coretesting.NewNopLogger() - sqliteDB, err := sqlite.New(s.T().TempDir()) + pebbleDB, err := pebbledb.New(s.T().TempDir()) s.Require().NoError(err) - ss := storage.NewStorageStore(sqliteDB, noopLog) + ss := storage.NewStorageStore(pebbleDB, noopLog) mdb := dbm.NewMemDB() multiTrees := make(map[string]commitment.Tree) @@ -535,9 +535,9 @@ func (s *RootStoreTestSuite) TestMultiStore_PruningRestart() { mdb1 := dbm.NewMemDB() mdb2 := dbm.NewMemDB() - sqliteDB, err := sqlite.New(s.T().TempDir()) + pebbleDB, err := pebbledb.New(s.T().TempDir()) s.Require().NoError(err) - ss := storage.NewStorageStore(sqliteDB, noopLog) + ss := storage.NewStorageStore(pebbleDB, noopLog) tree := iavl.NewIavlTree(mdb1, noopLog, iavl.DefaultConfig()) sc, err := commitment.NewCommitStore(map[string]commitment.Tree{testStoreKey: tree}, nil, mdb2, noopLog) @@ -566,9 +566,9 @@ func (s *RootStoreTestSuite) TestMultiStore_PruningRestart() { s.Require().Equal(uint64(0), actualHeightToPrune) // "restart" - sqliteDB, err = sqlite.New(s.T().TempDir()) + pebbleDB, err = pebbledb.New(s.T().TempDir()) s.Require().NoError(err) - ss = storage.NewStorageStore(sqliteDB, noopLog) + ss = storage.NewStorageStore(pebbleDB, noopLog) tree = iavl.NewIavlTree(mdb1, noopLog, iavl.DefaultConfig()) sc, err = commitment.NewCommitStore(map[string]commitment.Tree{testStoreKey: tree}, nil, mdb2, noopLog) @@ -616,10 +616,10 @@ func (s *RootStoreTestSuite) TestMultiStore_PruningRestart() { func (s *RootStoreTestSuite) TestMultiStoreRestart() { noopLog := coretesting.NewNopLogger() - sqliteDB, err := sqlite.New(s.T().TempDir()) + pebbleDB, err := pebbledb.New(s.T().TempDir()) s.Require().NoError(err) - ss := storage.NewStorageStore(sqliteDB, noopLog) + ss := storage.NewStorageStore(pebbleDB, noopLog) mdb1 := dbm.NewMemDB() mdb2 := dbm.NewMemDB() diff --git a/store/v2/root/upgrade_test.go b/store/v2/root/upgrade_test.go index 400ddb2c4d65..0e9da266a355 100644 --- a/store/v2/root/upgrade_test.go +++ b/store/v2/root/upgrade_test.go @@ -15,7 +15,7 @@ import ( dbm "cosmossdk.io/store/v2/db" "cosmossdk.io/store/v2/pruning" "cosmossdk.io/store/v2/storage" - "cosmossdk.io/store/v2/storage/sqlite" + "cosmossdk.io/store/v2/storage/pebbledb" ) type UpgradeStoreTestSuite struct { @@ -44,9 +44,9 @@ func (s *UpgradeStoreTestSuite) SetupTest() { } // create storage and commitment stores - sqliteDB, err := sqlite.New(s.T().TempDir()) + pebbleDB, err := pebbledb.New(s.T().TempDir()) s.Require().NoError(err) - ss := storage.NewStorageStore(sqliteDB, testLog) + ss := storage.NewStorageStore(pebbleDB, testLog) sc, err := commitment.NewCommitStore(multiTrees, nil, s.commitDB, testLog) s.Require().NoError(err) pm := pruning.NewManager(sc, ss, nil, nil) diff --git a/store/v2/storage/pebbledb/batch.go b/store/v2/storage/pebbledb/batch.go index fdd58f447435..7e9f7a347f98 100644 --- a/store/v2/storage/pebbledb/batch.go +++ b/store/v2/storage/pebbledb/batch.go @@ -88,6 +88,7 @@ func (b *Batch) Delete(storeKey, key []byte) error { return b.set(storeKey, b.version, key, []byte(tombstoneVal)) } +// Write flushes any accumulated data to disk and closes the batch. func (b *Batch) Write() (err error) { defer func() { err = errors.Join(err, b.batch.Close()) diff --git a/store/v2/storage/sqlite/batch.go b/store/v2/storage/sqlite/batch.go deleted file mode 100644 index 01252f2e0c7c..000000000000 --- a/store/v2/storage/sqlite/batch.go +++ /dev/null @@ -1,113 +0,0 @@ -package sqlite - -import ( - "errors" - "fmt" - "sync" - - "github.com/bvinc/go-sqlite-lite/sqlite3" - - "cosmossdk.io/store/v2" -) - -var _ store.Batch = (*Batch)(nil) - -type batchAction int - -const ( - batchActionSet batchAction = 0 - batchActionDel batchAction = 1 -) - -type batchOp struct { - action batchAction - storeKey []byte - key, value []byte -} - -type Batch struct { - db *sqlite3.Conn - lock *sync.Mutex - ops []batchOp - size int - version int64 -} - -func NewBatch(db *sqlite3.Conn, writeLock *sync.Mutex, version uint64) (*Batch, error) { - if isHighBitSet(version) { - return nil, fmt.Errorf("%d too large; uint64 with the highest bit set are not supported", version) - } - - return &Batch{ - db: db, - lock: writeLock, - ops: make([]batchOp, 0), - version: int64(version), - }, nil -} - -func (b *Batch) Size() int { - return b.size -} - -func (b *Batch) Reset() error { - b.ops = nil - b.ops = make([]batchOp, 0) - b.size = 0 - - err := b.db.Begin() - if err != nil { - return err - } - - return nil -} - -func (b *Batch) Set(storeKey, key, value []byte) error { - b.size += len(key) + len(value) - b.ops = append(b.ops, batchOp{action: batchActionSet, storeKey: storeKey, key: key, value: value}) - return nil -} - -func (b *Batch) Delete(storeKey, key []byte) error { - b.size += len(key) - b.ops = append(b.ops, batchOp{action: batchActionDel, storeKey: storeKey, key: key}) - return nil -} - -func (b *Batch) Write() (err error) { - b.lock.Lock() - defer b.lock.Unlock() - err = b.db.Begin() - if err != nil { - return fmt.Errorf("failed to start SQL transaction: %w", err) - } - defer func() { - if err != nil { - err = errors.Join(err, b.db.Rollback()) - } - }() - if err := b.db.Exec(reservedUpsertStmt, reservedStoreKey, keyLatestHeight, b.version, 0, b.version); err != nil { - return fmt.Errorf("failed to exec SQL statement: %w", err) - } - - for _, op := range b.ops { - switch op.action { - case batchActionSet: - if err := b.db.Exec(upsertStmt, op.storeKey, op.key, op.value, b.version, op.value); err != nil { - return fmt.Errorf("failed to exec SQL statement: %w", err) - } - - case batchActionDel: - if err := b.db.Exec(delStmt, b.version, op.storeKey, op.key, b.version); err != nil { - return fmt.Errorf("failed to exec SQL statement: %w", err) - } - } - } - - if err := b.db.Commit(); err != nil { - return fmt.Errorf("failed to write SQL transaction: %w", err) - } - - return nil -} diff --git a/store/v2/storage/sqlite/db.go b/store/v2/storage/sqlite/db.go deleted file mode 100644 index 6cd826757b01..000000000000 --- a/store/v2/storage/sqlite/db.go +++ /dev/null @@ -1,416 +0,0 @@ -package sqlite - -import ( - "bytes" - "errors" - "fmt" - "path/filepath" - "strings" - "sync" - - "github.com/bvinc/go-sqlite-lite/sqlite3" - - corestore "cosmossdk.io/core/store" - "cosmossdk.io/store/v2" - storeerrors "cosmossdk.io/store/v2/errors" - "cosmossdk.io/store/v2/storage" -) - -const ( - driverName = "sqlite3" - dbName = "ss.db" - reservedStoreKey = "_RESERVED_" - keyLatestHeight = "latest_height" - keyPruneHeight = "prune_height" - valueRemovedStore = "removed_store" - - reservedUpsertStmt = ` - INSERT INTO state_storage(store_key, key, value, version) - VALUES(?, ?, ?, ?) - ON CONFLICT(store_key, key, version) DO UPDATE SET - value = ?; - ` - upsertStmt = ` - INSERT INTO state_storage(store_key, key, value, version) - VALUES(?, ?, ?, ?) - ON CONFLICT(store_key, key, version) DO UPDATE SET - value = ?; - ` - delStmt = ` - UPDATE state_storage SET tombstone = ? - WHERE id = ( - SELECT id FROM state_storage WHERE store_key = ? AND key = ? AND version <= ? ORDER BY version DESC LIMIT 1 - ) AND tombstone = 0; - ` -) - -var ( - _ storage.Database = (*Database)(nil) - _ store.UpgradableDatabase = (*Database)(nil) -) - -type Database struct { - storage *sqlite3.Conn - connStr string - writeLock sync.Mutex - // earliestVersion defines the earliest version set in the database, which is - // only updated when the database is pruned. - earliestVersion uint64 -} - -func New(dataDir string) (*Database, error) { - connStr := fmt.Sprintf("file:%s", filepath.Join(dataDir, dbName)) - db, err := sqlite3.Open(connStr) - if err != nil { - return nil, fmt.Errorf("failed to open sqlite DB: %w", err) - } - err = db.Exec("PRAGMA journal_mode=WAL;") - if err != nil { - return nil, fmt.Errorf("failed to set journal mode: %w", err) - } - - stmt := ` - CREATE TABLE IF NOT EXISTS state_storage ( - id integer not null primary key, - store_key varchar not null, - key varchar not null, - value varchar not null, - version integer unsigned not null, - tombstone integer unsigned default 0, - unique (store_key, key, version) - ); - - CREATE UNIQUE INDEX IF NOT EXISTS idx_store_key_version ON state_storage (store_key, key, version); - ` - err = db.Exec(stmt) - if err != nil { - return nil, fmt.Errorf("failed to exec SQL statement: %w", err) - } - - pruneHeight, err := getPruneHeight(db) - if err != nil { - return nil, fmt.Errorf("failed to get prune height: %w", err) - } - - return &Database{ - storage: db, - connStr: connStr, - writeLock: sync.Mutex{}, - earliestVersion: pruneHeight, - }, nil -} - -func (db *Database) Close() error { - err := db.storage.Close() - db.storage = nil - return err -} - -func (db *Database) NewBatch(version uint64) (store.Batch, error) { - return NewBatch(db.storage, &db.writeLock, version) -} - -func (db *Database) GetLatestVersion() (version uint64, err error) { - stmt, err := db.storage.Prepare(` - SELECT value - FROM state_storage - WHERE store_key = ? AND key = ? - `) - if err != nil { - return 0, fmt.Errorf("failed to prepare SQL statement: %w", err) - } - - defer func(stmt *sqlite3.Stmt) { - cErr := stmt.Close() - if cErr != nil { - err = errors.Join(err, fmt.Errorf("failed to close GetLatestVersion statement: %w", cErr)) - } - }(stmt) - - err = stmt.Bind(reservedStoreKey, keyLatestHeight) - if err != nil { - return 0, fmt.Errorf("failed to bind GetLatestVersion statement: %w", err) - } - hasRow, err := stmt.Step() - if err != nil { - return 0, fmt.Errorf("failed to step through GetLatestVersion rows: %w", err) - } - if !hasRow { - // in case of a fresh database - return 0, nil - } - var v int64 - err = stmt.Scan(&v) - if err != nil { - return 0, fmt.Errorf("failed to scan GetLatestVersion row: %w", err) - } - version = uint64(v) - return version, nil -} - -func (db *Database) VersionExists(v uint64) (bool, error) { - latestVersion, err := db.GetLatestVersion() - if err != nil { - return false, err - } - - return latestVersion >= v && v >= db.earliestVersion, nil -} - -func (db *Database) SetLatestVersion(version uint64) error { - db.writeLock.Lock() - defer db.writeLock.Unlock() - err := db.storage.Exec(reservedUpsertStmt, reservedStoreKey, keyLatestHeight, int64(version), 0, int64(version)) - if err != nil { - return fmt.Errorf("failed to exec SQL statement: %w", err) - } - - return nil -} - -func (db *Database) Has(storeKey []byte, version uint64, key []byte) (bool, error) { - val, err := db.Get(storeKey, version, key) - if err != nil { - return false, err - } - - return val != nil, nil -} - -func (db *Database) Get(storeKey []byte, targetVersion uint64, key []byte) ([]byte, error) { - if targetVersion < db.earliestVersion { - return nil, storeerrors.ErrVersionPruned{EarliestVersion: db.earliestVersion, RequestedVersion: targetVersion} - } - - stmt, err := db.storage.Prepare(` - SELECT value, tombstone FROM state_storage - WHERE store_key = ? AND key = ? AND version <= ? - ORDER BY version DESC LIMIT 1; - `) - if err != nil { - return nil, fmt.Errorf("failed to prepare SQL statement: %w", err) - } - - defer stmt.Close() - - var ( - value []byte - tomb int64 - ) - err = stmt.Bind(storeKey, key, int64(targetVersion)) - if err != nil { - return nil, fmt.Errorf("failed to bind SQL statement: %w", err) - } - hasRow, err := stmt.Step() - if err != nil { - return nil, fmt.Errorf("failed to step through SQL rows: %w", err) - } - if !hasRow { - return nil, nil - } - err = stmt.Scan(&value, &tomb) - if err != nil { - return nil, fmt.Errorf("failed to scan row: %w", err) - } - - // A tombstone of zero or a target version that is less than the tombstone - // version means the key is not deleted at the target version. - if tomb == 0 || targetVersion < uint64(tomb) { - return value, nil - } - - // the value is considered deleted - return nil, nil -} - -// Prune removes all versions of all keys that are <= the given version. It keeps -// the latest (non-tombstoned) version of each key/value tuple to handle queries -// above the prune version. This is analogous to RocksDB full_history_ts_low. -// -// We perform the prune by deleting all versions of a key, excluding reserved keys, -// that are <= the given version, except for the latest version of the key. -func (db *Database) Prune(version uint64) (err error) { - v := int64(version) - db.writeLock.Lock() - defer db.writeLock.Unlock() - err = db.storage.Begin() - if err != nil { - return fmt.Errorf("failed to create SQL transaction: %w", err) - } - defer func() { - if err != nil { - err = errors.Join(err, db.storage.Rollback()) - } - }() - - // prune all keys of old versions - pruneStmt := `DELETE FROM state_storage - WHERE version < ( - SELECT max(version) FROM state_storage t2 WHERE - t2.store_key = state_storage.store_key AND - t2.key = state_storage.key AND - t2.version <= ? - ) AND store_key != ?; - ` - if err := db.storage.Exec(pruneStmt, v, reservedStoreKey); err != nil { - return fmt.Errorf("failed to exec prune keys statement: %w", err) - } - - // prune removed stores - pruneRemovedStoreKeysStmt := `DELETE FROM state_storage AS s - WHERE EXISTS ( - SELECT 1 FROM - ( - SELECT key, MAX(version) AS max_version - FROM state_storage - WHERE store_key = ? AND value = ? AND version <= ? - GROUP BY key - ) AS t - WHERE s.store_key = t.key AND s.version <= t.max_version LIMIT 1 - ); - ` - if err := db.storage.Exec(pruneRemovedStoreKeysStmt, reservedStoreKey, valueRemovedStore, v); err != nil { - return fmt.Errorf("failed to exec prune store keys statement: %w", err) - } - - // delete the removedKeys - if err := db.storage.Exec("DELETE FROM state_storage WHERE store_key = ? AND value = ? AND version <= ?", - reservedStoreKey, valueRemovedStore, v); err != nil { - return fmt.Errorf("failed to exec remove keys statement: %w", err) - } - - // set the prune height so we can return for queries below this height - if err := db.storage.Exec(reservedUpsertStmt, reservedStoreKey, keyPruneHeight, v, 0, v); err != nil { - return fmt.Errorf("failed to exec set prune height statement: %w", err) - } - - if err := db.storage.Commit(); err != nil { - return fmt.Errorf("failed to commit prune transaction: %w", err) - } - - db.earliestVersion = version + 1 - return nil -} - -func (db *Database) Iterator(storeKey []byte, version uint64, start, end []byte) (corestore.Iterator, error) { - if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) { - return nil, storeerrors.ErrKeyEmpty - } - - if start != nil && end != nil && bytes.Compare(start, end) > 0 { - return nil, storeerrors.ErrStartAfterEnd - } - - return newIterator(db, storeKey, version, start, end, false) -} - -func (db *Database) ReverseIterator(storeKey []byte, version uint64, start, end []byte) (corestore.Iterator, error) { - if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) { - return nil, storeerrors.ErrKeyEmpty - } - - if start != nil && end != nil && bytes.Compare(start, end) > 0 { - return nil, storeerrors.ErrStartAfterEnd - } - - return newIterator(db, storeKey, version, start, end, true) -} - -func (db *Database) PruneStoreKeys(storeKeys []string, version uint64) (err error) { - db.writeLock.Lock() - defer db.writeLock.Unlock() - err = db.storage.Begin() - if err != nil { - return fmt.Errorf("failed to create SQL transaction: %w", err) - } - defer func() { - if err != nil { - err = errors.Join(err, db.storage.Rollback()) - } - }() - - // flush removed store keys - flushRemovedStoreKeyStmt := `INSERT INTO state_storage(store_key, key, value, version) - VALUES (?, ?, ?, ?)` - for _, storeKey := range storeKeys { - if err := db.storage.Exec(flushRemovedStoreKeyStmt, reservedStoreKey, []byte(storeKey), valueRemovedStore, version); err != nil { - return fmt.Errorf("failed to exec SQL statement: %w", err) - } - } - - return db.storage.Commit() -} - -func (db *Database) PrintRowsDebug() { - stmt, err := db.storage.Prepare("SELECT store_key, key, value, version, tombstone FROM state_storage") - if err != nil { - panic(fmt.Errorf("failed to prepare SQL statement: %w", err)) - } - - defer stmt.Close() - - err = stmt.Exec() - if err != nil { - panic(fmt.Errorf("failed to execute SQL query: %w", err)) - } - - var sb strings.Builder - for { - hasRow, err := stmt.Step() - if err != nil { - panic(fmt.Errorf("failed to step through SQL rows: %w", err)) - } - if !hasRow { - break - } - var ( - storeKey []byte - key []byte - value []byte - version int64 - tomb int64 - ) - if err := stmt.Scan(&storeKey, &key, &value, &version, &tomb); err != nil { - panic(fmt.Sprintf("failed to scan row: %s", err)) - } - - sb.WriteString(fmt.Sprintf("STORE_KEY: %s, KEY: %s, VALUE: %s, VERSION: %d, TOMBSTONE: %d\n", storeKey, key, value, version, tomb)) - } - - fmt.Println(strings.TrimSpace(sb.String())) -} - -func getPruneHeight(storage *sqlite3.Conn) (height uint64, err error) { - stmt, err := storage.Prepare(`SELECT value FROM state_storage WHERE store_key = ? AND key = ?`) - if err != nil { - return 0, fmt.Errorf("failed to prepare SQL statement: %w", err) - } - - defer func(stmt *sqlite3.Stmt) { - cErr := stmt.Close() - if cErr != nil { - err = errors.Join(err, fmt.Errorf("failed to close SQL statement: %w", cErr)) - } - }(stmt) - - if err = stmt.Bind(reservedStoreKey, keyPruneHeight); err != nil { - return 0, fmt.Errorf("failed to bind prune height SQL statement: %w", err) - } - hasRows, err := stmt.Step() - if err != nil { - return 0, fmt.Errorf("failed to step prune height SQL statement: %w", err) - } - if !hasRows { - return 0, nil - } - var h int64 - if err = stmt.Scan(&h); err != nil { - return 0, fmt.Errorf("failed to scan prune height SQL statement: %w", err) - } - height = uint64(h) - return height, nil -} - -func isHighBitSet(version uint64) bool { - return version&(1<<63) != 0 -} diff --git a/store/v2/storage/sqlite/db_test.go b/store/v2/storage/sqlite/db_test.go deleted file mode 100644 index 7ec950bf93a0..000000000000 --- a/store/v2/storage/sqlite/db_test.go +++ /dev/null @@ -1,210 +0,0 @@ -package sqlite - -import ( - "fmt" - "math" - "sync" - "testing" - - "github.com/stretchr/testify/require" - "github.com/stretchr/testify/suite" - - coretesting "cosmossdk.io/core/testing" - "cosmossdk.io/store/v2/storage" -) - -var storeKey1 = []byte("store1") - -func TestStorageTestSuite(t *testing.T) { - s := &storage.StorageTestSuite{ - NewDB: func(dir string) (*storage.StorageStore, error) { - db, err := New(dir) - return storage.NewStorageStore(db, coretesting.NewNopLogger()), err - }, - EmptyBatchSize: 0, - } - suite.Run(t, s) -} - -func TestDatabase_ReverseIterator(t *testing.T) { - db, err := New(t.TempDir()) - require.NoError(t, err) - defer db.Close() - - batch, err := db.NewBatch(1) - require.NoError(t, err) - for i := 0; i < 100; i++ { - key := fmt.Sprintf("key%03d", i) // key000, key001, ..., key099 - val := fmt.Sprintf("val%03d", i) // val000, val001, ..., val099 - - require.NoError(t, batch.Set(storeKey1, []byte(key), []byte(val))) - } - - require.NoError(t, batch.Write()) - - // reverse iterator without an end key - iter, err := db.ReverseIterator(storeKey1, 1, []byte("key000"), nil) - require.NoError(t, err) - - defer iter.Close() - - i, count := 99, 0 - for ; iter.Valid(); iter.Next() { - require.Equal(t, []byte(fmt.Sprintf("key%03d", i)), iter.Key()) - require.Equal(t, []byte(fmt.Sprintf("val%03d", i)), iter.Value()) - - i-- - count++ - } - require.Equal(t, 100, count) - require.NoError(t, iter.Error()) - - // seek past domain, which should make the iterator invalid and produce an error - require.False(t, iter.Valid()) - - // reverse iterator with a start and end domain - iter2, err := db.ReverseIterator(storeKey1, 1, []byte("key010"), []byte("key019")) - require.NoError(t, err) - - defer iter2.Close() - - i, count = 18, 0 - for ; iter2.Valid(); iter2.Next() { - require.Equal(t, []byte(fmt.Sprintf("key%03d", i)), iter2.Key()) - require.Equal(t, []byte(fmt.Sprintf("val%03d", i)), iter2.Value()) - - i-- - count++ - } - require.Equal(t, 9, count) - require.NoError(t, iter2.Error()) - - // seek past domain, which should make the iterator invalid and produce an error - require.False(t, iter2.Valid()) - - // start must be <= end - iter3, err := db.ReverseIterator(storeKey1, 1, []byte("key020"), []byte("key019")) - require.Error(t, err) - require.Nil(t, iter3) -} - -func TestParallelWrites(t *testing.T) { - tmpDir := t.TempDir() - db, err := New(tmpDir) - require.NoError(t, err) - defer db.Close() - - latestVersion := 10 - kvCount := 100 - - wg := sync.WaitGroup{} - triggerStartCh := make(chan bool) - - // start 10 goroutines that write to the database - for i := 0; i < latestVersion; i++ { - wg.Add(1) - go func(i int) { - <-triggerStartCh - defer wg.Done() - batch, err := db.NewBatch(uint64(i + 1)) - require.NoError(t, err) - for j := 0; j < kvCount; j++ { - key := fmt.Sprintf("key-%d-%03d", i, j) - val := fmt.Sprintf("val-%d-%03d", i, j) - - require.NoError(t, batch.Set(storeKey1, []byte(key), []byte(val))) - } - - require.NoError(t, batch.Write()) - }(i) - - } - - // start the goroutines - close(triggerStartCh) - wg.Wait() - - // check that all the data is there - for i := 0; i < latestVersion; i++ { - for j := 0; j < kvCount; j++ { - version := uint64(i + 1) - key := fmt.Sprintf("key-%d-%03d", i, j) - val := fmt.Sprintf("val-%d-%03d", i, j) - - v, err := db.Get(storeKey1, version, []byte(key)) - require.NoError(t, err) - require.Equal(t, []byte(val), v) - } - } -} - -func TestParallelWriteAndPruning(t *testing.T) { - db, err := New(t.TempDir()) - require.NoError(t, err) - defer db.Close() - - latestVersion := 100 - kvCount := 100 - prunePeriod := 5 - - wg := sync.WaitGroup{} - triggerStartCh := make(chan bool) - - // start a goroutine that write to the database - wg.Add(1) - go func() { - <-triggerStartCh - defer wg.Done() - for i := 0; i < latestVersion; i++ { - batch, err := db.NewBatch(uint64(i + 1)) - require.NoError(t, err) - for j := 0; j < kvCount; j++ { - key := fmt.Sprintf("key-%d-%03d", i, j) - val := fmt.Sprintf("val-%d-%03d", i, j) - - require.NoError(t, batch.Set(storeKey1, []byte(key), []byte(val))) - } - - require.NoError(t, batch.Write()) - } - }() - // start a goroutine that prunes the database - wg.Add(1) - go func() { - <-triggerStartCh - defer wg.Done() - for i := 10; i < latestVersion; i += prunePeriod { - for { - v, err := db.GetLatestVersion() - require.NoError(t, err) - if v > uint64(i) { - require.NoError(t, db.Prune(v-1)) - break - } - } - } - }() - - // start the goroutines - close(triggerStartCh) - wg.Wait() - - // check if the data is pruned - version := uint64(latestVersion - prunePeriod) - val, err := db.Get(storeKey1, version, []byte(fmt.Sprintf("key-%d-%03d", version-1, 0))) - require.Error(t, err) - require.Nil(t, val) - - version = uint64(latestVersion) - val, err = db.Get(storeKey1, version, []byte(fmt.Sprintf("key-%d-%03d", version-1, 0))) - require.NoError(t, err) - require.Equal(t, []byte(fmt.Sprintf("val-%d-%03d", version-1, 0)), val) -} - -func TestUint64(t *testing.T) { - tmpDir := t.TempDir() - db, err := New(tmpDir) - require.NoError(t, err) - _, err = db.NewBatch(math.MaxUint64) - require.Error(t, err) -} diff --git a/store/v2/storage/sqlite/iterator.go b/store/v2/storage/sqlite/iterator.go deleted file mode 100644 index dbe3339c0c25..000000000000 --- a/store/v2/storage/sqlite/iterator.go +++ /dev/null @@ -1,186 +0,0 @@ -package sqlite - -import ( - "bytes" - "database/sql" - "fmt" - "slices" - "strings" - - "github.com/bvinc/go-sqlite-lite/sqlite3" - - corestore "cosmossdk.io/core/store" -) - -var _ corestore.Iterator = (*iterator)(nil) - -type iterator struct { - statement *sqlite3.Stmt - key, val []byte - start, end []byte - valid bool - err error -} - -func newIterator(db *Database, storeKey []byte, version uint64, start, end []byte, reverse bool) (*iterator, error) { - if isHighBitSet(version) { - return nil, fmt.Errorf("%d too large; uint64 with the highest bit set are not supported", version) - } - if version < db.earliestVersion { - return &iterator{ - start: start, - end: end, - valid: false, - }, nil - } - - var ( - targetVersion = int64(version) - keyClause = []string{"store_key = ?", "version <= ?"} - queryArgs []any - ) - - switch { - case len(start) > 0 && len(end) > 0: - keyClause = append(keyClause, "key >= ?", "key < ?") - queryArgs = []any{storeKey, targetVersion, start, end, targetVersion} - - case len(start) > 0 && len(end) == 0: - keyClause = append(keyClause, "key >= ?") - queryArgs = []any{storeKey, targetVersion, start, targetVersion} - - case len(start) == 0 && len(end) > 0: - keyClause = append(keyClause, "key < ?") - queryArgs = []any{storeKey, targetVersion, end, targetVersion} - - default: - queryArgs = []any{storeKey, targetVersion, targetVersion} - } - - orderBy := "ASC" - if reverse { - orderBy = "DESC" - } - - // Note, this is not susceptible to SQL injection because placeholders are used - // for parts of the query outside the store's direct control. - stmt, err := db.storage.Prepare(fmt.Sprintf(` - SELECT x.key, x.value - FROM ( - SELECT key, value, version, tombstone, - row_number() OVER (PARTITION BY key ORDER BY version DESC) AS _rn - FROM state_storage WHERE %s - ) x - WHERE x._rn = 1 AND (x.tombstone = 0 OR x.tombstone > ?) ORDER BY x.key %s; - `, strings.Join(keyClause, " AND "), orderBy)) - if err != nil { - return nil, fmt.Errorf("failed to prepare SQL statement: %w", err) - } - - err = stmt.Bind(queryArgs...) - if err != nil { - _ = stmt.Close() - return nil, fmt.Errorf("failed to bind SQL iterator query arguments: %w", err) - } - - itr := &iterator{ - statement: stmt, - start: start, - end: end, - } - itr.valid, err = itr.statement.Step() - if err != nil { - itr.err = fmt.Errorf("failed to step SQL iterator: %w", err) - return itr, nil - } - if !itr.valid { - itr.err = fmt.Errorf("iterator invalid: %w", sql.ErrNoRows) - return itr, nil - } - - // read the first row - itr.parseRow() - if !itr.valid { - return itr, nil - } - - return itr, nil -} - -func (itr *iterator) Close() (err error) { - if itr.statement != nil { - err = itr.statement.Close() - } - - itr.valid = false - itr.statement = nil - - return err -} - -// Domain returns the domain of the iterator. The caller must not modify the -// return values. -func (itr *iterator) Domain() ([]byte, []byte) { - return itr.start, itr.end -} - -func (itr *iterator) Key() []byte { - itr.assertIsValid() - return slices.Clone(itr.key) -} - -func (itr *iterator) Value() []byte { - itr.assertIsValid() - return slices.Clone(itr.val) -} - -func (itr *iterator) Valid() bool { - if !itr.valid { - return itr.valid - } - - // if key is at the end or past it, consider it invalid - if end := itr.end; end != nil { - if bytes.Compare(end, itr.Key()) <= 0 { - itr.valid = false - return itr.valid - } - } - - return true -} - -func (itr *iterator) Next() { - var hasRow bool - hasRow, itr.err = itr.statement.Step() - if itr.err != nil || !hasRow { - itr.valid = false - return - } - itr.parseRow() -} - -func (itr *iterator) Error() error { - return itr.err -} - -func (itr *iterator) parseRow() { - var ( - key []byte - value []byte - ) - if err := itr.statement.Scan(&key, &value); err != nil { - itr.err = fmt.Errorf("failed to scan row: %w", err) - itr.valid = false - return - } - - itr.key = key - itr.val = value -} - -func (itr *iterator) assertIsValid() { - if !itr.valid { - panic("iterator is invalid") - } -} diff --git a/store/v2/storage/store.go b/store/v2/storage/store.go index 5ca5a30132b7..d53e323774ea 100644 --- a/store/v2/storage/store.go +++ b/store/v2/storage/store.go @@ -128,7 +128,8 @@ func (ss *StorageStore) Restore(version uint64, chStorage <-chan *corestore.Stat if err := b.Write(); err != nil { return err } - if err := b.Reset(); err != nil { + b, err = ss.db.NewBatch(version) + if err != nil { return err } } From d97ad63bd8236fefb63630c445a0cd6b0997a8b9 Mon Sep 17 00:00:00 2001 From: Matt Kocubinski Date: Fri, 29 Nov 2024 11:18:49 -0600 Subject: [PATCH 17/20] moar rm sqlite --- runtime/v2/go.mod | 1 - runtime/v2/go.sum | 2 -- server/v2/cometbft/go.mod | 1 - server/v2/cometbft/go.sum | 2 -- server/v2/cometbft/internal/mock/mock_store.go | 4 ++-- server/v2/go.mod | 1 - server/v2/go.sum | 2 -- simapp/v2/go.mod | 1 - simapp/v2/go.sum | 2 -- store/v2/go.mod | 2 +- store/v2/storage/storage_bench_test.go | 5 ----- tests/go.mod | 1 - tests/go.sum | 2 -- 13 files changed, 3 insertions(+), 23 deletions(-) diff --git a/runtime/v2/go.mod b/runtime/v2/go.mod index 8166b338be46..c02c29466ff9 100644 --- a/runtime/v2/go.mod +++ b/runtime/v2/go.mod @@ -35,7 +35,6 @@ require ( cosmossdk.io/errors/v2 v2.0.0-20240731132947-df72853b3ca5 // indirect github.com/DataDog/zstd v1.5.5 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/bvinc/go-sqlite-lite v0.6.1 // indirect github.com/bytedance/sonic v1.12.4 // indirect github.com/bytedance/sonic/loader v0.2.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect diff --git a/runtime/v2/go.sum b/runtime/v2/go.sum index 690a675f2e22..723b257f47e0 100644 --- a/runtime/v2/go.sum +++ b/runtime/v2/go.sum @@ -23,8 +23,6 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bvinc/go-sqlite-lite v0.6.1 h1:JU8Rz5YAOZQiU3WEulKF084wfXpytRiqD2IaW2QjPz4= -github.com/bvinc/go-sqlite-lite v0.6.1/go.mod h1:2GiE60NUdb0aNhDdY+LXgrqAVDpi2Ijc6dB6ZMp9x6s= github.com/bytedance/sonic v1.12.4 h1:9Csb3c9ZJhfUWeMtpCDCq6BUoH5ogfDFLUgQ/jG+R0k= github.com/bytedance/sonic v1.12.4/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= diff --git a/server/v2/cometbft/go.mod b/server/v2/cometbft/go.mod index e0a5cd741e1e..4433e91d8759 100644 --- a/server/v2/cometbft/go.mod +++ b/server/v2/cometbft/go.mod @@ -60,7 +60,6 @@ require ( github.com/Microsoft/go-winio v0.6.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bgentry/speakeasy v0.2.0 // indirect - github.com/bvinc/go-sqlite-lite v0.6.1 // indirect github.com/bytedance/sonic v1.12.4 // indirect github.com/bytedance/sonic/loader v0.2.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect diff --git a/server/v2/cometbft/go.sum b/server/v2/cometbft/go.sum index 204943777173..4480ca40f3af 100644 --- a/server/v2/cometbft/go.sum +++ b/server/v2/cometbft/go.sum @@ -58,8 +58,6 @@ github.com/btcsuite/btcd/btcutil v1.1.6 h1:zFL2+c3Lb9gEgqKNzowKUPQNb8jV7v5Oaodi/ github.com/btcsuite/btcd/btcutil v1.1.6/go.mod h1:9dFymx8HpuLqBnsPELrImQeTQfKBQqzqGbbV3jK55aE= github.com/bufbuild/protocompile v0.4.0 h1:LbFKd2XowZvQ/kajzguUp2DC9UEIQhIq77fZZlaQsNA= github.com/bufbuild/protocompile v0.4.0/go.mod h1:3v93+mbWn/v3xzN+31nwkJfrEpAUwp+BagBSZWx+TP8= -github.com/bvinc/go-sqlite-lite v0.6.1 h1:JU8Rz5YAOZQiU3WEulKF084wfXpytRiqD2IaW2QjPz4= -github.com/bvinc/go-sqlite-lite v0.6.1/go.mod h1:2GiE60NUdb0aNhDdY+LXgrqAVDpi2Ijc6dB6ZMp9x6s= github.com/bytedance/sonic v1.12.4 h1:9Csb3c9ZJhfUWeMtpCDCq6BUoH5ogfDFLUgQ/jG+R0k= github.com/bytedance/sonic v1.12.4/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= diff --git a/server/v2/cometbft/internal/mock/mock_store.go b/server/v2/cometbft/internal/mock/mock_store.go index e9f7be3edcb3..b485a75d876b 100644 --- a/server/v2/cometbft/internal/mock/mock_store.go +++ b/server/v2/cometbft/internal/mock/mock_store.go @@ -12,7 +12,7 @@ import ( dbm "cosmossdk.io/store/v2/db" "cosmossdk.io/store/v2/proof" "cosmossdk.io/store/v2/storage" - "cosmossdk.io/store/v2/storage/sqlite" + "cosmossdk.io/store/v2/storage/pebbledb" ) type MockStore struct { @@ -21,7 +21,7 @@ type MockStore struct { } func NewMockStorage(logger log.Logger, dir string) storev2.VersionedWriter { - storageDB, _ := sqlite.New(dir) + storageDB, _ := pebbledb.New(dir) ss := storage.NewStorageStore(storageDB, logger) return ss } diff --git a/server/v2/go.mod b/server/v2/go.mod index 5d2e67326fc3..0f84cf2ef70e 100644 --- a/server/v2/go.mod +++ b/server/v2/go.mod @@ -47,7 +47,6 @@ require ( github.com/DataDog/zstd v1.5.5 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/bvinc/go-sqlite-lite v0.6.1 // indirect github.com/bytedance/sonic v1.12.4 // indirect github.com/bytedance/sonic/loader v0.2.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect diff --git a/server/v2/go.sum b/server/v2/go.sum index 94c8173f7bd4..dcd56eab1e9b 100644 --- a/server/v2/go.sum +++ b/server/v2/go.sum @@ -29,8 +29,6 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bufbuild/protocompile v0.14.1 h1:iA73zAf/fyljNjQKwYzUHD6AD4R8KMasmwa/FBatYVw= github.com/bufbuild/protocompile v0.14.1/go.mod h1:ppVdAIhbr2H8asPk6k4pY7t9zB1OU5DoEw9xY/FUi1c= -github.com/bvinc/go-sqlite-lite v0.6.1 h1:JU8Rz5YAOZQiU3WEulKF084wfXpytRiqD2IaW2QjPz4= -github.com/bvinc/go-sqlite-lite v0.6.1/go.mod h1:2GiE60NUdb0aNhDdY+LXgrqAVDpi2Ijc6dB6ZMp9x6s= github.com/bytedance/sonic v1.12.4 h1:9Csb3c9ZJhfUWeMtpCDCq6BUoH5ogfDFLUgQ/jG+R0k= github.com/bytedance/sonic v1.12.4/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= diff --git a/simapp/v2/go.mod b/simapp/v2/go.mod index dffbac74044b..04e50d0a6397 100644 --- a/simapp/v2/go.mod +++ b/simapp/v2/go.mod @@ -78,7 +78,6 @@ require ( github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect github.com/bgentry/speakeasy v0.2.0 // indirect github.com/bits-and-blooms/bitset v1.10.0 // indirect - github.com/bvinc/go-sqlite-lite v0.6.1 // indirect github.com/bytedance/sonic v1.12.4 // indirect github.com/bytedance/sonic/loader v0.2.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect diff --git a/simapp/v2/go.sum b/simapp/v2/go.sum index 1263e44742d7..7c0da2ed72d9 100644 --- a/simapp/v2/go.sum +++ b/simapp/v2/go.sum @@ -256,8 +256,6 @@ github.com/btcsuite/btcd/btcutil v1.1.6 h1:zFL2+c3Lb9gEgqKNzowKUPQNb8jV7v5Oaodi/ github.com/btcsuite/btcd/btcutil v1.1.6/go.mod h1:9dFymx8HpuLqBnsPELrImQeTQfKBQqzqGbbV3jK55aE= github.com/bufbuild/protocompile v0.4.0 h1:LbFKd2XowZvQ/kajzguUp2DC9UEIQhIq77fZZlaQsNA= github.com/bufbuild/protocompile v0.4.0/go.mod h1:3v93+mbWn/v3xzN+31nwkJfrEpAUwp+BagBSZWx+TP8= -github.com/bvinc/go-sqlite-lite v0.6.1 h1:JU8Rz5YAOZQiU3WEulKF084wfXpytRiqD2IaW2QjPz4= -github.com/bvinc/go-sqlite-lite v0.6.1/go.mod h1:2GiE60NUdb0aNhDdY+LXgrqAVDpi2Ijc6dB6ZMp9x6s= github.com/bytedance/sonic v1.12.4 h1:9Csb3c9ZJhfUWeMtpCDCq6BUoH5ogfDFLUgQ/jG+R0k= github.com/bytedance/sonic v1.12.4/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= diff --git a/store/v2/go.mod b/store/v2/go.mod index a2351efc3b92..ac07130d2e10 100644 --- a/store/v2/go.mod +++ b/store/v2/go.mod @@ -7,7 +7,6 @@ require ( cosmossdk.io/core/testing v0.0.0-20241108153815-606544c7be7e cosmossdk.io/errors/v2 v2.0.0-20240731132947-df72853b3ca5 cosmossdk.io/log v1.5.0 - github.com/bvinc/go-sqlite-lite v0.6.1 github.com/cockroachdb/pebble v1.1.0 github.com/cosmos/cosmos-proto v1.0.0-beta.5 github.com/cosmos/gogoproto v1.7.0 @@ -29,6 +28,7 @@ require ( github.com/DataDog/zstd v1.5.5 // indirect github.com/aybabtme/uniplot v0.0.0-20151203143629-039c559e5e7e // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/bvinc/go-sqlite-lite v0.6.1 // indirect github.com/bytedance/sonic v1.12.4 // indirect github.com/bytedance/sonic/loader v0.2.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect diff --git a/store/v2/storage/storage_bench_test.go b/store/v2/storage/storage_bench_test.go index 2e2030bec6bb..36de5772ef6a 100644 --- a/store/v2/storage/storage_bench_test.go +++ b/store/v2/storage/storage_bench_test.go @@ -18,7 +18,6 @@ import ( "cosmossdk.io/store/v2/storage" "cosmossdk.io/store/v2/storage/pebbledb" "cosmossdk.io/store/v2/storage/rocksdb" - "cosmossdk.io/store/v2/storage/sqlite" ) var storeKey1 = []byte("store1") @@ -37,10 +36,6 @@ var ( return storage.NewStorageStore(db, coretesting.NewNopLogger()), err }, - "btree_sqlite": func(dataDir string) (store.VersionedWriter, error) { - db, err := sqlite.New(dataDir) - return storage.NewStorageStore(db, coretesting.NewNopLogger()), err - }, } rng = rand.New(rand.NewSource(567320)) ) diff --git a/tests/go.mod b/tests/go.mod index 1e417ff7635a..5fcc4afbacf9 100644 --- a/tests/go.mod +++ b/tests/go.mod @@ -88,7 +88,6 @@ require ( github.com/bgentry/speakeasy v0.2.0 // indirect github.com/bits-and-blooms/bitset v1.10.0 // indirect github.com/bufbuild/protocompile v0.14.1 // indirect - github.com/bvinc/go-sqlite-lite v0.6.1 // indirect github.com/bytedance/sonic v1.12.4 // indirect github.com/bytedance/sonic/loader v0.2.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect diff --git a/tests/go.sum b/tests/go.sum index 94f80dca69bd..a98e6f08c7a3 100644 --- a/tests/go.sum +++ b/tests/go.sum @@ -253,8 +253,6 @@ github.com/btcsuite/btcd/btcutil v1.1.6 h1:zFL2+c3Lb9gEgqKNzowKUPQNb8jV7v5Oaodi/ github.com/btcsuite/btcd/btcutil v1.1.6/go.mod h1:9dFymx8HpuLqBnsPELrImQeTQfKBQqzqGbbV3jK55aE= github.com/bufbuild/protocompile v0.14.1 h1:iA73zAf/fyljNjQKwYzUHD6AD4R8KMasmwa/FBatYVw= github.com/bufbuild/protocompile v0.14.1/go.mod h1:ppVdAIhbr2H8asPk6k4pY7t9zB1OU5DoEw9xY/FUi1c= -github.com/bvinc/go-sqlite-lite v0.6.1 h1:JU8Rz5YAOZQiU3WEulKF084wfXpytRiqD2IaW2QjPz4= -github.com/bvinc/go-sqlite-lite v0.6.1/go.mod h1:2GiE60NUdb0aNhDdY+LXgrqAVDpi2Ijc6dB6ZMp9x6s= github.com/bytedance/sonic v1.12.4 h1:9Csb3c9ZJhfUWeMtpCDCq6BUoH5ogfDFLUgQ/jG+R0k= github.com/bytedance/sonic v1.12.4/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= From e8e2f1bdb466a8d10d82bf27622b3ce2efbaca5f Mon Sep 17 00:00:00 2001 From: Matt Kocubinski Date: Fri, 29 Nov 2024 11:25:17 -0600 Subject: [PATCH 18/20] fix config --- server/v2/testdata/app.toml | 4 ++-- tools/confix/data/v2-app.toml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/server/v2/testdata/app.toml b/server/v2/testdata/app.toml index 3d32be7e57ef..3f0e640a8c3e 100644 --- a/server/v2/testdata/app.toml +++ b/server/v2/testdata/app.toml @@ -25,8 +25,8 @@ minimum-gas-prices = '0stake' app-db-backend = 'goleveldb' [store.options] -# State storage database type. Currently we support: "sqlite", "pebble" and "rocksdb" -ss-type = 'sqlite' +# State storage database type. Currently we support: "pebble" and "rocksdb" +ss-type = 'pebble' # State commitment database type. Currently we support: "iavl" and "iavl-v2" sc-type = 'iavl' diff --git a/tools/confix/data/v2-app.toml b/tools/confix/data/v2-app.toml index e478eed3609a..394cc41e7b09 100644 --- a/tools/confix/data/v2-app.toml +++ b/tools/confix/data/v2-app.toml @@ -56,8 +56,8 @@ minimum-gas-prices = '0stake' app-db-backend = 'goleveldb' [store.options] -# State storage database type. Currently we support: "sqlite", "pebble" and "rocksdb" -ss-type = 'sqlite' +# State storage database type. Currently we support: "pebble" and "rocksdb" +ss-type = 'pebble' # State commitment database type. Currently we support: "iavl" and "iavl-v2" sc-type = 'iavl' From a912d0fd4ab9dac99b81fe2c7b06599437af5cfd Mon Sep 17 00:00:00 2001 From: Matt Kocubinski Date: Fri, 29 Nov 2024 12:40:37 -0600 Subject: [PATCH 19/20] make lint-fix --- server/v2/cometbft/abci_test.go | 26 +++++++++++-------- .../cometbft/oe/optimistic_execution_test.go | 5 ++-- server/v2/cometbft/server.go | 4 ++- store/v2/commitment/iavlv2/tree.go | 2 +- 4 files changed, 22 insertions(+), 15 deletions(-) diff --git a/server/v2/cometbft/abci_test.go b/server/v2/cometbft/abci_test.go index 7b20c354d786..b9bb0fa293f1 100644 --- a/server/v2/cometbft/abci_test.go +++ b/server/v2/cometbft/abci_test.go @@ -2,18 +2,20 @@ package cometbft import ( "context" - "cosmossdk.io/core/server" "crypto/sha256" "encoding/json" "errors" - abci "github.com/cometbft/cometbft/abci/types" "io" "strings" "sync" "testing" "time" - "cosmossdk.io/server/v2/cometbft/oe" + "cosmossdk.io/core/server" + + abci "github.com/cometbft/cometbft/abci/types" + + abci "github.com/cometbft/cometbft/abci/types" abciproto "github.com/cometbft/cometbft/api/cometbft/abci/v1" v1 "github.com/cometbft/cometbft/api/cometbft/types/v1" "github.com/cosmos/gogoproto/proto" @@ -21,6 +23,7 @@ import ( "github.com/stretchr/testify/require" appmodulev2 "cosmossdk.io/core/appmodule/v2" + "cosmossdk.io/core/server" "cosmossdk.io/core/store" "cosmossdk.io/core/transaction" "cosmossdk.io/log" @@ -28,6 +31,7 @@ import ( "cosmossdk.io/server/v2/cometbft/handlers" cometmock "cosmossdk.io/server/v2/cometbft/internal/mock" "cosmossdk.io/server/v2/cometbft/mempool" + "cosmossdk.io/server/v2/cometbft/oe" "cosmossdk.io/server/v2/cometbft/types" "cosmossdk.io/server/v2/stf" "cosmossdk.io/server/v2/stf/branch" @@ -60,10 +64,10 @@ func getQueryRouterBuilder[T any, PT interface { *T proto.Message }, -U any, UT interface { - *U - proto.Message -}]( + U any, UT interface { + *U + proto.Message + }]( t *testing.T, handler func(ctx context.Context, msg PT) (UT, error), ) *stf.MsgRouterBuilder { @@ -90,10 +94,10 @@ func getMsgRouterBuilder[T any, PT interface { *T transaction.Msg }, -U any, UT interface { - *U - transaction.Msg -}]( + U any, UT interface { + *U + transaction.Msg + }]( t *testing.T, handler func(ctx context.Context, msg PT) (UT, error), ) *stf.MsgRouterBuilder { diff --git a/server/v2/cometbft/oe/optimistic_execution_test.go b/server/v2/cometbft/oe/optimistic_execution_test.go index c0eb28c2a5e9..cd3c9263c784 100644 --- a/server/v2/cometbft/oe/optimistic_execution_test.go +++ b/server/v2/cometbft/oe/optimistic_execution_test.go @@ -5,12 +5,13 @@ import ( "errors" "testing" + abci "github.com/cometbft/cometbft/api/cometbft/abci/v1" + "github.com/stretchr/testify/assert" + "cosmossdk.io/core/server" "cosmossdk.io/core/store" "cosmossdk.io/core/transaction" "cosmossdk.io/log" - abci "github.com/cometbft/cometbft/api/cometbft/abci/v1" - "github.com/stretchr/testify/assert" ) func testFinalizeBlock[T transaction.Tx](context.Context, *abci.FinalizeBlockRequest) (*server.BlockResponse, store.WriterMap, []T, error) { diff --git a/server/v2/cometbft/server.go b/server/v2/cometbft/server.go index 6567e01caef0..fc7380795d28 100644 --- a/server/v2/cometbft/server.go +++ b/server/v2/cometbft/server.go @@ -2,7 +2,6 @@ package cometbft import ( "context" - "cosmossdk.io/server/v2/cometbft/oe" "crypto/sha256" "encoding/json" "fmt" @@ -11,6 +10,8 @@ import ( "sync" "sync/atomic" + "cosmossdk.io/server/v2/cometbft/oe" + abciserver "github.com/cometbft/cometbft/abci/server" abci "github.com/cometbft/cometbft/abci/types" cmtcmd "github.com/cometbft/cometbft/cmd/cometbft/commands" @@ -35,6 +36,7 @@ import ( "cosmossdk.io/server/v2/appmanager" cometlog "cosmossdk.io/server/v2/cometbft/log" "cosmossdk.io/server/v2/cometbft/mempool" + "cosmossdk.io/server/v2/cometbft/oe" "cosmossdk.io/server/v2/cometbft/types" "cosmossdk.io/store/v2/snapshots" diff --git a/store/v2/commitment/iavlv2/tree.go b/store/v2/commitment/iavlv2/tree.go index 003c5f4f45e0..997a0a60cc1a 100644 --- a/store/v2/commitment/iavlv2/tree.go +++ b/store/v2/commitment/iavlv2/tree.go @@ -4,10 +4,10 @@ import ( "errors" "fmt" - corestore "cosmossdk.io/core/store" "github.com/cosmos/iavl/v2" ics23 "github.com/cosmos/ics23/go" + corestore "cosmossdk.io/core/store" "cosmossdk.io/store/v2" "cosmossdk.io/store/v2/commitment" ) From f557fdd8a2f522258f995784177771d8d402d1c1 Mon Sep 17 00:00:00 2001 From: Matt Kocubinski Date: Fri, 29 Nov 2024 12:46:44 -0600 Subject: [PATCH 20/20] fix import statements --- server/v2/cometbft/abci_test.go | 4 ---- server/v2/cometbft/server.go | 2 -- 2 files changed, 6 deletions(-) diff --git a/server/v2/cometbft/abci_test.go b/server/v2/cometbft/abci_test.go index b9bb0fa293f1..ab1fdc722879 100644 --- a/server/v2/cometbft/abci_test.go +++ b/server/v2/cometbft/abci_test.go @@ -11,10 +11,6 @@ import ( "testing" "time" - "cosmossdk.io/core/server" - - abci "github.com/cometbft/cometbft/abci/types" - abci "github.com/cometbft/cometbft/abci/types" abciproto "github.com/cometbft/cometbft/api/cometbft/abci/v1" v1 "github.com/cometbft/cometbft/api/cometbft/types/v1" diff --git a/server/v2/cometbft/server.go b/server/v2/cometbft/server.go index fc7380795d28..55a38b5e9646 100644 --- a/server/v2/cometbft/server.go +++ b/server/v2/cometbft/server.go @@ -10,8 +10,6 @@ import ( "sync" "sync/atomic" - "cosmossdk.io/server/v2/cometbft/oe" - abciserver "github.com/cometbft/cometbft/abci/server" abci "github.com/cometbft/cometbft/abci/types" cmtcmd "github.com/cometbft/cometbft/cmd/cometbft/commands"