-
Notifications
You must be signed in to change notification settings - Fork 15
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
merge: branch '3632-database-improvements' into 'main'
Database improvements [#3632] Closes #3632 See merge request accumulatenetwork/accumulate!1096
- Loading branch information
Showing
13 changed files
with
346 additions
and
55 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,101 @@ | ||
| // Copyright 2024 The Accumulate Authors | ||
| // | ||
| // Use of this source code is governed by an MIT-style | ||
| // license that can be found in the LICENSE file or at | ||
| // https://opensource.org/licenses/MIT. | ||
|
|
||
| package overlay | ||
|
|
||
| import ( | ||
| "gitlab.com/accumulatenetwork/accumulate/pkg/database" | ||
| "gitlab.com/accumulatenetwork/accumulate/pkg/database/keyvalue" | ||
| "gitlab.com/accumulatenetwork/accumulate/pkg/database/keyvalue/memory" | ||
| "gitlab.com/accumulatenetwork/accumulate/pkg/errors" | ||
| "gitlab.com/accumulatenetwork/accumulate/pkg/types/record" | ||
| ) | ||
|
|
||
| func Open(a, b keyvalue.Beginner) keyvalue.Beginner { | ||
| return &Database{a, b} | ||
| } | ||
|
|
||
| type Database struct { | ||
| a, b keyvalue.Beginner | ||
| } | ||
|
|
||
| func (d *Database) Begin(prefix *database.Key, writable bool) keyvalue.ChangeSet { | ||
| a := d.a.Begin(prefix, writable) | ||
| b := d.b.Begin(prefix, false) | ||
|
|
||
| return memory.NewChangeSet(memory.ChangeSetOptions{ | ||
| Get: func(key *record.Key) ([]byte, error) { return get(a, b, key) }, | ||
| ForEach: func(fn func(*record.Key, []byte) error) error { return forEach(a, b, fn) }, | ||
| Discard: func() { a.Discard(); b.Discard() }, | ||
| Commit: func(m map[[32]byte]memory.Entry) error { return commit(a, b, m) }, | ||
| }) | ||
| } | ||
|
|
||
| func get(a, b keyvalue.ChangeSet, key *record.Key) ([]byte, error) { | ||
| // Get from a | ||
| v, err := a.Get(key) | ||
| switch { | ||
| case err == nil: | ||
| return v, nil | ||
| case !errors.Is(err, errors.NotFound): | ||
| return nil, err | ||
| } | ||
|
|
||
| // Get from b | ||
| v, err = b.Get(key) | ||
| switch { | ||
| case err == nil: | ||
| return v, nil | ||
| case !errors.Is(err, errors.NotFound): | ||
| return nil, err | ||
| } | ||
|
|
||
| return nil, (*database.NotFoundError)(key) | ||
| } | ||
|
|
||
| func forEach(a, b keyvalue.ChangeSet, fn func(*record.Key, []byte) error) error { | ||
| seen := map[[32]byte]bool{} | ||
|
|
||
| err := a.ForEach(func(key *record.Key, value []byte) error { | ||
| seen[key.Hash()] = true | ||
| return fn(key, value) | ||
| }) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| return b.ForEach(func(key *record.Key, value []byte) error { | ||
| if seen[key.Hash()] { | ||
| return nil | ||
| } | ||
| return fn(key, value) | ||
| }) | ||
| } | ||
|
|
||
| func commit(a, b keyvalue.ChangeSet, m map[[32]byte]memory.Entry) error { | ||
| for _, entry := range m { | ||
| if !entry.Delete { | ||
| err := a.Put(entry.Key, entry.Value) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| continue | ||
| } | ||
|
|
||
| _, err := b.Get(entry.Key) | ||
| switch { | ||
| case err == nil: | ||
| return errors.NotAllowed.WithFormat("cannot delete %v: it is present in the underlying database", entry.Key) | ||
| case !errors.Is(err, errors.NotFound): | ||
| return err | ||
| } | ||
|
|
||
| err = a.Delete(entry.Key) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| } | ||
| return a.Commit() | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,43 @@ | ||
| // Copyright 2024 The Accumulate Authors | ||
| // | ||
| // Use of this source code is governed by an MIT-style | ||
| // license that can be found in the LICENSE file or at | ||
| // https://opensource.org/licenses/MIT. | ||
|
|
||
| package overlay | ||
|
|
||
| import ( | ||
| "testing" | ||
|
|
||
| "gitlab.com/accumulatenetwork/accumulate/pkg/database/keyvalue" | ||
| "gitlab.com/accumulatenetwork/accumulate/pkg/database/keyvalue/kvtest" | ||
| "gitlab.com/accumulatenetwork/accumulate/pkg/database/keyvalue/memory" | ||
| ) | ||
|
|
||
| func open(*testing.T) kvtest.Opener { | ||
| db := Open(memory.New(nil), memory.New(nil)) | ||
| return func() (keyvalue.Beginner, error) { | ||
| return db, nil | ||
| } | ||
| } | ||
|
|
||
| func TestDatabase(t *testing.T) { | ||
| kvtest.TestDatabase(t, open(t)) | ||
| } | ||
|
|
||
| func TestIsolation(t *testing.T) { | ||
| t.Skip("Not supported by the underlying databases") | ||
| kvtest.TestIsolation(t, open(t)) | ||
| } | ||
|
|
||
| func TestSubBatch(t *testing.T) { | ||
| kvtest.TestSubBatch(t, open(t)) | ||
| } | ||
|
|
||
| func TestPrefix(t *testing.T) { | ||
| kvtest.TestPrefix(t, open(t)) | ||
| } | ||
|
|
||
| func TestDelete(t *testing.T) { | ||
| kvtest.TestDelete(t, open(t)) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,77 @@ | ||
| // Copyright 2024 The Accumulate Authors | ||
| // | ||
| // Use of this source code is governed by an MIT-style | ||
| // license that can be found in the LICENSE file or at | ||
| // https://opensource.org/licenses/MIT. | ||
|
|
||
| package overlay_test | ||
|
|
||
| import ( | ||
| "testing" | ||
|
|
||
| "github.com/stretchr/testify/require" | ||
| "gitlab.com/accumulatenetwork/accumulate/pkg/build" | ||
| "gitlab.com/accumulatenetwork/accumulate/pkg/errors" | ||
| . "gitlab.com/accumulatenetwork/accumulate/protocol" | ||
| . "gitlab.com/accumulatenetwork/accumulate/test/harness" | ||
| "gitlab.com/accumulatenetwork/accumulate/test/simulator" | ||
| ) | ||
|
|
||
| func TestOverlay(t *testing.T) { | ||
| // Setup | ||
| alice := build. | ||
| Identity("alice").Create("book"). | ||
| Tokens("tokens").Create("ACME").Add(1e9).Identity(). | ||
| Book("book").Page(1).Create().AddCredits(1e9).Book().Identity() | ||
| aliceKey := alice.Book("book").Page(1). | ||
| GenerateKey(SignatureTypeED25519) | ||
|
|
||
| badger := simulator.BadgerDbOpener(t.TempDir(), func(err error) { require.NoError(t, err) }) | ||
| simOpts := []simulator.Option{ | ||
| simulator.SimpleNetwork(t.Name(), 1, 1), | ||
| simulator.Genesis(GenesisTime).With(alice).WithVersion(ExecutorVersionV2Vandenberg), | ||
| } | ||
|
|
||
| // Execute a transaction with the original database (timestamp = 1) | ||
| sim := NewSim(t, append(simOpts, | ||
| simulator.WithDatabase(badger), | ||
| )...) | ||
|
|
||
| st := sim.BuildAndSubmitTxnSuccessfully( | ||
| build.Transaction().For(alice, "book", "1"). | ||
| BurnCredits(1). | ||
| SignWith(alice, "book", "1").Version(1).Timestamp(1).PrivateKey(aliceKey)) | ||
| sim.StepUntil( | ||
| Txn(st.TxID).Completes()) | ||
|
|
||
| // Execute a transaction in an overlay (timestamp = 2) | ||
| sim = NewSim(t, append(simOpts, | ||
| simulator.OverlayDatabase(simulator.MemoryDbOpener, badger), | ||
| )...) | ||
|
|
||
| st = sim.BuildAndSubmitTxnSuccessfully( | ||
| build.Transaction().For(alice, "book", "1"). | ||
| BurnCredits(1). | ||
| SignWith(alice, "book", "1").Version(1).Timestamp(2).PrivateKey(aliceKey)) | ||
| sim.StepUntil( | ||
| Txn(st.TxID).Completes()) | ||
|
|
||
| // Verify that the same timestamp fails | ||
| st = sim.BuildAndSubmit( | ||
| build.Transaction().For(alice, "book", "1"). | ||
| BurnCredits(1). | ||
| SignWith(alice, "book", "1").Version(1).Timestamp(2).PrivateKey(aliceKey))[1] | ||
| require.ErrorIs(t, st.AsError(), errors.BadTimestamp) | ||
|
|
||
| // Executing the transaction with the same timestamp in another overlay succeeds | ||
| sim = NewSim(t, append(simOpts, | ||
| simulator.OverlayDatabase(simulator.MemoryDbOpener, badger), | ||
| )...) | ||
|
|
||
| st = sim.BuildAndSubmitTxnSuccessfully( | ||
| build.Transaction().For(alice, "book", "1"). | ||
| BurnCredits(1). | ||
| SignWith(alice, "book", "1").Version(1).Timestamp(2).PrivateKey(aliceKey)) | ||
| sim.StepUntil( | ||
| Txn(st.TxID).Completes()) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.