diff --git a/buildSrc/src/main/groovy/Mockery.groovy b/buildSrc/src/main/groovy/Mockery.groovy index 44a9f574d..91686b286 100644 --- a/buildSrc/src/main/groovy/Mockery.groovy +++ b/buildSrc/src/main/groovy/Mockery.groovy @@ -59,7 +59,11 @@ class Mockery extends DefaultTask { @Optional String outputPackage + @Input + boolean inpackage = false + @OutputDirectory + @Optional File outputDir void inputDir(Object dir) { @@ -82,6 +86,10 @@ class Mockery extends DefaultTask { outputDir = project.file(output) } + void inpackage(boolean inPackage) { + inpackage = inPackage + } + protected void configure(ExecSpec spec) { if (inputDir != null) { spec.args '--dir', inputDir @@ -98,6 +106,9 @@ class Mockery extends DefaultTask { if (outputDir != null) { spec.args '--output', outputDir } + if (inpackage) { + spec.args '--inpackage' + } } } diff --git a/config/pkg/pldconf/config.go b/config/pkg/pldconf/config.go index e902a2f9f..92c5a4420 100644 --- a/config/pkg/pldconf/config.go +++ b/config/pkg/pldconf/config.go @@ -31,10 +31,12 @@ type PaladinConfig struct { TransportManagerConfig `json:",inline"` RegistryManagerConfig `json:",inline"` KeyManagerConfig `json:",inline"` + Startup StartupConfig `json:"startup"` Log LogConfig `json:"log"` Blockchain EthClientConfig `json:"blockchain"` DB DBConfig `json:"db"` RPCServer RPCServerConfig `json:"rpcServer"` + DebugServer DebugServerConfig `json:"debugServer"` StateStore StateStoreConfig `json:"statestore"` BlockIndexer BlockIndexerConfig `json:"blockIndexer"` TempDir *string `json:"tempDir"` diff --git a/config/pkg/pldconf/domainmgr.go b/config/pkg/pldconf/domainmgr.go index e86d5aa18..000912659 100644 --- a/config/pkg/pldconf/domainmgr.go +++ b/config/pkg/pldconf/domainmgr.go @@ -17,6 +17,7 @@ package pldconf import ( "github.com/kaleido-io/paladin/config/pkg/confutil" + "github.com/kaleido-io/paladin/toolkit/pkg/tktypes" ) // Intended to be embedded at root level of paladin config @@ -35,8 +36,11 @@ type DomainConfig struct { Config map[string]any `json:"config"` RegistryAddress string `json:"registryAddress"` AllowSigning bool `json:"allowSigning"` + DefaultGasLimit *uint64 `json:"defaultGasLimit"` } +var DefaultDefaultGasLimit tktypes.HexUint64 = 4000000 // high gas limit by default (accommodating zkp transactions) + var ContractCacheDefaults = &CacheConfig{ Capacity: confutil.P(1000), } diff --git a/config/pkg/pldconf/httpserver.go b/config/pkg/pldconf/httpserver.go index d9a65a6a7..bf9c9273e 100644 --- a/config/pkg/pldconf/httpserver.go +++ b/config/pkg/pldconf/httpserver.go @@ -53,3 +53,12 @@ type StaticServerConfig struct { StaticPath string `json:"staticPath"` // Path to the static files in the server FS e.g /app/ui URLPath string `json:"urlPath"` // URL path to serve the static files e.g /ui -> http://host:port/ui } + +type DebugServerConfig struct { + Enabled *bool `json:"enabled"` + HTTPServerConfig +} + +var DebugServerDefaults = &DebugServerConfig{ + Enabled: confutil.P(false), +} diff --git a/config/pkg/pldconf/privatetxmgr.go b/config/pkg/pldconf/privatetxmgr.go index fbf097b3d..de6337b2b 100644 --- a/config/pkg/pldconf/privatetxmgr.go +++ b/config/pkg/pldconf/privatetxmgr.go @@ -37,19 +37,25 @@ var DistributerWriterConfigDefaults = FlushWriterConfig{ var PrivateTxManagerDefaults = &PrivateTxManagerConfig{ Sequencer: PrivateTxManagerSequencerConfig{ - MaxConcurrentProcess: confutil.P(500), - EvaluationInterval: confutil.P("5m"), - PersistenceRetryTimeout: confutil.P("5s"), - StaleTimeout: confutil.P("10m"), - MaxPendingEvents: confutil.P(500), + MaxConcurrentProcess: confutil.P(500), + MaxInflightTransactions: confutil.P(500), + EvaluationInterval: confutil.P("5m"), + PersistenceRetryTimeout: confutil.P("5s"), + StaleTimeout: confutil.P("10m"), + MaxPendingEvents: confutil.P(500), + RoundRobinCoordinatorBlockRangeSize: confutil.P(100), + AssembleRequestTimeout: confutil.P("1s"), }, - RequestTimeout: confutil.P("15s"), + RequestTimeout: confutil.P("1s"), } type PrivateTxManagerSequencerConfig struct { - MaxConcurrentProcess *int `json:"maxConcurrentProcess,omitempty"` - MaxPendingEvents *int `json:"maxPendingEvents,omitempty"` - EvaluationInterval *string `json:"evalInterval,omitempty"` - PersistenceRetryTimeout *string `json:"persistenceRetryTimeout,omitempty"` - StaleTimeout *string `json:"staleTimeout,omitempty"` + MaxConcurrentProcess *int `json:"maxConcurrentProcess,omitempty"` + MaxInflightTransactions *int `json:"maxInflightTransactions,omitempty"` + MaxPendingEvents *int `json:"maxPendingEvents,omitempty"` + EvaluationInterval *string `json:"evalInterval,omitempty"` + PersistenceRetryTimeout *string `json:"persistenceRetryTimeout,omitempty"` + StaleTimeout *string `json:"staleTimeout,omitempty"` + RoundRobinCoordinatorBlockRangeSize *int `json:"roundRobinCoordinatorBlockRangeSize,omitempty"` + AssembleRequestTimeout *string `json:"assembleRequestTimeout,omitempty"` } diff --git a/config/pkg/pldconf/startup.go b/config/pkg/pldconf/startup.go new file mode 100644 index 000000000..32b6123f3 --- /dev/null +++ b/config/pkg/pldconf/startup.go @@ -0,0 +1,36 @@ +// Copyright © 2024 Kaleido, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pldconf + +import ( + "github.com/kaleido-io/paladin/config/pkg/confutil" +) + +type StartupConfig struct { + BlockchainConnectRetry RetryConfigWithMax `json:"blockchainConnectRetry"` +} + +var StartupConfigDefaults = StartupConfig{ + BlockchainConnectRetry: RetryConfigWithMax{ + RetryConfig: RetryConfig{ + InitialDelay: confutil.P("500ms"), + MaxDelay: confutil.P("2s"), + Factor: confutil.P(2.0), + }, + MaxAttempts: confutil.P(10), + }, +} diff --git a/core/go/build.gradle b/core/go/build.gradle index 71a423dc3..b6c21292f 100644 --- a/core/go/build.gradle +++ b/core/go/build.gradle @@ -31,9 +31,10 @@ ext { include "mocks/**/*.go" } - targetCoverage = 93 + targetCoverage = 92.5 maxCoverageBarGap = 1 coverageExcludedPackages = [ + 'github.com/kaleido-io/paladin/core/internal/privatetxnmgr/ptmgrtypes/mock_transaction_flow.go', 'github.com/kaleido-io/paladin/core/pkg/proto', 'github.com/kaleido-io/paladin/core/pkg/proto/transaction', 'github.com/kaleido-io/paladin/toolkit/prototk', @@ -211,12 +212,6 @@ task makeMocks(type: Mockery, dependsOn: [":installMockery", protoc, goGet]) { outputPackage 'componentmocks' outputDir 'mocks/componentmocks' } - mock { - inputDir 'internal/components' - name 'PublicTxBatch' - outputPackage 'componentmocks' - outputDir 'mocks/componentmocks' - } mock { inputDir "go list -f {{.Dir}} github.com/kaleido-io/paladin/toolkit/pkg/rpcserver".execute().text.trim() includeAll true @@ -247,6 +242,11 @@ task makeMocks(type: Mockery, dependsOn: [":installMockery", protoc, goGet]) { outputPackage 'preparedtxdistributionmocks' outputDir 'mocks/preparedtxdistributionmocks' } + mock { + inputDir 'internal/privatetxnmgr/ptmgrtypes' + name "TransactionFlow" + inpackage true + } } task clean(type: Delete, dependsOn: ':testinfra:stopTestInfra') { diff --git a/core/go/componenttest/component_test.go b/core/go/componenttest/component_test.go index 9687ac085..9ec12e59e 100644 --- a/core/go/componenttest/component_test.go +++ b/core/go/componenttest/component_test.go @@ -21,12 +21,13 @@ package componenttest import ( "context" + "encoding/json" + "fmt" "testing" "time" "github.com/google/uuid" "github.com/hyperledger/firefly-signer/pkg/abi" - "github.com/kaleido-io/paladin/config/pkg/pldconf" "github.com/kaleido-io/paladin/core/componenttest/domains" "github.com/kaleido-io/paladin/core/pkg/blockindexer" @@ -34,6 +35,7 @@ import ( "github.com/kaleido-io/paladin/toolkit/pkg/pldapi" "github.com/kaleido-io/paladin/toolkit/pkg/pldclient" "github.com/kaleido-io/paladin/toolkit/pkg/query" + "github.com/kaleido-io/paladin/toolkit/pkg/rpcclient" "github.com/kaleido-io/paladin/toolkit/pkg/solutils" "github.com/kaleido-io/paladin/toolkit/pkg/tktypes" "github.com/kaleido-io/paladin/toolkit/pkg/verifiers" @@ -41,7 +43,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "gorm.io/gorm" - "sigs.k8s.io/yaml" ) func TestRunSimpleStorageEthTransaction(t *testing.T) { @@ -49,48 +50,9 @@ func TestRunSimpleStorageEthTransaction(t *testing.T) { ctx := context.Background() logrus.SetLevel(logrus.DebugLevel) - var testConfig pldconf.PaladinConfig - - err := yaml.Unmarshal([]byte(` -db: - type: sqlite - sqlite: - dsn: ":memory:" - autoMigrate: true - migrationsDir: ../db/migrations/sqlite - debugQueries: false -blockIndexer: - fromBlock: latest -blockchain: - http: - url: http://localhost:8545 - ws: - url: ws://localhost:8546 - initialConnectAttempts: 25 -log: - level: debug - output: file - file: - filename: build/testbed.component-test.log -wallets: -- name: wallet1 - keySelector: .* - signer: - keyDerivation: - type: "bip32" - keyStore: - type: "static" - static: - keys: - seed: - encoding: none - inline: polar mechanic crouch jungle field room dry sure machine brisk seed bulk student total ethics -`), &testConfig) - require.NoError(t, err) - instance := newInstanceForComponentTesting(t, deployDomainRegistry(t), nil, nil, nil) cm := instance.cm - c := pldclient.Wrap(instance.client).ReceiptPollingInterval(100 * time.Millisecond) + c := pldclient.Wrap(instance.client).ReceiptPollingInterval(250 * time.Millisecond) build, err := solutils.LoadBuild(ctx, simpleStorageBuildJSON) require.NoError(t, err) @@ -365,7 +327,6 @@ func TestPrivateTransactionsMintThenTransfer(t *testing.T) { func TestPrivateTransactionRevertedAssembleFailed(t *testing.T) { // Invoke a transaction that will fail to assemble // in this case, we use the simple token domain and attempt to transfer from a wallet that has no tokens - ctx := context.Background() instance := newInstanceForComponentTesting(t, deployDomainRegistry(t), nil, nil, nil) rpcClient := instance.client @@ -437,7 +398,7 @@ func TestPrivateTransactionRevertedAssembleFailed(t *testing.T) { require.NotNil(t, txFull.Receipt) assert.False(t, txFull.Receipt.Success) assert.Regexp(t, domains.SimpleDomainInsufficientFundsError, txFull.Receipt.FailureMessage) - assert.Regexp(t, "PD011802", txFull.Receipt.FailureMessage) + assert.Regexp(t, "SDE0001", txFull.Receipt.FailureMessage) } @@ -628,7 +589,7 @@ func TestCreateStateOnOneNodeSpendOnAnother(t *testing.T) { alice.peer(bob.nodeConfig) bob.peer(alice.nodeConfig) - domainConfig := domains.SimpleDomainConfig{ + domainConfig := &domains.SimpleDomainConfig{ SubmitMode: domains.ENDORSER_SUBMISSION, } alice.start(t, domainConfig) @@ -812,7 +773,6 @@ func TestNotaryDelegated(t *testing.T) { ) } - func TestNotaryDelegatedPrepare(t *testing.T) { //Similar to the TestNotaryDelegated test except in this case, the transaction is not submitted to the base ledger by the notary. //instead, the assembled and prepared transaction is returned to the sender node to submit to the base ledger whenever it is deemed appropriate @@ -944,10 +904,583 @@ func TestNotaryDelegatedPrepare(t *testing.T) { } -func TestPrivateTransactions100PercentEndorsement(t *testing.T) { +func TestSingleNodeSelfEndorseConcurrentSpends(t *testing.T) { + // Invoke a bunch of transactions on the same contract on a single node, in self endorsement mode ( a la zeto ) + // where there is a reasonable possibility of contention between transactions + + //start by minting 5 coins then send 5 transactions to spend them + // if there is no contention, each transfer should be able to spend a coin each + + ctx := context.Background() + instance := newInstanceForComponentTesting(t, deployDomainRegistry(t), nil, nil, nil) + rpcClient := instance.client + + var dplyTxID uuid.UUID + err := rpcClient.CallRPC(ctx, &dplyTxID, "ptx_sendTransaction", &pldapi.TransactionInput{ + ABI: *domains.SimpleTokenConstructorABI(domains.SelfEndorsement), + TransactionBase: pldapi.TransactionBase{ + IdempotencyKey: "deploy1", + Type: pldapi.TransactionTypePrivate.Enum(), + Domain: "domain1", + From: "wallets.org1.aaaaaa", + Data: tktypes.RawJSON(`{ + "from": "wallets.org1.aaaaaa", + "name": "FakeToken1", + "symbol": "FT1", + "endorsementMode": "` + domains.SelfEndorsement + `" + }`), + }, + }) + require.NoError(t, err) + assert.Eventually(t, + transactionReceiptCondition(t, ctx, dplyTxID, rpcClient, true), + transactionLatencyThreshold(t)+5*time.Second, //TODO deploy transaction seems to take longer than expected + 100*time.Millisecond, + "Deploy transaction did not receive a receipt", + ) + + var dplyTxFull pldapi.TransactionFull + err = rpcClient.CallRPC(ctx, &dplyTxFull, "ptx_getTransactionFull", dplyTxID) + require.NoError(t, err) + require.NotNil(t, dplyTxFull.Receipt) + require.True(t, dplyTxFull.Receipt.Success) + require.NotNil(t, dplyTxFull.Receipt.ContractAddress) + contractAddress := dplyTxFull.Receipt.ContractAddress + + var receiptData pldapi.TransactionReceiptData + err = rpcClient.CallRPC(ctx, &receiptData, "ptx_getTransactionReceipt", dplyTxID) + assert.NoError(t, err) + assert.True(t, receiptData.Success) + assert.Equal(t, contractAddress, receiptData.ContractAddress) + + // Do the 5 mints + mint := func() (id uuid.UUID) { + var txID uuid.UUID + err = rpcClient.CallRPC(ctx, &txID, "ptx_sendTransaction", &pldapi.TransactionInput{ + ABI: *domains.SimpleTokenTransferABI(), + TransactionBase: pldapi.TransactionBase{ + To: contractAddress, + Domain: "domain1", + IdempotencyKey: tktypes.RandHex(8), + Type: pldapi.TransactionTypePrivate.Enum(), + From: "wallets.org1.aaaaaa", + Data: tktypes.RawJSON(`{ + "from": "", + "to": "wallets.org1.aaaaaa", + "amount": "1" + }`), + }, + }) + require.NoError(t, err) + assert.NotEqual(t, uuid.UUID{}, txID) + return txID + } + waitForTransaction := func(txID uuid.UUID) { + assert.Eventually(t, + transactionReceiptCondition(t, ctx, txID, rpcClient, false), + transactionLatencyThreshold(t), + 100*time.Millisecond, + "Transaction did not receive a receipt", + ) + } + + mint1 := mint() + mint2 := mint() + mint3 := mint() + mint4 := mint() + mint5 := mint() + + waitForTransaction(mint1) + waitForTransaction(mint2) + waitForTransaction(mint3) + waitForTransaction(mint4) + waitForTransaction(mint5) + //time.Sleep(1000 * time.Millisecond) + // Now kick off the 5 transfers + transfer := func() (id uuid.UUID) { + var txID uuid.UUID + err = rpcClient.CallRPC(ctx, &txID, "ptx_sendTransaction", &pldapi.TransactionInput{ + ABI: *domains.SimpleTokenTransferABI(), + TransactionBase: pldapi.TransactionBase{ + To: contractAddress, + Domain: "domain1", + IdempotencyKey: tktypes.RandHex(8), + Type: pldapi.TransactionTypePrivate.Enum(), + From: "wallets.org1.aaaaaa", + Data: tktypes.RawJSON(`{ + "from": "wallets.org1.aaaaaa", + "to": "wallets.org1.bbbbbb", + "amount": "1" + }`), + }, + }) + + require.NoError(t, err) + assert.NotEqual(t, uuid.UUID{}, txID) + return txID + } + transfer1 := transfer() + transfer2 := transfer() + transfer3 := transfer() + transfer4 := transfer() + transfer5 := transfer() + + waitForTransaction(transfer1) + waitForTransaction(transfer2) + waitForTransaction(transfer3) + waitForTransaction(transfer4) + waitForTransaction(transfer5) + +} + +func TestSingleNodeSelfEndorseSeriesOfTransfers(t *testing.T) { + // Invoke a series of transactions on the same contract on a single node, in self endorsement mode ( a la zeto ) + //where each transaction relies on the state created by the previous + + ctx := context.Background() + instance := newInstanceForComponentTesting(t, deployDomainRegistry(t), nil, nil, nil) + rpcClient := instance.client + + // Check there are no transactions before we start + var txns []*pldapi.TransactionFull + err := rpcClient.CallRPC(ctx, &txns, "ptx_queryTransactionsFull", query.NewQueryBuilder().Limit(1).Query()) + require.NoError(t, err) + assert.Len(t, txns, 0) + var dplyTxID uuid.UUID + err = rpcClient.CallRPC(ctx, &dplyTxID, "ptx_sendTransaction", &pldapi.TransactionInput{ + ABI: *domains.SimpleTokenConstructorABI(domains.SelfEndorsement), + TransactionBase: pldapi.TransactionBase{ + IdempotencyKey: "deploy1", + Type: pldapi.TransactionTypePrivate.Enum(), + Domain: "domain1", + From: "wallets.org1.aaaaaa", + Data: tktypes.RawJSON(`{ + "from": "wallets.org1.aaaaaa", + "name": "FakeToken1", + "symbol": "FT1", + "endorsementMode": "` + domains.SelfEndorsement + `" + }`), + }, + }) + require.NoError(t, err) + assert.Eventually(t, + transactionReceiptCondition(t, ctx, dplyTxID, rpcClient, true), + transactionLatencyThreshold(t)+5*time.Second, //TODO deploy transaction seems to take longer than expected + 100*time.Millisecond, + "Deploy transaction did not receive a receipt", + ) + + var dplyTxFull pldapi.TransactionFull + err = rpcClient.CallRPC(ctx, &dplyTxFull, "ptx_getTransactionFull", dplyTxID) + require.NoError(t, err) + require.NotNil(t, dplyTxFull.Receipt) + require.True(t, dplyTxFull.Receipt.Success) + require.NotNil(t, dplyTxFull.Receipt.ContractAddress) + contractAddress := dplyTxFull.Receipt.ContractAddress + + var receiptData pldapi.TransactionReceiptData + err = rpcClient.CallRPC(ctx, &receiptData, "ptx_getTransactionReceipt", dplyTxID) + assert.NoError(t, err) + assert.True(t, receiptData.Success) + assert.Equal(t, contractAddress, receiptData.ContractAddress) + + // Start a private transaction - Mint to alice + var tx1ID uuid.UUID + err = rpcClient.CallRPC(ctx, &tx1ID, "ptx_sendTransaction", &pldapi.TransactionInput{ + ABI: *domains.SimpleTokenTransferABI(), + TransactionBase: pldapi.TransactionBase{ + To: contractAddress, + Domain: "domain1", + IdempotencyKey: "tx1", + Type: pldapi.TransactionTypePrivate.Enum(), + From: "wallets.org1.aaaaaa", + Data: tktypes.RawJSON(`{ + "from": "", + "to": "wallets.org1.bbbbbb", + "amount": "100" + }`), + }, + }) + + require.NoError(t, err) + assert.NotEqual(t, uuid.UUID{}, tx1ID) + + // Start a private transaction - Transfer from alice to bob + var tx2ID uuid.UUID + err = rpcClient.CallRPC(ctx, &tx2ID, "ptx_sendTransaction", &pldapi.TransactionInput{ + ABI: *domains.SimpleTokenTransferABI(), + TransactionBase: pldapi.TransactionBase{ + To: contractAddress, + Domain: "domain1", + IdempotencyKey: "tx2", + Type: pldapi.TransactionTypePrivate.Enum(), + From: "wallets.org1.bbbbbb", + Data: tktypes.RawJSON(`{ + "from": "wallets.org1.bbbbbb", + "to": "wallets.org1.aaaaaa", + "amount": "99" + }`), + }, + }) + + require.NoError(t, err) + assert.NotEqual(t, uuid.UUID{}, tx2ID) + //time.Sleep(1000 * time.Millisecond) // Add a small delay to avoid a tight loop + // Start a private transaction - Transfer from alice to bob + var tx3ID uuid.UUID + err = rpcClient.CallRPC(ctx, &tx3ID, "ptx_sendTransaction", &pldapi.TransactionInput{ + ABI: *domains.SimpleTokenTransferABI(), + TransactionBase: pldapi.TransactionBase{ + To: contractAddress, + Domain: "domain1", + IdempotencyKey: "tx3", + Type: pldapi.TransactionTypePrivate.Enum(), + From: "wallets.org1.aaaaaa", + Data: tktypes.RawJSON(`{ + "from": "wallets.org1.aaaaaa", + "to": "wallets.org1.bbbbbb", + "amount": "98" + }`), + }, + }) + + require.NoError(t, err) + assert.NotEqual(t, uuid.UUID{}, tx3ID) + + assert.Eventually(t, + transactionReceiptCondition(t, ctx, tx1ID, rpcClient, false), + transactionLatencyThreshold(t), + 100*time.Millisecond, + "Transaction did not receive a receipt", + ) + + assert.Eventually(t, + transactionReceiptCondition(t, ctx, tx2ID, rpcClient, false), + transactionLatencyThreshold(t), + 100*time.Millisecond, + "Transaction did not receive a receipt", + ) + + assert.Eventually(t, + transactionReceiptCondition(t, ctx, tx3ID, rpcClient, false), + transactionLatencyThreshold(t), + 100*time.Millisecond, + "Transaction did not receive a receipt", + ) + +} + +func TestNotaryEndorseConcurrentSpends(t *testing.T) { + // Invoke a bunch of transactions on the same contract in self endorsement mode ( a la noto ) + // perform the transfers from the same identity so that there is high likelihood of contention + + //start by minting 5 coins then send 5 transactions to spend them + // if there is no contention, each transfer should be able to spend a coin each + + ctx := context.Background() + + aliceNodeConfig := newNodeConfiguration(t, "alice") + bobNodeConfig := newNodeConfiguration(t, "bob") + notaryNodeConfig := newNodeConfiguration(t, "notary") + + domainRegistryAddress := deployDomainRegistry(t) + + instance1 := newInstanceForComponentTesting(t, domainRegistryAddress, aliceNodeConfig, []*nodeConfiguration{bobNodeConfig, notaryNodeConfig}, nil) + client1 := instance1.client + aliceIdentity := "wallets.org1.alice@" + instance1.name + + instance2 := newInstanceForComponentTesting(t, domainRegistryAddress, bobNodeConfig, []*nodeConfiguration{aliceNodeConfig, notaryNodeConfig}, nil) + bobIdentity := "wallets.org2.bob@" + instance2.name + + instance3 := newInstanceForComponentTesting(t, domainRegistryAddress, notaryNodeConfig, []*nodeConfiguration{aliceNodeConfig, bobNodeConfig}, nil) + client3 := instance3.client + notaryIdentity := "wallets.org3.notary@" + instance3.name + + // send JSON RPC message to node 3 ( notary) to deploy a private contract + var dplyTxID uuid.UUID + err := client3.CallRPC(ctx, &dplyTxID, "ptx_sendTransaction", &pldapi.TransactionInput{ + ABI: *domains.SimpleTokenConstructorABI(domains.NotaryEndorsement), + TransactionBase: pldapi.TransactionBase{ + IdempotencyKey: "deploy1", + Type: pldapi.TransactionTypePrivate.Enum(), + Domain: "domain1", + From: notaryIdentity, + Data: tktypes.RawJSON(`{ + "notary": "` + notaryIdentity + `", + "name": "FakeToken1", + "symbol": "FT1", + "endorsementMode": "NotaryEndorsement" + }`), + }, + }) + require.NoError(t, err) + assert.Eventually(t, + transactionReceiptCondition(t, ctx, dplyTxID, client3, true), + transactionLatencyThreshold(t)+5*time.Second, //TODO deploy transaction seems to take longer than expected + 100*time.Millisecond, + "Deploy transaction did not receive a receipt", + ) + + var dplyTxFull pldapi.TransactionFull + err = client3.CallRPC(ctx, &dplyTxFull, "ptx_getTransactionFull", dplyTxID) + require.NoError(t, err) + contractAddress := dplyTxFull.Receipt.ContractAddress + + // Start a private transaction on notary node + // this is a mint to alice so alice should later be able to do a transfer to bob + + // Do the 5 mints + mint := func() (id uuid.UUID) { + var txID uuid.UUID + err = client3.CallRPC(ctx, &txID, "ptx_sendTransaction", &pldapi.TransactionInput{ + ABI: *domains.SimpleTokenTransferABI(), + TransactionBase: pldapi.TransactionBase{ + To: contractAddress, + Domain: "domain1", + IdempotencyKey: tktypes.RandHex(8), + Type: pldapi.TransactionTypePrivate.Enum(), + From: notaryIdentity, + Data: tktypes.RawJSON(`{ + "from": "", + "to": "` + aliceIdentity + `", + "amount": "100" + }`), + }, + }) + require.NoError(t, err) + assert.NotEqual(t, uuid.UUID{}, txID) + return txID + } + waitForTransaction := func(txID uuid.UUID, client rpcclient.Client) { + assert.Eventually(t, + transactionReceiptCondition(t, ctx, txID, client, false), + transactionLatencyThreshold(t), + 100*time.Millisecond, + "Transaction did not receive a receipt", + ) + } + + mint1 := mint() + mint2 := mint() + mint3 := mint() + mint4 := mint() + mint5 := mint() + + waitForTransaction(mint1, client3) + waitForTransaction(mint2, client3) + waitForTransaction(mint3, client3) + waitForTransaction(mint4, client3) + waitForTransaction(mint5, client3) + + // Now kick off the 5 transfers + transfer := func() (id uuid.UUID) { + var txID uuid.UUID + err = client1.CallRPC(ctx, &txID, "ptx_sendTransaction", &pldapi.TransactionInput{ + ABI: *domains.SimpleTokenTransferABI(), + TransactionBase: pldapi.TransactionBase{ + To: contractAddress, + Domain: "domain1", + IdempotencyKey: tktypes.RandHex(8), + Type: pldapi.TransactionTypePrivate.Enum(), + From: aliceIdentity, + Data: tktypes.RawJSON(`{ + "from": "` + aliceIdentity + `", + "to": "` + bobIdentity + `", + "amount": "100" + }`), + }, + }) + + require.NoError(t, err) + assert.NotEqual(t, uuid.UUID{}, txID) + return txID + } + transfer1 := transfer() + transfer2 := transfer() + transfer3 := transfer() + transfer4 := transfer() + transfer5 := transfer() + + waitForTransaction(transfer1, client1) + waitForTransaction(transfer2, client1) + waitForTransaction(transfer3, client1) + waitForTransaction(transfer4, client1) + waitForTransaction(transfer5, client1) + +} + +func TestNotaryEndorseSeriesOfTransfers(t *testing.T) { + + // Invoke a series of transactions on the same contract in self endorsement mode ( a la neto ) + // where each transaction relies on the state created by the previous + t.Skip("This is an invalid test because there is no meaning to the term 'previous' in a concurrent system") + + ctx := context.Background() + + aliceNodeConfig := newNodeConfiguration(t, "alice") + bobNodeConfig := newNodeConfiguration(t, "bob") + notaryNodeConfig := newNodeConfiguration(t, "notary") + + domainRegistryAddress := deployDomainRegistry(t) + + instance1 := newInstanceForComponentTesting(t, domainRegistryAddress, aliceNodeConfig, []*nodeConfiguration{bobNodeConfig, notaryNodeConfig}, nil) + client1 := instance1.client + aliceIdentity := "wallets.org1.alice@" + instance1.name + + instance2 := newInstanceForComponentTesting(t, domainRegistryAddress, bobNodeConfig, []*nodeConfiguration{aliceNodeConfig, notaryNodeConfig}, nil) + client2 := instance2.client + bobIdentity := "wallets.org2.bob@" + instance2.name + + instance3 := newInstanceForComponentTesting(t, domainRegistryAddress, notaryNodeConfig, []*nodeConfiguration{aliceNodeConfig, bobNodeConfig}, nil) + client3 := instance3.client + notaryIdentity := "wallets.org3.notary@" + instance3.name + + // send JSON RPC message to node 3 ( notary) to deploy a private contract + var dplyTxID uuid.UUID + err := client3.CallRPC(ctx, &dplyTxID, "ptx_sendTransaction", &pldapi.TransactionInput{ + ABI: *domains.SimpleTokenConstructorABI(domains.NotaryEndorsement), + TransactionBase: pldapi.TransactionBase{ + IdempotencyKey: "deploy1", + Type: pldapi.TransactionTypePrivate.Enum(), + Domain: "domain1", + From: notaryIdentity, + Data: tktypes.RawJSON(`{ + "notary": "` + notaryIdentity + `", + "name": "FakeToken1", + "symbol": "FT1", + "endorsementMode": "NotaryEndorsement" + }`), + }, + }) + require.NoError(t, err) + assert.Eventually(t, + transactionReceiptCondition(t, ctx, dplyTxID, client3, true), + transactionLatencyThreshold(t)+5*time.Second, //TODO deploy transaction seems to take longer than expected + 100*time.Millisecond, + "Deploy transaction did not receive a receipt", + ) + var dplyTxFull pldapi.TransactionFull + err = client3.CallRPC(ctx, &dplyTxFull, "ptx_getTransactionFull", dplyTxID) + require.NoError(t, err) + contractAddress := dplyTxFull.Receipt.ContractAddress + + //Start with a mint to alice + var mintTxID uuid.UUID + err = client3.CallRPC(ctx, &mintTxID, "ptx_sendTransaction", &pldapi.TransactionInput{ + ABI: *domains.SimpleTokenTransferABI(), + TransactionBase: pldapi.TransactionBase{ + To: contractAddress, + Domain: "domain1", + IdempotencyKey: "tx1-mint", + Type: pldapi.TransactionTypePrivate.Enum(), + From: notaryIdentity, + Data: tktypes.RawJSON(`{ + "from": "", + "to": "` + aliceIdentity + `", + "amount": "100" + }`), + }, + }) + + require.NoError(t, err) + assert.NotEqual(t, uuid.UUID{}, mintTxID) + + // Alice transfers some of her recently minted tokens to bob + var transferA2B1TxId uuid.UUID + err = client1.CallRPC(ctx, &transferA2B1TxId, "ptx_sendTransaction", &pldapi.TransactionInput{ + ABI: *domains.SimpleTokenTransferABI(), + TransactionBase: pldapi.TransactionBase{ + To: contractAddress, + Domain: "domain1", + IdempotencyKey: "transferA2B1", + Type: pldapi.TransactionTypePrivate.Enum(), + From: aliceIdentity, + Data: tktypes.RawJSON(`{ + "from": "` + aliceIdentity + `", + "to": "` + bobIdentity + `", + "amount": "99" + }`), + }, + }) + + require.NoError(t, err) + assert.NotEqual(t, uuid.UUID{}, transferA2B1TxId) + + // Bob sends some tokens back to alice + var transferB2A1TxId uuid.UUID + err = client2.CallRPC(ctx, &transferB2A1TxId, "ptx_sendTransaction", &pldapi.TransactionInput{ + ABI: *domains.SimpleTokenTransferABI(), + TransactionBase: pldapi.TransactionBase{ + To: contractAddress, + Domain: "domain1", + IdempotencyKey: "transferB2A1", + Type: pldapi.TransactionTypePrivate.Enum(), + From: bobIdentity, + Data: tktypes.RawJSON(`{ + "from": "` + bobIdentity + `", + "to": "` + aliceIdentity + `", + "amount": "95" + }`), + }, + }) + + require.NoError(t, err) + assert.NotEqual(t, uuid.UUID{}, transferB2A1TxId) + + //Alice can transfer 99 to bob. The 98 bob just transferred to here and the 1 change from her earlier transfer to bob + var transferA2B2TxId uuid.UUID + err = client1.CallRPC(ctx, &transferA2B2TxId, "ptx_sendTransaction", &pldapi.TransactionInput{ + ABI: *domains.SimpleTokenTransferABI(), + TransactionBase: pldapi.TransactionBase{ + To: contractAddress, + Domain: "domain1", + IdempotencyKey: "transferA2B2", + Type: pldapi.TransactionTypePrivate.Enum(), + From: aliceIdentity, + Data: tktypes.RawJSON(`{ + "from": "` + aliceIdentity + `", + "to": "` + bobIdentity + `", + "amount": "90" + }`), + }, + }) + + require.NoError(t, err) + assert.NotEqual(t, uuid.UUID{}, transferA2B2TxId) + + assert.Eventually(t, + transactionReceiptCondition(t, ctx, mintTxID, client3, false), + transactionLatencyThreshold(t), + 100*time.Millisecond, + "Transaction did not receive a receipt", + ) + assert.Eventually(t, + transactionReceiptCondition(t, ctx, transferA2B1TxId, client1, false), + transactionLatencyThreshold(t), + 100*time.Millisecond, + "Transaction did not receive a receipt", + ) + assert.Eventually(t, + transactionReceiptCondition(t, ctx, transferB2A1TxId, client2, false), + transactionLatencyThreshold(t), + 100*time.Millisecond, + "Transaction did not receive a receipt", + ) + assert.Eventually(t, + transactionReceiptCondition(t, ctx, transferA2B2TxId, client1, false), + transactionLatencyThreshold(t), + 100*time.Millisecond, + "Transaction did not receive a receipt", + ) + +} + +func TestPrivacyGroupEndorsement(t *testing.T) { // This test is intended to emulate the pente domain where all transactions must be endorsed by all parties in the predefined privacy group // in this case, we have 3 nodes, each representing a different party in the privacy group // and we expect that all transactions must be endorsed by all 3 nodes and that all output states are distributed to all 3 nodes + // Unlike the coin based domains, this is a "world state" based domain so there is only ever one available state at any one time and each + // transaction spends that state and creates a new one. So there is contention between parties ctx := context.Background() domainRegistryAddress := deployDomainRegistry(t) @@ -959,7 +1492,7 @@ func TestPrivateTransactions100PercentEndorsement(t *testing.T) { bob.peer(alice.nodeConfig, carol.nodeConfig) carol.peer(alice.nodeConfig, bob.nodeConfig) - domainConfig := domains.SimpleDomainConfig{ + domainConfig := &domains.SimpleStorageDomainConfig{ SubmitMode: domains.ONE_TIME_USE_KEYS, } alice.start(t, domainConfig) @@ -968,30 +1501,28 @@ func TestPrivateTransactions100PercentEndorsement(t *testing.T) { endorsementSet := []string{alice.identityLocator, bob.identityLocator, carol.identityLocator} - constructorParameters := &domains.ConstructorParameters{ + constructorParameters := &domains.SimpleStorageConstructorParameters{ EndorsementSet: endorsementSet, - Name: "FakeToken1", - Symbol: "FT1", + Name: "SimpleStorage1", EndorsementMode: domains.PrivacyGroupEndorsement, } // send JSON RPC message to node 1 to deploy a private contract - contractAddress := alice.deploySimpleDomainInstanceContract(t, domains.PrivacyGroupEndorsement, constructorParameters) + contractAddress := alice.deploySimpleStorageDomainInstanceContract(t, domains.PrivacyGroupEndorsement, constructorParameters) // Start a private transaction on alice's node // this should require endorsement from bob and carol + // Initialise a new map var aliceTxID uuid.UUID err := alice.client.CallRPC(ctx, &aliceTxID, "ptx_sendTransaction", &pldapi.TransactionInput{ - ABI: *domains.SimpleTokenTransferABI(), + ABI: *domains.SimpleStorageInitABI(), TransactionBase: pldapi.TransactionBase{ To: contractAddress, - Domain: "domain1", + Domain: "simpleStorageDomain", IdempotencyKey: "tx1-alice", Type: pldapi.TransactionTypePrivate.Enum(), From: alice.identity, Data: tktypes.RawJSON(`{ - "from": "", - "to": "` + bob.identityLocator + `", - "amount": "123000000000000000000" + "map":"map1" }`), }, }) @@ -1005,4 +1536,265 @@ func TestPrivateTransactions100PercentEndorsement(t *testing.T) { "Transaction did not receive a receipt", ) + // Start a private transaction on bob's node + // this should require endorsement from alice and carol + var bobTxID uuid.UUID + err = bob.client.CallRPC(ctx, &bobTxID, "ptx_sendTransaction", &pldapi.TransactionInput{ + ABI: *domains.SimpleStorageSetABI(), + TransactionBase: pldapi.TransactionBase{ + To: contractAddress, + Domain: "simpleStorageDomain", + IdempotencyKey: "tx1-bob", + Type: pldapi.TransactionTypePrivate.Enum(), + From: bob.identity, + Data: tktypes.RawJSON(`{ + "map":"map1", + "key": "foo", + "value": "quz" + }`), + }, + }) + + require.NoError(t, err) + assert.NotEqual(t, uuid.UUID{}, bobTxID) + assert.Eventually(t, + transactionReceiptCondition(t, ctx, bobTxID, bob.client, false), + transactionLatencyThreshold(t), + 100*time.Millisecond, + "Transaction did not receive a receipt", + ) + + var bobSchemas []*pldapi.Schema + err = bob.client.CallRPC(ctx, &bobSchemas, "pstate_listSchemas", "simpleStorageDomain") + require.NoError(t, err) + require.Len(t, bobSchemas, 1) + + var bobStates []*pldapi.State + err = bob.client.CallRPC(ctx, &bobStates, "pstate_queryContractStates", "simpleStorageDomain", contractAddress.String(), bobSchemas[0].ID, tktypes.RawJSON(`{}`), "available") + require.NoError(t, err) + require.Len(t, bobStates, 1) + stateData := make(map[string]string) + storage := make(map[string]string) + jsonErr := json.Unmarshal(bobStates[0].Data.Bytes(), &stateData) + require.NoError(t, jsonErr) + + jsonErr = json.Unmarshal([]byte(stateData["records"]), &storage) + require.NoError(t, jsonErr) + + assert.Equal(t, "quz", storage["foo"]) + + // Alice should see the same latest state of the world as Bob + var aliceSchemas []*pldapi.Schema + + err = alice.client.CallRPC(ctx, &aliceSchemas, "pstate_listSchemas", "simpleStorageDomain") + require.NoError(t, err) + require.Len(t, aliceSchemas, 1) + assert.Equal(t, bobSchemas[0].ID, aliceSchemas[0].ID) + + var aliceStates []*pldapi.State + + err = alice.client.CallRPC(ctx, &aliceStates, "pstate_queryContractStates", "simpleStorageDomain", contractAddress.String(), aliceSchemas[0].ID, tktypes.RawJSON(`{}`), "available") + require.NoError(t, err) + require.Len(t, aliceStates, 1) + assert.Equal(t, bobStates[0].ID, aliceStates[0].ID) + assert.Equal(t, bobStates[0].Data, aliceStates[0].Data) + +} + +func TestPrivacyGroupEndorsementConcurrent(t *testing.T) { + // This test is identical to TestPrivacyGroupEndorsement except that it sends the transactions concurrently + // For manual exploratory testing of longevity , it is possible to increase the number of iterations and the test should still be valid + // however, it is hard coded to a small number by default so that it can be run in CI + NUM_ITERATIONS := 2 + NUM_TRANSACTIONS_PER_NODE_PER_ITERATION := 2 + ctx := context.Background() + domainRegistryAddress := deployDomainRegistry(t) + + alice := newPartyForTesting(t, "alice", domainRegistryAddress) + bob := newPartyForTesting(t, "bob", domainRegistryAddress) + carol := newPartyForTesting(t, "carol", domainRegistryAddress) + + alice.peer(bob.nodeConfig, carol.nodeConfig) + bob.peer(alice.nodeConfig, carol.nodeConfig) + carol.peer(alice.nodeConfig, bob.nodeConfig) + + domainConfig := &domains.SimpleStorageDomainConfig{ + SubmitMode: domains.ONE_TIME_USE_KEYS, + } + alice.start(t, domainConfig) + bob.start(t, domainConfig) + carol.start(t, domainConfig) + + endorsementSet := []string{alice.identityLocator, bob.identityLocator, carol.identityLocator} + + constructorParameters := &domains.SimpleStorageConstructorParameters{ + EndorsementSet: endorsementSet, + Name: "SimpleStorage1", + EndorsementMode: domains.PrivacyGroupEndorsement, + } + // send JSON RPC message to node 1 to deploy a private contract + contractAddress := alice.deploySimpleStorageDomainInstanceContract(t, domains.PrivacyGroupEndorsement, constructorParameters) + var initTxID uuid.UUID + err := alice.client.CallRPC(ctx, &initTxID, "ptx_sendTransaction", &pldapi.TransactionInput{ + ABI: *domains.SimpleStorageInitABI(), + TransactionBase: pldapi.TransactionBase{ + To: contractAddress, + Domain: "simpleStorageDomain", + IdempotencyKey: "init-tx", + Type: pldapi.TransactionTypePrivate.Enum(), + From: alice.identity, + Data: tktypes.RawJSON(`{ + "map":"TestPrivacyGroupEndorsementConcurrent" + }`), + }, + }) + require.NoError(t, err) + assert.NotEqual(t, uuid.UUID{}, initTxID) + assert.Eventually(t, + transactionReceiptCondition(t, ctx, initTxID, alice.client, false), + transactionLatencyThreshold(t), + 100*time.Millisecond, + "Init map transaction did not receive a receipt", + ) + + //initialize a map that all parties should be able to access concurrently + // we wait for the confirmation of this transaction to ensure that there is no race condition of someone trying to call `set` before the map is initialized + // TODO - so long as we have the transaction id for the init transaction, we could declare a dependency on it for the set transactions + for i := 0; i < NUM_ITERATIONS; i++ { + // Start a number of private transaction on alice's node + // this should require endorsement from bob and carol + aliceTxID := make([]uuid.UUID, NUM_TRANSACTIONS_PER_NODE_PER_ITERATION) + bobTxID := make([]uuid.UUID, NUM_TRANSACTIONS_PER_NODE_PER_ITERATION) + carolTxID := make([]uuid.UUID, NUM_TRANSACTIONS_PER_NODE_PER_ITERATION) + + for j := 0; j < NUM_TRANSACTIONS_PER_NODE_PER_ITERATION; j++ { + err := alice.client.CallRPC(ctx, &aliceTxID[j], "ptx_sendTransaction", &pldapi.TransactionInput{ + ABI: *domains.SimpleStorageSetABI(), + TransactionBase: pldapi.TransactionBase{ + To: contractAddress, + Domain: "simpleStorageDomain", + IdempotencyKey: fmt.Sprintf("tx1-alice-%d-%d", i, j), + Type: pldapi.TransactionTypePrivate.Enum(), + From: alice.identity, + Data: tktypes.RawJSON(fmt.Sprintf(`{ + "map":"TestPrivacyGroupEndorsementConcurrent", + "key": "alice_key_%d_%d", + "value": "alice_value_%d_%d" + }`, i, j, i, j)), + }, + }) + require.NoError(t, err) + assert.NotEqual(t, uuid.UUID{}, aliceTxID[j]) + + // Start a private transaction on bob's node + // this should require endorsement from alice and carol + err = bob.client.CallRPC(ctx, &bobTxID[j], "ptx_sendTransaction", &pldapi.TransactionInput{ + ABI: *domains.SimpleStorageSetABI(), + TransactionBase: pldapi.TransactionBase{ + To: contractAddress, + Domain: "simpleStorageDomain", + IdempotencyKey: fmt.Sprintf("tx1-bob-%d-%d", i, j), + Type: pldapi.TransactionTypePrivate.Enum(), + From: bob.identity, + Data: tktypes.RawJSON(fmt.Sprintf(`{ + "map":"TestPrivacyGroupEndorsementConcurrent", + "key": "bob_key_%d_%d", + "value": "bob_value_%d_%d" + }`, i, j, i, j)), + }, + }) + require.NoError(t, err) + assert.NotEqual(t, uuid.UUID{}, bobTxID[j]) + + err = carol.client.CallRPC(ctx, &carolTxID[j], "ptx_sendTransaction", &pldapi.TransactionInput{ + ABI: *domains.SimpleStorageSetABI(), + TransactionBase: pldapi.TransactionBase{ + To: contractAddress, + Domain: "simpleStorageDomain", + IdempotencyKey: fmt.Sprintf("tx1-carol-%d-%d", i, j), + Type: pldapi.TransactionTypePrivate.Enum(), + From: bob.identity, + Data: tktypes.RawJSON(fmt.Sprintf(`{ + "map":"TestPrivacyGroupEndorsementConcurrent", + "key": "carol_key_%d_%d", + "value": "carol_value_%d_%d" + }`, i, j, i, j)), + }, + }) + require.NoError(t, err) + assert.NotEqual(t, uuid.UUID{}, carolTxID[j]) + } + + //once all transactions for this iteration are sent, wait for all of them to be confirmed before starting the next iteration + for j := 0; j < NUM_TRANSACTIONS_PER_NODE_PER_ITERATION; j++ { + assert.Eventually(t, + transactionReceiptCondition(t, ctx, aliceTxID[j], alice.client, false), + transactionLatencyThreshold(t), + 100*time.Millisecond, + "Transaction did not receive a receipt", + ) + + assert.Eventually(t, + transactionReceiptCondition(t, ctx, bobTxID[j], bob.client, false), + transactionLatencyThreshold(t), + 100*time.Millisecond, + "Transaction did not receive a receipt", + ) + + assert.Eventually(t, + transactionReceiptCondition(t, ctx, carolTxID[j], carol.client, false), + transactionLatencyThreshold(t), + 100*time.Millisecond, + fmt.Sprintf("Carol's transaction did not receive a receipt on iteration %d", i), + ) + } + } + + var schemas []*pldapi.Schema + + err = alice.client.CallRPC(ctx, &schemas, "pstate_listSchemas", "simpleStorageDomain") + require.NoError(t, err) + require.Len(t, schemas, 1) + + err = bob.client.CallRPC(ctx, &schemas, "pstate_listSchemas", "simpleStorageDomain") + require.NoError(t, err) + require.Len(t, schemas, 1) + + err = carol.client.CallRPC(ctx, &schemas, "pstate_listSchemas", "simpleStorageDomain") + require.NoError(t, err) + require.Len(t, schemas, 1) + + var aliceStates []*pldapi.State + err = alice.client.CallRPC(ctx, &aliceStates, "pstate_queryContractStates", "simpleStorageDomain", contractAddress.String(), schemas[0].ID, tktypes.RawJSON(`{}`), "available") + require.NoError(t, err) + require.Len(t, aliceStates, 1) + + var bobStates []*pldapi.State + err = bob.client.CallRPC(ctx, &bobStates, "pstate_queryContractStates", "simpleStorageDomain", contractAddress.String(), schemas[0].ID, tktypes.RawJSON(`{}`), "available") + require.NoError(t, err) + require.Len(t, bobStates, 1) + assert.Equal(t, aliceStates[0].Data, bobStates[0].Data) + + var carolStates []*pldapi.State + err = carol.client.CallRPC(ctx, &carolStates, "pstate_queryContractStates", "simpleStorageDomain", contractAddress.String(), schemas[0].ID, tktypes.RawJSON(`{}`), "available") + require.NoError(t, err) + require.Len(t, carolStates, 1) + assert.Equal(t, aliceStates[0].Data, carolStates[0].Data) + + stateData := make(map[string]string) + storage := make(map[string]string) + jsonErr := json.Unmarshal(aliceStates[0].Data.Bytes(), &stateData) + require.NoError(t, jsonErr) + + jsonErr = json.Unmarshal([]byte(stateData["records"]), &storage) + require.NoError(t, jsonErr) + + for i := 0; i < NUM_ITERATIONS; i++ { + for j := 0; j < NUM_TRANSACTIONS_PER_NODE_PER_ITERATION; j++ { + assert.Equal(t, fmt.Sprintf("alice_value_%d_%d", i, j), storage[fmt.Sprintf("alice_key_%d_%d", i, j)]) + assert.Equal(t, fmt.Sprintf("bob_value_%d_%d", i, j), storage[fmt.Sprintf("bob_key_%d_%d", i, j)]) + assert.Equal(t, fmt.Sprintf("carol_value_%d_%d", i, j), storage[fmt.Sprintf("carol_key_%d_%d", i, j)]) + } + } + } diff --git a/core/go/componenttest/domains/simple_domain.go b/core/go/componenttest/domains/simple_domain.go index dc5dc0f8c..18c62e774 100644 --- a/core/go/componenttest/domains/simple_domain.go +++ b/core/go/componenttest/domains/simple_domain.go @@ -31,6 +31,7 @@ import ( "github.com/kaleido-io/paladin/config/pkg/confutil" "github.com/kaleido-io/paladin/core/internal/components" "github.com/kaleido-io/paladin/toolkit/pkg/algorithms" + "github.com/kaleido-io/paladin/toolkit/pkg/log" "github.com/kaleido-io/paladin/toolkit/pkg/pldapi" "github.com/kaleido-io/paladin/toolkit/pkg/plugintk" "github.com/kaleido-io/paladin/toolkit/pkg/prototk" @@ -69,6 +70,7 @@ const ( NotaryEndorsement = "NotaryEndorsement" //PrivacyGroupEndorsement is kinda like pente. + //But pente is not a token based domain so we tend to use simpleStorageDomain to emulate pente like behavior //An endorsement set is provided in the constructor and every member of that group must endorse every transaction. PrivacyGroupEndorsement = "PrivacyGroupEndorsement" ) @@ -309,6 +311,23 @@ type SimpleDomainConfig struct { SubmitMode string `json:"submitMode"` } +// ABI for the config field in the PaladinRegisterSmartContract_V0 event +// this must match the type and order of arguments passed to the abi.encode function call in the solidity contract +var contractDataABI = &abi.ParameterArray{ + {Name: "endorsementMode", Type: "string"}, + {Name: "notaryLocator", Type: "string"}, + {Name: "endorsementSetLocators", Type: "string[]"}, +} + +// golang struct to parse and serialize the data received from the block indexer when the base ledger factor contract +// emits a PaladinRegisterSmartContract_V0 event +// this must match the ABI above +type simpleTokenConfigParser struct { + EndorsementMode string `json:"endorsementMode"` + NotaryLocator string `json:"notaryLocator"` + EndorsementSet []string `json:"endorsementSetLocators"` +} + func SimpleTokenDomain(t *testing.T, ctx context.Context) plugintk.PluginBase { simpleDomainABI := mustParseBuildABI(simpleDomainBuild) simpleTokenABI := mustParseBuildABI(simpleTokenBuild) @@ -338,28 +357,11 @@ func SimpleTokenDomain(t *testing.T, ctx context.Context) plugintk.PluginBase { ] }` - // ABI for the config field in the PaladinRegisterSmartContract_V0 event - // this must match the type and order of arguments passed to the abi.encode function call in the solidity contract - contractDataABI := &abi.ParameterArray{ - {Name: "endorsementMode", Type: "string"}, - {Name: "notaryLocator", Type: "string"}, - {Name: "endorsementSetLocators", Type: "string[]"}, - } - - // golang struct to parse and serialize the data received from the block indexer when the base ledger factor contract - //emits a PaladinRegisterSmartContract_V0 event - // this must match the ABI above - type simpleTokenConfigParser struct { - EndorsementMode string `json:"endorsementMode"` - NotaryLocator string `json:"notaryLocator"` - EndorsementSet []string `json:"endorsementSetLocators"` - } - return plugintk.NewDomain(func(callbacks plugintk.DomainCallbacks) plugintk.DomainAPI { var simpleTokenSchemaID string var chainID int64 - simpleTokenSelection := func(ctx context.Context, stateQueryContext string, fromAddr *ethtypes.Address0xHex, amount *big.Int) ([]*simpleTokenParser, []*prototk.StateRef, *big.Int, error) { + simpleTokenSelection := func(ctx context.Context, stateQueryContext string, fromAddr *ethtypes.Address0xHex, amount *big.Int) ([]*simpleTokenParser, []*prototk.StateRef, *big.Int, string, error) { var lastStateTimestamp int64 total := big.NewInt(0) coins := []*simpleTokenParser{} @@ -388,18 +390,18 @@ func SimpleTokenDomain(t *testing.T, ctx context.Context) plugintk.PluginBase { QueryJson: tktypes.JSONString(jq).String(), }) if err != nil { - return nil, nil, nil, err + return nil, nil, nil, "", err } states := res.States if len(states) == 0 { - return nil, nil, nil, fmt.Errorf("%s: insufficient funds (available=%s)", SimpleDomainInsufficientFundsError, total.Text(10)) + return nil, nil, nil, fmt.Sprintf("%s: insufficient funds (available=%s)", SimpleDomainInsufficientFundsError, total.Text(10)), nil } for _, state := range states { lastStateTimestamp = state.CreatedAt // Note: More sophisticated coin selection might prefer states that aren't locked to a sequence var coin simpleTokenParser if err := json.Unmarshal([]byte(state.DataJson), &coin); err != nil { - return nil, nil, nil, fmt.Errorf("coin %s is invalid: %s", state.Id, err) + return nil, nil, nil, "", fmt.Errorf("coin %s is invalid: %s", state.Id, err) } total = total.Add(total, coin.Amount.BigInt()) stateRefs = append(stateRefs, &prototk.StateRef{ @@ -409,7 +411,7 @@ func SimpleTokenDomain(t *testing.T, ctx context.Context) plugintk.PluginBase { coins = append(coins, &coin) if total.Cmp(amount) >= 0 { // We've got what we need - return how much over we are - return coins, stateRefs, new(big.Int).Sub(total, amount), nil + return coins, stateRefs, new(big.Int).Sub(total, amount), "", nil } } } @@ -655,10 +657,11 @@ func SimpleTokenDomain(t *testing.T, ctx context.Context) plugintk.PluginBase { contractConfig.CoordinatorSelection = prototk.ContractConfig_COORDINATOR_SENDER contractConfig.SubmitterSelection = prototk.ContractConfig_SUBMITTER_SENDER } else if constructorParameters.EndorsementMode == NotaryEndorsement { - // TODO: Use static option? - contractConfig.CoordinatorSelection = prototk.ContractConfig_COORDINATOR_ENDORSER + contractConfig.CoordinatorSelection = prototk.ContractConfig_COORDINATOR_STATIC + contractConfig.StaticCoordinator = &constructorParameters.NotaryLocator contractConfig.SubmitterSelection = prototk.ContractConfig_SUBMITTER_COORDINATOR } else if constructorParameters.EndorsementMode == PrivacyGroupEndorsement { + //This combination is less common on a token based domain but may use it in some tests contractConfig.CoordinatorSelection = prototk.ContractConfig_COORDINATOR_ENDORSER contractConfig.SubmitterSelection = prototk.ContractConfig_SUBMITTER_COORDINATOR } else { @@ -734,12 +737,24 @@ func SimpleTokenDomain(t *testing.T, ctx context.Context) plugintk.PluginBase { toKeep := new(big.Int) coinsToSpend := []*simpleTokenParser{} stateRefsToSpend := []*prototk.StateRef{} + revertMessage := "" if txInputs.From != "" { - coinsToSpend, stateRefsToSpend, toKeep, err = simpleTokenSelection(ctx, req.StateQueryContext, fromAddr, amount) + coinsToSpend, stateRefsToSpend, toKeep, revertMessage, err = simpleTokenSelection(ctx, req.StateQueryContext, fromAddr, amount) if err != nil { return nil, err } + for _, state := range stateRefsToSpend { + log.L(ctx).Infof("Spend coin %s", state.Id) + } + } + if revertMessage != "" { + return &prototk.AssembleTransactionResponse{ + AssembledTransaction: &prototk.AssembledTransaction{}, + AssemblyResult: prototk.AssembleTransactionResponse_REVERT, + RevertReason: &revertMessage, + }, nil } + newStates := []*prototk.NewState{} newCoins := []*simpleTokenParser{} if fromAddr != nil && toKeep.Sign() > 0 { @@ -954,7 +969,8 @@ func SimpleTokenDomain(t *testing.T, ctx context.Context) plugintk.PluginBase { switch config.EndorsementMode { case SelfEndorsement: return &prototk.EndorseTransactionResponse{ - EndorsementResult: prototk.EndorseTransactionResponse_ENDORSER_SUBMIT, + EndorsementResult: prototk.EndorseTransactionResponse_SIGN, + Payload: []byte("i-hereby-endorse-this-transaction"), }, nil case NotaryEndorsement: return &prototk.EndorseTransactionResponse{ diff --git a/core/go/componenttest/domains/simple_storage_domain.go b/core/go/componenttest/domains/simple_storage_domain.go new file mode 100644 index 000000000..a5d9c3c64 --- /dev/null +++ b/core/go/componenttest/domains/simple_storage_domain.go @@ -0,0 +1,737 @@ +/* + * Copyright © 2024 Kaleido, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package domains + +import ( + "context" + _ "embed" + "encoding/json" + "fmt" + "strconv" + "testing" + + "github.com/hyperledger/firefly-signer/pkg/abi" + "github.com/hyperledger/firefly-signer/pkg/ethtypes" + "github.com/kaleido-io/paladin/config/pkg/confutil" + "github.com/kaleido-io/paladin/toolkit/pkg/algorithms" + "github.com/kaleido-io/paladin/toolkit/pkg/plugintk" + "github.com/kaleido-io/paladin/toolkit/pkg/prototk" + "github.com/kaleido-io/paladin/toolkit/pkg/query" + "github.com/kaleido-io/paladin/toolkit/pkg/signpayloads" + "github.com/kaleido-io/paladin/toolkit/pkg/tktypes" + "github.com/kaleido-io/paladin/toolkit/pkg/verifiers" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + SimpleStorageDomainStateNotAvailableError = "SDE0101" + SimpleStorageDomainTooManyStates = "SDE0102" +) + +// The set function sets ( or overwrites) the value for a given key in the map +// If no state for the given map name exists, this function will fail +// Key may be already present in the map, or not. +const simpleStorageSetABI = `{ + "type": "function", + "name": "set", + "inputs": [ + { + "name": "map", + "type": "string" + }, + { + "name": "key", + "type": "string" + }, + { + "name": "value", + "type": "string" + } + ], + "outputs": null + }` + +func SimpleStorageSetABI() *abi.ABI { + return &abi.ABI{mustParseABIEntry(simpleStorageSetABI)} +} + +const simpleStorageSetFunctionSignature = "function set(string memory map, string memory key, string memory value) external { }" + +// The init function initializes a new map +// this fails if there is already a state for the given map name +const simpleStorageInitABI = `{ + "type": "function", + "name": "init", + "inputs": [ + { + "name": "map", + "type": "string" + } + ], + "outputs": null +}` + +func SimpleStorageInitABI() *abi.ABI { + return &abi.ABI{mustParseABIEntry(simpleStorageInitABI)} +} + +const simpleStorageInitFunctionSignature = "function init(string memory map) external { }" + +// ABI used by paladin to parse the constructor parameters +// different for each endorsement mode +const simpleStorageConstructorABI = `{ + "type": "constructor", + "inputs": [ + { + "name": "endorsementSet", + "type": "string[]" + }, + { + "name": "from", + "type": "string" + }, + { + "name": "name", + "type": "string" + }, + { + "name": "endorsementMode", + "type": "string" + } + ], + "outputs": null +}` + +func SimpleStorageConstructorABI(endorsementMode string) *abi.ABI { + return &abi.ABI{mustParseABIEntry(simpleStorageConstructorABI)} +} + +// Go struct used in test (test + domain) to work with JSON structure passed into the paladin transaction for the constructor +// This is a union of the 3 ABI above +type SimpleStorageConstructorParameters struct { + EndorsementSet []string `json:"endorsementSet"` + From string `json:"from"` + Name string `json:"name"` + EndorsementMode string `json:"endorsementMode"` +} + +// Go struct used in test (test + domain) to work with JSON structure for `params` on the base ledger factory function +// This must match (including ordering of fields) the function signature for newSimpleTokenNotarized defined in the solidity contract +type SimpleStorageFactoryParameters struct { + TxId string `json:"txId"` + EndorsementMode string `json:"endorsementMode"` + NotaryLocator string `json:"notaryLocator"` + EndorsementSetLocators []string `json:"endorsementSetLocators"` +} + +// JSON structure passed into the paladin transaction for the storage set function +type simpleStorageSetParser struct { + Map string `json:"map"` + Key string `json:"key"` + Value string `json:"value"` +} + +// JSON structure passed into the paladin transaction for the storage init function +type simpleStorageInitParser struct { + Map string `json:"map"` +} + +// JSON structure for the state data +type StorageState struct { + Salt tktypes.HexBytes `json:"salt"` + Records string `json:"records"` //JSON string that can be parsed as a map of keys to values of StorageRecord + Map string `json:"map"` +} + +const simpleStorageStateSchema = `{ + "type": "tuple", + "internalType": "struct SimpleStorage", + "components": [ + { + "name": "map", + "type": "string", + "indexed": true + }, + { + "name": "salt", + "type": "bytes32" + }, + { + "name": "records", + "type": "string" + } + ] +}` + +// contract instance config( i.e. data field of event PaladinRegisterSmartContract_V0) +type simpleStorageConfigParser simpleTokenConfigParser // for now, we are re-sing the .sol from simpleDomain. + +// domain config +type SimpleStorageDomainConfig struct { + SubmitMode string `json:"submitMode"` + EndorsementMode string `json:"endorsementMode"` + EndorsementSet []string `json:"endorsementSet"` +} + +func SimpleStorageDomain(t *testing.T, ctx context.Context) plugintk.PluginBase { + simpleDomainABI := mustParseBuildABI(simpleDomainBuild) + simpleTokenABI := mustParseBuildABI(simpleTokenBuild) + + transferABI := simpleTokenABI.Events()["UTXOTransfer"] + require.NotEmpty(t, transferABI) + transferSignature := transferABI.SolString() + + return plugintk.NewDomain(func(callbacks plugintk.DomainCallbacks) plugintk.DomainAPI { + + var simpleStorageSchemaID string + getAllStates := func(ctx context.Context, stateQueryContext string, mapName string) ([]*prototk.StoredState, error) { + var lastStateTimestamp int64 + // There is only one state per map name per domain instance + jq := &query.QueryJSON{ + Limit: confutil.P(10), + Sort: []string{".created"}, + Statements: query.Statements{ + Ops: query.Ops{ + Eq: []*query.OpSingleVal{ + {Op: query.Op{Field: "map"}, Value: tktypes.JSONString(mapName)}, + }, + }, + }, + } + if lastStateTimestamp > 0 { + jq.GT = []*query.OpSingleVal{ + {Op: query.Op{Field: ".created"}, Value: tktypes.RawJSON(strconv.FormatInt(lastStateTimestamp, 10))}, + } + } + res, err := callbacks.FindAvailableStates(ctx, &prototk.FindAvailableStatesRequest{ + StateQueryContext: stateQueryContext, + SchemaId: simpleStorageSchemaID, + QueryJson: tktypes.JSONString(jq).String(), + }) + if err != nil { + return nil, err + } + + return res.States, nil + + } + simpleStorageSelection := func(ctx context.Context, stateQueryContext string, mapName string) (*StorageState, *prototk.StateRef, error) { + var lastStateTimestamp int64 + // There is only one state per domain instance + // (might be more realistic emulation of the Pente domain if we had one state per record / key name?) + jq := &query.QueryJSON{ + Limit: confutil.P(10), + Sort: []string{".created"}, + Statements: query.Statements{ + Ops: query.Ops{ + Eq: []*query.OpSingleVal{ + {Op: query.Op{Field: "map"}, Value: tktypes.JSONString(mapName)}, + }, + }, + }, + } + if lastStateTimestamp > 0 { + jq.GT = []*query.OpSingleVal{ + {Op: query.Op{Field: ".created"}, Value: tktypes.RawJSON(strconv.FormatInt(lastStateTimestamp, 10))}, + } + } + + res, err := callbacks.FindAvailableStates(ctx, &prototk.FindAvailableStatesRequest{ + StateQueryContext: stateQueryContext, + SchemaId: simpleStorageSchemaID, + QueryJson: tktypes.JSONString(jq).String(), + }) + if err != nil { + return nil, nil, err + } + states := res.States + if len(states) == 0 { + //assume this is the first transaction + return nil, nil, nil + + //return nil, nil, fmt.Errorf("%s: state not available", SimpleStorageDomainStateNotAvailableError) + } + if len(states) > 1 { + return nil, nil, fmt.Errorf("%s: too many states", SimpleStorageDomainTooManyStates) + } + state := states[0] + stateData := new(StorageState) + if err := json.Unmarshal([]byte(state.DataJson), stateData); err != nil { + return nil, nil, fmt.Errorf("stateData %s is invalid: %s", state.Id, err) + } + return stateData, + &prototk.StateRef{ + Id: state.Id, + SchemaId: state.SchemaId, + }, nil + } + + validateSimpleStorageSetInput := func(tx *prototk.TransactionSpecification) (*ethtypes.Address0xHex, simpleStorageConfigParser, *simpleStorageSetParser) { + //assert.JSONEq(t, simpleTokenTransferABI, tx.FunctionAbiJson) + //assert.Equal(t, "function transfer(string memory from, string memory to, uint256 amount) external { }", tx.FunctionSignature) + var inputs simpleStorageSetParser + err := json.Unmarshal([]byte(tx.FunctionParamsJson), &inputs) + require.NoError(t, err) + + contractAddr, err := ethtypes.NewAddress(tx.ContractInfo.ContractAddress) + require.NoError(t, err) + + require.NoError(t, err) + var config simpleStorageConfigParser + err = json.Unmarshal([]byte(tx.ContractInfo.ContractConfigJson), &config) + require.NoError(t, err) + + return contractAddr, config, &inputs + } + + validateSimpleStorageInitInput := func(tx *prototk.TransactionSpecification) (*ethtypes.Address0xHex, simpleStorageConfigParser, *simpleStorageInitParser) { + //assert.JSONEq(t, simpleTokenTransferABI, tx.FunctionAbiJson) + //assert.Equal(t, "function transfer(string memory from, string memory to, uint256 amount) external { }", tx.FunctionSignature) + var inputs simpleStorageInitParser + err := json.Unmarshal([]byte(tx.FunctionParamsJson), &inputs) + require.NoError(t, err) + + contractAddr, err := ethtypes.NewAddress(tx.ContractInfo.ContractAddress) + require.NoError(t, err) + + require.NoError(t, err) + var config simpleStorageConfigParser + err = json.Unmarshal([]byte(tx.ContractInfo.ContractConfigJson), &config) + require.NoError(t, err) + + return contractAddr, config, &inputs + } + + endorseInitTransaction := func(ctx context.Context, req *prototk.EndorseTransactionRequest) (*prototk.EndorseTransactionResponse, error) { + _, config, txInputs := validateSimpleStorageInitInput(req.Transaction) + + states, err := getAllStates(ctx, req.StateQueryContext, txInputs.Map) + if err != nil { + return nil, err + } + if len(states) > 0 { + // if there is only one state and it is the output of this transaction then it is valid to have no inputs + // otherwise it is an error + if len(states) == 1 && states[0].Id == req.Outputs[0].Id { + } else { + return &prototk.EndorseTransactionResponse{ + EndorsementResult: prototk.EndorseTransactionResponse_REVERT, + RevertReason: confutil.P("already initialised"), + }, nil + } + } + + if len(req.Outputs) != 1 { + return nil, fmt.Errorf("endorseSetTransaction: invalid number of outputs [%d]", len(req.Outputs)) + } + + if len(req.Inputs) == 1 { + inputState := &StorageState{} + if err := json.Unmarshal([]byte(req.Inputs[0].StateDataJson), &inputState); err != nil { + return nil, fmt.Errorf("endorseSetTransaction: invalid input (%s): %s", req.Inputs[0].Id, err) + } + assert.Equal(t, simpleStorageSchemaID, req.Inputs[0].SchemaId) + + } + + outputState := &StorageState{} + //TODO should validate that the diffs between inputState and outputState match the txInputs + + if err := json.Unmarshal([]byte(req.Outputs[0].StateDataJson), &outputState); err != nil { + return nil, fmt.Errorf("invalid output (%s): %s", req.Outputs[0].Id, err) + } + assert.Equal(t, simpleStorageSchemaID, req.Outputs[0].SchemaId) + + switch config.EndorsementMode { + case PrivacyGroupEndorsement: + return &prototk.EndorseTransactionResponse{ + EndorsementResult: prototk.EndorseTransactionResponse_SIGN, + Payload: tktypes.RandBytes(32), + }, nil + default: + return nil, fmt.Errorf("unsupported endorsement mode: %s", config.EndorsementMode) + + } + + } + + endorseSetTransaction := func(_ context.Context, req *prototk.EndorseTransactionRequest) (*prototk.EndorseTransactionResponse, error) { + _, config, _ := validateSimpleStorageSetInput(req.Transaction) + + //notaryLocator := config.NotaryLocator + //senderAddr, fromAddr, toAddr := extractTransferVerifiers(req.Transaction, txInputs, req.ResolvedVerifiers) + //assert.Equal(t, req.EndorsementVerifier.Lookup, req.EndorsementRequest.Parties[0]) + //assert.Equal(t, req.EndorsementVerifier.Lookup, notaryLocator) + if len(req.Inputs) != 1 { + return nil, fmt.Errorf("endorseSetTransaction: invalid number of inputs [%d]", len(req.Inputs)) + } + if len(req.Outputs) != 1 { + return nil, fmt.Errorf("endorseSetTransaction: invalid number of outputs [%d]", len(req.Outputs)) + } + + inputState := &StorageState{} + if err := json.Unmarshal([]byte(req.Inputs[0].StateDataJson), &inputState); err != nil { + return nil, fmt.Errorf("invalid input (%s): %s", req.Inputs[0].Id, err) + } + assert.Equal(t, simpleStorageSchemaID, req.Inputs[0].SchemaId) + + outputState := &StorageState{} + //TODO should validate that the diffs between inputState and outputState match the txInputs + + if err := json.Unmarshal([]byte(req.Outputs[0].StateDataJson), &outputState); err != nil { + return nil, fmt.Errorf("invalid output (%s): %s", req.Outputs[0].Id, err) + } + assert.Equal(t, simpleStorageSchemaID, req.Outputs[0].SchemaId) + + switch config.EndorsementMode { + case PrivacyGroupEndorsement: + return &prototk.EndorseTransactionResponse{ + EndorsementResult: prototk.EndorseTransactionResponse_SIGN, + Payload: tktypes.RandBytes(32), + }, nil + default: + return nil, fmt.Errorf("unsupported endorsement mode: %s", config.EndorsementMode) + } + } + + return &plugintk.DomainAPIBase{Functions: &plugintk.DomainAPIFunctions{ + + ConfigureDomain: func(ctx context.Context, req *prototk.ConfigureDomainRequest) (*prototk.ConfigureDomainResponse, error) { + assert.Equal(t, "simpleStorageDomain", req.Name) + domainConfig := &SimpleStorageDomainConfig{} + err := json.Unmarshal([]byte(req.ConfigJson), domainConfig) + require.NoError(t, err) + + var eventsABI abi.ABI + eventsABI = append(eventsABI, transferABI) + eventsJSON, err := json.Marshal(eventsABI) + require.NoError(t, err) + + return &prototk.ConfigureDomainResponse{ + DomainConfig: &prototk.DomainConfig{ + AbiStateSchemasJson: []string{simpleStorageStateSchema}, + AbiEventsJson: string(eventsJSON), + }, + }, nil + }, + + InitDomain: func(ctx context.Context, req *prototk.InitDomainRequest) (*prototk.InitDomainResponse, error) { + assert.Len(t, req.AbiStateSchemas, 1) + simpleStorageSchemaID = req.AbiStateSchemas[0].Id + assert.Equal(t, "type=SimpleStorage(string map,bytes32 salt,string records),labels=[map]", req.AbiStateSchemas[0].Signature) + return &prototk.InitDomainResponse{}, nil + }, + + InitDeploy: func(ctx context.Context, req *prototk.InitDeployRequest) (*prototk.InitDeployResponse, error) { + constructorParameters := &SimpleStorageConstructorParameters{} + err := json.Unmarshal([]byte(req.Transaction.ConstructorParamsJson), constructorParameters) + require.NoError(t, err) + + switch constructorParameters.EndorsementMode { + case PrivacyGroupEndorsement: + requiredVerifiers := make([]*prototk.ResolveVerifierRequest, len(constructorParameters.EndorsementSet)) + for i, v := range constructorParameters.EndorsementSet { + requiredVerifiers[i] = &prototk.ResolveVerifierRequest{ + Lookup: v, + Algorithm: algorithms.ECDSA_SECP256K1, + VerifierType: verifiers.ETH_ADDRESS, + } + } + return &prototk.InitDeployResponse{ + RequiredVerifiers: requiredVerifiers, + }, nil + } + return nil, fmt.Errorf("unknown endorsement mode %s", constructorParameters.EndorsementMode) + }, + + PrepareDeploy: func(ctx context.Context, req *prototk.PrepareDeployRequest) (*prototk.PrepareDeployResponse, error) { + /*assert.JSONEq(t, `{ + "notary": "domain1.contract1.notary", + "name": "FakeToken1", + "symbol": "FT1", + "endorsementMode": "NoEndorsement" + }`, req.Transaction.ConstructorParamsJson)*/ + constructorParameters := &SimpleStorageConstructorParameters{} + err := json.Unmarshal([]byte(req.Transaction.ConstructorParamsJson), constructorParameters) + require.NoError(t, err) + + switch constructorParameters.EndorsementMode { + case PrivacyGroupEndorsement: + assert.Len(t, req.ResolvedVerifiers, len(constructorParameters.EndorsementSet)) + // We don't know that the order of the ResolvedVerifiers will match the order of the endorsement set, + // so we just check that they are all there + for _, v := range constructorParameters.EndorsementSet { + found := false + for j := range req.ResolvedVerifiers { + if req.ResolvedVerifiers[j].Lookup == v { + assert.Equal(t, algorithms.ECDSA_SECP256K1, req.ResolvedVerifiers[j].Algorithm) + assert.Equal(t, verifiers.ETH_ADDRESS, req.ResolvedVerifiers[j].VerifierType) + assert.Equal(t, v, req.ResolvedVerifiers[j].Lookup) + assert.NotEmpty(t, req.ResolvedVerifiers[j].Verifier) + found = true + } + } + assert.True(t, found, "endorser %s not found in ResolvedVerifiers", v) + } + } + if constructorParameters.EndorsementSet == nil { + constructorParameters.EndorsementSet = []string{} + } + params := FactoryParameters{ + TxId: req.Transaction.TransactionId, + EndorsementSetLocators: constructorParameters.EndorsementSet, + EndorsementMode: constructorParameters.EndorsementMode, + } + return &prototk.PrepareDeployResponse{ + Signer: confutil.P(fmt.Sprintf("domain1.transactions.%s", req.Transaction.TransactionId)), + Transaction: &prototk.PreparedTransaction{ + FunctionAbiJson: toJSONString(t, simpleDomainABI.Functions()["newSimpleTokenNotarized"]), + ParamsJson: tktypes.JSONString(params).String(), + }, + }, nil + }, + + InitContract: func(ctx context.Context, icr *prototk.InitContractRequest) (*prototk.InitContractResponse, error) { + + configValues, err := contractDataABI.DecodeABIData(icr.ContractConfig, 0) + str := tktypes.HexBytes(icr.ContractConfig).HexString0xPrefix() + assert.NotEqual(t, "", str) + require.NoError(t, err) + + configJSON, err := tktypes.StandardABISerializer().SerializeJSON(configValues) + require.NoError(t, err) + contractConfig := &prototk.ContractConfig{ + ContractConfigJson: string(configJSON), + } + var constructorParameters SimpleStorageConstructorParameters + err = json.Unmarshal([]byte(configJSON), &constructorParameters) + require.NoError(t, err) + + if constructorParameters.EndorsementMode == SelfEndorsement { + contractConfig.CoordinatorSelection = prototk.ContractConfig_COORDINATOR_SENDER + contractConfig.SubmitterSelection = prototk.ContractConfig_SUBMITTER_SENDER + } else if constructorParameters.EndorsementMode == PrivacyGroupEndorsement { + contractConfig.CoordinatorSelection = prototk.ContractConfig_COORDINATOR_ENDORSER + contractConfig.SubmitterSelection = prototk.ContractConfig_SUBMITTER_COORDINATOR + } else { + return nil, fmt.Errorf("unknown endorsement mode %s", constructorParameters.EndorsementMode) + } + + return &prototk.InitContractResponse{ + Valid: true, + ContractConfig: contractConfig, + }, nil + }, + + InitTransaction: func(ctx context.Context, req *prototk.InitTransactionRequest) (*prototk.InitTransactionResponse, error) { + var config simpleStorageConfigParser + switch req.Transaction.FunctionSignature { + case simpleStorageInitFunctionSignature: + _, config, _ = validateSimpleStorageInitInput(req.Transaction) + case simpleStorageSetFunctionSignature: + _, config, _ = validateSimpleStorageSetInput(req.Transaction) + default: + return nil, fmt.Errorf("unknown function signature %s", req.Transaction.FunctionSignature) + + } + + requiredVerifiers := []*prototk.ResolveVerifierRequest{ + { + Lookup: req.Transaction.From, + Algorithm: algorithms.ECDSA_SECP256K1, + VerifierType: verifiers.ETH_ADDRESS, + }, + } + + switch config.EndorsementMode { + + case PrivacyGroupEndorsement: + for _, v := range config.EndorsementSet { + requiredVerifiers = append(requiredVerifiers, &prototk.ResolveVerifierRequest{ + Lookup: v, + Algorithm: algorithms.ECDSA_SECP256K1, + VerifierType: verifiers.ETH_ADDRESS, + }) + } + default: + return nil, fmt.Errorf("unknown endorsement mode %s", config.EndorsementMode) + } + + return &prototk.InitTransactionResponse{ + RequiredVerifiers: requiredVerifiers, + }, nil + }, + + AssembleTransaction: func(ctx context.Context, req *prototk.AssembleTransactionRequest) (_ *prototk.AssembleTransactionResponse, err error) { + + var config simpleStorageConfigParser + storage := make(map[string]string) + stateRefsToSpend := make([]*prototk.StateRef, 0) + var mapName string + switch req.Transaction.FunctionSignature { + case simpleStorageInitFunctionSignature: + _, c, txInputs := validateSimpleStorageInitInput(req.Transaction) + config = c + mapName = txInputs.Map + + existingState, _, err := simpleStorageSelection(ctx, req.StateQueryContext, mapName) + if err != nil { + return nil, err + } + if existingState != nil { + return nil, fmt.Errorf("state already exists") + } + + case simpleStorageSetFunctionSignature: + _, c, txInputs := validateSimpleStorageSetInput(req.Transaction) + config = c + mapName = txInputs.Map + + stateToSpend, stateRefToSpend, err := simpleStorageSelection(ctx, req.StateQueryContext, mapName) + if err != nil { + return nil, err + } + + if stateToSpend == nil { + return nil, fmt.Errorf("no state available for map %s ", mapName) + } + + stateRefsToSpend = append(stateRefsToSpend, stateRefToSpend) + if err := json.Unmarshal([]byte(stateToSpend.Records), &storage); err != nil { + return nil, fmt.Errorf("invalid state data: %s", err) + } + + storage[txInputs.Key] = txInputs.Value + default: + return nil, fmt.Errorf("unknown function signature %s", req.Transaction.FunctionSignature) + + } + + newStateData := &StorageState{ + Records: toJSONString(t, storage), + Salt: tktypes.RandBytes(32), + Map: mapName, + } + newState := &prototk.NewState{ + SchemaId: simpleStorageSchemaID, + StateDataJson: toJSONString(t, newStateData), + DistributionList: config.EndorsementSet, + } + + //eip712Payload, err := typedDataV4TransferWithSalts(contractAddr, coinsToSpend, newCoins) + //require.NoError(t, err) + eip712Payload := tktypes.RandBytes(32) + + switch config.EndorsementMode { + + case PrivacyGroupEndorsement: + + return &prototk.AssembleTransactionResponse{ + AssembledTransaction: &prototk.AssembledTransaction{ + InputStates: stateRefsToSpend, + OutputStates: []*prototk.NewState{newState}, + }, + AssemblyResult: prototk.AssembleTransactionResponse_OK, + AttestationPlan: []*prototk.AttestationRequest{ + { + Name: "sender", + AttestationType: prototk.AttestationType_SIGN, + Algorithm: algorithms.ECDSA_SECP256K1, + VerifierType: verifiers.ETH_ADDRESS, + PayloadType: signpayloads.OPAQUE_TO_RSV, + Payload: eip712Payload, + Parties: []string{ + req.Transaction.From, + }, + }, + { + Name: "privacyGroup", + AttestationType: prototk.AttestationType_ENDORSE, + // we expect an endorsement is of the form ENDORSER_SUBMIT - so we need an eth signing key to exist + Algorithm: algorithms.ECDSA_SECP256K1, + VerifierType: verifiers.ETH_ADDRESS, + PayloadType: signpayloads.OPAQUE_TO_RSV, + Parties: config.EndorsementSet, + }, + }, + }, nil + default: + return nil, fmt.Errorf("unsupported endorsement mode: %s", config.EndorsementMode) + } + }, + + EndorseTransaction: func(ctx context.Context, req *prototk.EndorseTransactionRequest) (*prototk.EndorseTransactionResponse, error) { + + switch req.Transaction.FunctionSignature { + case simpleStorageInitFunctionSignature: + return endorseInitTransaction(ctx, req) + case simpleStorageSetFunctionSignature: + return endorseSetTransaction(ctx, req) + default: + return nil, fmt.Errorf("unknown function signature %s", req.Transaction.FunctionSignature) + + } + }, + + PrepareTransaction: func(ctx context.Context, req *prototk.PrepareTransactionRequest) (*prototk.PrepareTransactionResponse, error) { + var signerSignature tktypes.HexBytes + for _, att := range req.AttestationResult { + if att.AttestationType == prototk.AttestationType_SIGN && att.Name == "sender" { + signerSignature = att.Payload + } + } + spentStateIds := make([]string, len(req.InputStates)) + for i, s := range req.InputStates { + spentStateIds[i] = s.Id + } + newStateIds := make([]string, len(req.OutputStates)) + for i, s := range req.OutputStates { + newStateIds[i] = s.Id + } + return &prototk.PrepareTransactionResponse{ + Transaction: &prototk.PreparedTransaction{ + FunctionAbiJson: toJSONString(t, simpleTokenABI.Functions()["executeNotarized"]), + ParamsJson: toJSONString(t, map[string]interface{}{ + "txId": req.Transaction.TransactionId, + "inputs": spentStateIds, + "outputs": newStateIds, + "signature": signerSignature, + }), + }, + }, nil + }, + + HandleEventBatch: func(ctx context.Context, req *prototk.HandleEventBatchRequest) (*prototk.HandleEventBatchResponse, error) { + var res prototk.HandleEventBatchResponse + for _, ev := range req.Events { + switch ev.SoliditySignature { + case transferSignature: + var transfer UTXOTransfer_Event + if err := json.Unmarshal([]byte(ev.DataJson), &transfer); err == nil { + res.TransactionsComplete = append(res.TransactionsComplete, &prototk.CompletedTransaction{ + TransactionId: transfer.TX.String(), + Location: ev.Location, + }) + res.SpentStates = append(res.SpentStates, parseStatesFromEvent(transfer.TX, transfer.Inputs)...) + res.ConfirmedStates = append(res.ConfirmedStates, parseStatesFromEvent(transfer.TX, transfer.Outputs)...) + } + } + } + return &res, nil + }, + }} + }) +} diff --git a/core/go/componenttest/utils_for_test.go b/core/go/componenttest/utils_for_test.go index b546b62df..634698d31 100644 --- a/core/go/componenttest/utils_for_test.go +++ b/core/go/componenttest/utils_for_test.go @@ -84,7 +84,7 @@ func transactionLatencyThreshold(t *testing.T) time.Duration { deadline, ok := t.Deadline() if !ok { - //there was no -timeout flag, default to a long time becuase this is most likely a debug session + //there was no -timeout flag, default to a long time because this is most likely a debug session threshold = time.Hour } else { timeRemaining := time.Until(deadline) @@ -165,34 +165,7 @@ func newNodeConfiguration(t *testing.T, nodeName string) *nodeConfiguration { } } -func initPostgres(t *testing.T, ctx context.Context) (dns string, cleanup func()) { - dbDSN := func(dbname string) string { - return fmt.Sprintf("postgres://postgres:my-secret@localhost:5432/%s?sslmode=disable", dbname) - } - componentTestdbName := "ct_" + uuid.New().String() - log.L(ctx).Infof("Component test Postgres DB: %s", componentTestdbName) - - // First create the database - using the super user - - adminDB, err := sql.Open("postgres", dbDSN("postgres")) - if err == nil { - _, err = adminDB.Exec(fmt.Sprintf(`CREATE DATABASE "%s";`, componentTestdbName)) - } - if err == nil { - err = adminDB.Close() - } - require.NoError(t, err) - - return dbDSN(componentTestdbName), func() { - adminDB, err := sql.Open("postgres", dbDSN("postgres")) - if err == nil { - _, _ = adminDB.Exec(fmt.Sprintf(`DROP DATABASE "%s" WITH(FORCE);`, componentTestdbName)) - adminDB.Close() - } - } -} - -func newInstanceForComponentTesting(t *testing.T, domainRegistryAddress *tktypes.EthAddress, binding *nodeConfiguration, peerNodes []*nodeConfiguration, domainConfig *domains.SimpleDomainConfig) *componentTestInstance { +func newInstanceForComponentTesting(t *testing.T, domainRegistryAddress *tktypes.EthAddress, binding *nodeConfiguration, peerNodes []*nodeConfiguration, domainConfig interface{}) *componentTestInstance { if binding == nil { binding = newNodeConfiguration(t, "default") } @@ -223,14 +196,36 @@ func newInstanceForComponentTesting(t *testing.T, domainRegistryAddress *tktypes SubmitMode: domains.ENDORSER_SUBMISSION, } } - i.conf.DomainManagerConfig.Domains["domain1"] = &pldconf.DomainConfig{ - AllowSigning: true, - Plugin: pldconf.PluginConfig{ - Type: string(tktypes.LibraryTypeCShared), - Library: "loaded/via/unit/test/loader", - }, - Config: map[string]any{"submitMode": domainConfig.SubmitMode}, - RegistryAddress: domainRegistryAddress.String(), + switch domainConfig := domainConfig.(type) { + case *domains.SimpleDomainConfig: + i.conf.DomainManagerConfig.Domains["domain1"] = &pldconf.DomainConfig{ + AllowSigning: true, + Plugin: pldconf.PluginConfig{ + Type: string(tktypes.LibraryTypeCShared), + Library: "loaded/via/unit/test/loader", + }, + Config: map[string]any{"submitMode": domainConfig.SubmitMode}, + RegistryAddress: domainRegistryAddress.String(), + } + case *domains.SimpleStorageDomainConfig: + endorsementSet := make([]string, 1+len(peerNodes)) + endorsementSet[0] = binding.name + for i, peerNode := range peerNodes { + endorsementSet[i+1] = peerNode.name + } + i.conf.DomainManagerConfig.Domains["simpleStorageDomain"] = &pldconf.DomainConfig{ + AllowSigning: true, + Plugin: pldconf.PluginConfig{ + Type: string(tktypes.LibraryTypeCShared), + Library: "loaded/via/unit/test/loader", + }, + Config: map[string]any{ + "submitMode": domainConfig.SubmitMode, + "endorsementSet": endorsementSet, + }, + RegistryAddress: domainRegistryAddress.String(), + } + } i.conf.NodeName = binding.name @@ -297,6 +292,7 @@ func newInstanceForComponentTesting(t *testing.T, domainRegistryAddress *tktypes dns, cleanUp := initPostgres(t, context.Background()) i.conf.DB.Postgres.DSN = dns t.Cleanup(cleanUp) + } var pl plugins.UnitTestPluginLoader @@ -310,9 +306,10 @@ func newInstanceForComponentTesting(t *testing.T, domainRegistryAddress *tktypes require.NoError(t, err) loaderMap := map[string]plugintk.Plugin{ - "domain1": domains.SimpleTokenDomain(t, i.ctx), - "grpc": grpc.NewPlugin(i.ctx), - "registry1": static.NewPlugin(i.ctx), + "domain1": domains.SimpleTokenDomain(t, i.ctx), + "simpleStorageDomain": domains.SimpleStorageDomain(t, i.ctx), + "grpc": grpc.NewPlugin(i.ctx), + "registry1": static.NewPlugin(i.ctx), } pc := i.cm.PluginManager() pl, err = plugins.NewUnitTestPluginLoader(pc.GRPCTargetURL(), pc.LoaderID().String(), loaderMap) @@ -343,6 +340,33 @@ func newInstanceForComponentTesting(t *testing.T, domainRegistryAddress *tktypes } +func initPostgres(t *testing.T, ctx context.Context) (dns string, cleanup func()) { + dbDSN := func(dbname string) string { + return fmt.Sprintf("postgres://postgres:my-secret@localhost:5432/%s?sslmode=disable", dbname) + } + componentTestdbName := "ct_" + uuid.New().String() + log.L(ctx).Infof("Component test Postgres DB: %s", componentTestdbName) + + // First create the database - using the super user + + adminDB, err := sql.Open("postgres", dbDSN("postgres")) + if err == nil { + _, err = adminDB.Exec(fmt.Sprintf(`CREATE DATABASE "%s";`, componentTestdbName)) + } + if err == nil { + err = adminDB.Close() + } + require.NoError(t, err) + + return dbDSN(componentTestdbName), func() { + adminDB, err := sql.Open("postgres", dbDSN("postgres")) + if err == nil { + _, _ = adminDB.Exec(fmt.Sprintf(`DROP DATABASE "%s" WITH(FORCE);`, componentTestdbName)) + adminDB.Close() + } + } +} + func testConfig(t *testing.T) pldconf.PaladinConfig { ctx := context.Background() @@ -351,7 +375,9 @@ func testConfig(t *testing.T) pldconf.PaladinConfig { assert.NoError(t, err) // For running in this unit test the dirs are different to the sample config + // conf.DB.SQLite.DebugQueries = true conf.DB.SQLite.MigrationsDir = "../db/migrations/sqlite" + // conf.DB.Postgres.DebugQueries = true conf.DB.Postgres.MigrationsDir = "../db/migrations/postgres" port, err := getFreePort() @@ -448,8 +474,8 @@ func (p *partyForTesting) peer(peers ...*nodeConfiguration) { p.peers = append(p.peers, peers...) } -func (p *partyForTesting) start(t *testing.T, domainConfig domains.SimpleDomainConfig) { - p.instance = newInstanceForComponentTesting(t, p.domainRegistryAddress, p.nodeConfig, p.peers, &domainConfig) +func (p *partyForTesting) start(t *testing.T, domainConfig interface{}) { + p.instance = newInstanceForComponentTesting(t, p.domainRegistryAddress, p.nodeConfig, p.peers, domainConfig) p.client = p.instance.client } @@ -483,3 +509,33 @@ func (p *partyForTesting) deploySimpleDomainInstanceContract(t *testing.T, endor require.NotNil(t, dplyTxFull.Receipt.ContractAddress) return dplyTxFull.Receipt.ContractAddress } + +func (p *partyForTesting) deploySimpleStorageDomainInstanceContract(t *testing.T, endorsementMode string, constructorParameters *domains.SimpleStorageConstructorParameters) *tktypes.EthAddress { + + var dplyTxID uuid.UUID + + err := p.client.CallRPC(context.Background(), &dplyTxID, "ptx_sendTransaction", &pldapi.TransactionInput{ + ABI: *domains.SimpleStorageConstructorABI(endorsementMode), + TransactionBase: pldapi.TransactionBase{ + Type: pldapi.TransactionTypePrivate.Enum(), + Domain: "simpleStorageDomain", + From: p.identity, + Data: tktypes.JSONString(constructorParameters), + }, + }) + require.NoError(t, err) + assert.Eventually(t, + transactionReceiptCondition(t, context.Background(), dplyTxID, p.client, true), + transactionLatencyThreshold(t)+5*time.Second, //TODO deploy transaction seems to take longer than expected + 100*time.Millisecond, + "Deploy transaction did not receive a receipt", + ) + + var dplyTxFull pldapi.TransactionFull + err = p.client.CallRPC(context.Background(), &dplyTxFull, "ptx_getTransactionFull", dplyTxID) + require.NoError(t, err) + require.NotNil(t, dplyTxFull.Receipt) + require.True(t, dplyTxFull.Receipt.Success) + require.NotNil(t, dplyTxFull.Receipt.ContractAddress) + return dplyTxFull.Receipt.ContractAddress +} diff --git a/core/go/db/migrations/postgres/000008_create_private_transaction_tables.down.sql b/core/go/db/migrations/postgres/000008_create_private_transaction_tables.down.sql index 18466bfe3..dcfd2c6d2 100644 --- a/core/go/db/migrations/postgres/000008_create_private_transaction_tables.down.sql +++ b/core/go/db/migrations/postgres/000008_create_private_transaction_tables.down.sql @@ -2,7 +2,5 @@ BEGIN; DROP TABLE dispatches; DROP TABLE state_distribution_acknowledgments; DROP TABLE state_distributions; -DROP TABLE transaction_delegation_acknowledgements; -DROP TABLE transaction_delegations; COMMIT; diff --git a/core/go/db/migrations/postgres/000008_create_private_transaction_tables.up.sql b/core/go/db/migrations/postgres/000008_create_private_transaction_tables.up.sql index 54b334b82..686ad0308 100644 --- a/core/go/db/migrations/postgres/000008_create_private_transaction_tables.up.sql +++ b/core/go/db/migrations/postgres/000008_create_private_transaction_tables.up.sql @@ -2,7 +2,7 @@ BEGIN; CREATE TABLE dispatches ( "public_transaction_address" TEXT NOT NULL, - "public_transaction_nonce" BIGINT NOT NULL, + "public_transaction_nonce" BIGINT NOT NULL, -- moved to public_transaction_id in 13 "private_transaction_id" TEXT NOT NULL, "id" TEXT NOT NULL, PRIMARY KEY ("id") @@ -36,21 +36,3 @@ CREATE TABLE state_distribution_acknowledgments ( CREATE UNIQUE INDEX state_distribution_acknowledgments_state_distribution ON state_distribution_acknowledgments("state_distribution"); -CREATE TABLE transaction_delegations ( - "id" UUID NOT NULL, - "transaction_id" UUID NOT NULL, - "delegate_node_id" TEXT NOT NULL, - PRIMARY KEY ("id") - -- need to reorder the migrations before we can define a foreign key to the transactions table FOREIGN KEY ("transaction_id") REFERENCES transactions ("id") ON DELETE CASCADE -); - -CREATE TABLE transaction_delegation_acknowledgements ( - "delegation" UUID NOT NULL, - "id" UUID NOT NULL, - PRIMARY KEY ("id"), - FOREIGN KEY ("delegation") REFERENCES transaction_delegations ("id") ON DELETE CASCADE -); -CREATE UNIQUE INDEX transaction_delegation_acknowledgements_delegation ON transaction_delegation_acknowledgements("delegation"); - -COMMIT; - diff --git a/core/go/db/migrations/postgres/000012_create_prepared_tx_tables.down.sql b/core/go/db/migrations/postgres/000012_create_prepared_tx_tables.down.sql index ea1ce8fd9..23a2858f0 100644 --- a/core/go/db/migrations/postgres/000012_create_prepared_tx_tables.down.sql +++ b/core/go/db/migrations/postgres/000012_create_prepared_tx_tables.down.sql @@ -1,6 +1,6 @@ BEGIN; DROP TABLE prepared_txn_states; -DROP TABLE prepared_txns; DROP TABLE prepared_txn_distribution_acknowledgments; DROP TABLE prepared_txn_distributions; +DROP TABLE prepared_txns; COMMIT; \ No newline at end of file diff --git a/core/go/db/migrations/postgres/000013_delayed_nonce_allocation_updates.down.sql b/core/go/db/migrations/postgres/000013_delayed_nonce_allocation_updates.down.sql new file mode 100644 index 000000000..15db4b209 --- /dev/null +++ b/core/go/db/migrations/postgres/000013_delayed_nonce_allocation_updates.down.sql @@ -0,0 +1,5 @@ +BEGIN; + +-- There is no ability to re-instate the signer_nonce column (and remove the pub_txn_id column) via a down migration + +COMMIT; \ No newline at end of file diff --git a/core/go/db/migrations/postgres/000013_delayed_nonce_allocation_updates.up.sql b/core/go/db/migrations/postgres/000013_delayed_nonce_allocation_updates.up.sql new file mode 100644 index 000000000..63dd59726 --- /dev/null +++ b/core/go/db/migrations/postgres/000013_delayed_nonce_allocation_updates.up.sql @@ -0,0 +1,66 @@ +BEGIN; + +-- Generate an identity column for the public_txns table +ALTER TABLE public_txns ADD "pub_txn_id" BIGINT GENERATED ALWAYS AS IDENTITY; + +-- And add the foreign relationship to the other tables as an empty column +ALTER TABLE public_submissions ADD "pub_txn_id" BIGINT; +CREATE UNIQUE INDEX public_submissions_pub_txn_id on public_submissions("pub_txn_id"); +ALTER TABLE public_completions ADD "pub_txn_id" BIGINT; +CREATE UNIQUE INDEX public_completions_pub_txn_id on public_completions("pub_txn_id"); +ALTER TABLE public_txn_bindings ADD "pub_txn_id" BIGINT; +CREATE UNIQUE INDEX public_txn_bindings_pub_txn_id on public_txn_bindings("pub_txn_id"); + +-- Add new new reference column to the public_submissions +UPDATE public_submissions +SET "pub_txn_id" = updates."pub_txn_id" +FROM ( SELECT "signer_nonce", "pub_txn_id" FROM public_txns ) AS updates +WHERE ( public_submissions."signer_nonce" = updates."signer_nonce" ); + +-- Add new new reference column to the public_completions +UPDATE public_completions +SET "pub_txn_id" = updates."pub_txn_id" +FROM ( SELECT "signer_nonce", "pub_txn_id" FROM public_txns ) AS updates +WHERE ( public_completions."signer_nonce" = updates."signer_nonce" ); + +-- Add new new reference column to the public_txn_bindings +UPDATE public_txn_bindings +SET "pub_txn_id" = updates."pub_txn_id" +FROM ( SELECT "signer_nonce", "pub_txn_id" FROM public_txns ) AS updates +WHERE ( public_txn_bindings."signer_nonce" = updates."signer_nonce" ); + +-- Drop the old references +ALTER TABLE public_submissions DROP CONSTRAINT public_submissions_signer_nonce_fkey; +ALTER TABLE public_completions DROP CONSTRAINT public_completions_signer_nonce_fkey; +ALTER TABLE public_txn_bindings DROP CONSTRAINT public_txn_bindings_signer_nonce_fkey; +ALTER TABLE public_submissions DROP COLUMN "signer_nonce"; +ALTER TABLE public_completions DROP COLUMN "signer_nonce"; +ALTER TABLE public_txn_bindings DROP COLUMN "signer_nonce"; +ALTER TABLE public_txns DROP COLUMN "signer_nonce"; + +-- Make the new column the primary key +ALTER TABLE public_txns ADD CONSTRAINT public_txns_pkey PRIMARY KEY ("pub_txn_id"); + +-- Update the submissions/completions tables to lock in the new reference +ALTER TABLE public_submissions ALTER COLUMN "pub_txn_id" SET NOT NULL; +ALTER TABLE public_submissions ADD CONSTRAINT public_submissions_pub_txn_id_fkey FOREIGN KEY ("pub_txn_id") REFERENCES public_txns ("pub_txn_id") ON DELETE CASCADE; +ALTER TABLE public_completions ALTER COLUMN "pub_txn_id" SET NOT NULL; +ALTER TABLE public_completions ADD CONSTRAINT public_completions_pub_txn_id_fkey FOREIGN KEY ("pub_txn_id") REFERENCES public_txns ("pub_txn_id") ON DELETE CASCADE; +ALTER TABLE public_txn_bindings ALTER COLUMN "pub_txn_id" SET NOT NULL; +ALTER TABLE public_txn_bindings ADD CONSTRAINT public_txn_bindings_pub_txn_id_fkey FOREIGN KEY ("pub_txn_id") REFERENCES public_txns ("pub_txn_id") ON DELETE CASCADE; + +-- Now we can make the nonce optional on the public_txns +ALTER TABLE public_txns ALTER COLUMN "nonce" DROP NOT NULL; + +-- We also need to update the dispatches table +ALTER TABLE dispatches ADD "public_transaction_id" BIGINT; +UPDATE dispatches +SET "public_transaction_id" = updates."pub_txn_id" +FROM ( SELECT "nonce", "from", "pub_txn_id" FROM public_txns ) AS updates +WHERE dispatches."public_transaction_address" = updates."from" AND dispatches."public_transaction_nonce" = updates."nonce"; +DROP INDEX dispatches_public_private; +ALTER TABLE dispatches DROP COLUMN "public_transaction_nonce"; +ALTER TABLE dispatches ALTER COLUMN "public_transaction_id" SET NOT NULL; +CREATE UNIQUE INDEX dispatches_public_private ON dispatches("public_transaction_address","public_transaction_id","private_transaction_id"); + +COMMIT; \ No newline at end of file diff --git a/core/go/db/migrations/sqlite/000008_create_private_transaction_tables.down.sql b/core/go/db/migrations/sqlite/000008_create_private_transaction_tables.down.sql index 04e6989b1..7f7aaa276 100644 --- a/core/go/db/migrations/sqlite/000008_create_private_transaction_tables.down.sql +++ b/core/go/db/migrations/sqlite/000008_create_private_transaction_tables.down.sql @@ -1,5 +1,3 @@ DROP TABLE dispatches; DROP TABLE state_distribution_acknowledgments; DROP TABLE state_distributions; -DROP TABLE transaction_delegation_acknowledgements; -DROP TABLE transaction_delegations; diff --git a/core/go/db/migrations/sqlite/000008_create_private_transaction_tables.up.sql b/core/go/db/migrations/sqlite/000008_create_private_transaction_tables.up.sql index 8e2077cdf..a78841ec5 100644 --- a/core/go/db/migrations/sqlite/000008_create_private_transaction_tables.up.sql +++ b/core/go/db/migrations/sqlite/000008_create_private_transaction_tables.up.sql @@ -1,6 +1,6 @@ CREATE TABLE dispatches ( "public_transaction_address" TEXT NOT NULL, - "public_transaction_nonce" BIGINT NOT NULL, + "public_transaction_nonce" BIGINT NOT NULL, -- moved to public_transaction_id in 13 "private_transaction_id" TEXT NOT NULL, "id" TEXT NOT NULL, PRIMARY KEY ("id") @@ -34,18 +34,3 @@ CREATE TABLE state_distribution_acknowledgments ( CREATE UNIQUE INDEX state_distribution_acknowledgments_state_distribution ON state_distribution_acknowledgments("state_distribution"); -CREATE TABLE transaction_delegations ( - "id" UUID NOT NULL, - "transaction_id" UUID NOT NULL, - "delegate_node_id" TEXT NOT NULL, - PRIMARY KEY ("id") --- need to reorder the migrations before we can define a foreign key to the transactions table FOREIGN KEY ("transaction_id") REFERENCES transactions ("id") ON DELETE CASCADE -); - -CREATE TABLE transaction_delegation_acknowledgements ( - "delegation" UUID NOT NULL, - "id" UUID NOT NULL, - PRIMARY KEY ("id"), - FOREIGN KEY ("delegation") REFERENCES transaction_delegations ("id") ON DELETE CASCADE -); -CREATE UNIQUE INDEX transaction_delegation_acknowledgements_delegation ON transaction_delegation_acknowledgements("delegation"); diff --git a/core/go/db/migrations/sqlite/000012_create_prepared_tx_tables.down.sql b/core/go/db/migrations/sqlite/000012_create_prepared_tx_tables.down.sql index 7ad868906..0ee252c5a 100644 --- a/core/go/db/migrations/sqlite/000012_create_prepared_tx_tables.down.sql +++ b/core/go/db/migrations/sqlite/000012_create_prepared_tx_tables.down.sql @@ -1,4 +1,4 @@ DROP TABLE prepared_txn_states; -DROP TABLE prepared_txns; DROP TABLE prepared_txn_distribution_acknowledgments; -DROP TABLE prepared_txn_distributions \ No newline at end of file +DROP TABLE prepared_txn_distributions; +DROP TABLE prepared_txns; diff --git a/core/go/db/migrations/sqlite/000013_delayed_nonce_allocation_updates.down.sql b/core/go/db/migrations/sqlite/000013_delayed_nonce_allocation_updates.down.sql new file mode 100644 index 000000000..fed56538e --- /dev/null +++ b/core/go/db/migrations/sqlite/000013_delayed_nonce_allocation_updates.down.sql @@ -0,0 +1 @@ +-- There is no ability to re-instate the signer_nonce column (and remove the pub_txn_id column) via a down migration diff --git a/core/go/db/migrations/sqlite/000013_delayed_nonce_allocation_updates.up.sql b/core/go/db/migrations/sqlite/000013_delayed_nonce_allocation_updates.up.sql new file mode 100644 index 000000000..6726f084b --- /dev/null +++ b/core/go/db/migrations/sqlite/000013_delayed_nonce_allocation_updates.up.sql @@ -0,0 +1,58 @@ +-- For SQLite we do not propagate public Tx data in created before migration 13 +DROP TABLE IF EXISTS public_txn_bindings; +DROP TABLE IF EXISTS public_submissions; +DROP TABLE IF EXISTS public_completions; +DROP TABLE IF EXISTS public_txns; +DROP TABLE IF EXISTS dispatches; + +CREATE TABLE public_txns ( + "pub_txn_id" INTEGER PRIMARY KEY AUTOINCREMENT, + "from" TEXT NOT NULL, + "nonce" BIGINT, + "created" BIGINT NOT NULL, + "to" TEXT, + "gas" BIGINT NOT NULL, + "fixed_gas_pricing" TEXT, + "value" TEXT, + "data" TEXT, + "suspended" BOOLEAN NOT NULL +); +CREATE UNIQUE INDEX public_txns_from_nonce ON public_txns("from", "nonce"); + +CREATE TABLE public_submissions ( + "tx_hash" TEXT NOT NULL, + "pub_txn_id" BIGINT NOT NULL, + "created" BIGINT NOT NULL, + "gas_pricing" TEXT, + PRIMARY KEY("tx_hash"), + FOREIGN KEY ("pub_txn_id") REFERENCES public_txns ("pub_txn_id") ON DELETE CASCADE +); +CREATE INDEX public_submissions_pub_txn_id on public_submissions("pub_txn_id"); + +CREATE TABLE public_completions ( + "pub_txn_id" INTEGER NOT NULL, + "created" BIGINT NOT NULL, + "tx_hash" TEXT NOT NULL, + "success" BOOLEAN NOT NULL, + "revert_data" TEXT, + FOREIGN KEY ("pub_txn_id") REFERENCES public_txns ("pub_txn_id") ON DELETE CASCADE, + PRIMARY KEY("pub_txn_id") +); + +CREATE TABLE public_txn_bindings ( + "pub_txn_id" INTEGER NOT NULL, + "transaction" UUID NOT NULL, + "tx_type" VARCHAR NOT NULL, + PRIMARY KEY ("pub_txn_id"), + FOREIGN KEY ("pub_txn_id") REFERENCES public_txns ("pub_txn_id") ON DELETE CASCADE +); +CREATE INDEX public_txn_bindings_transaction ON public_txn_bindings("transaction"); + +CREATE TABLE dispatches ( + "public_transaction_address" TEXT NOT NULL, + "public_transaction_id" BIGINT NOT NULL, + "private_transaction_id" TEXT NOT NULL, + "id" TEXT NOT NULL, + PRIMARY KEY ("id") +); +CREATE UNIQUE INDEX dispatches_public_private ON dispatches("public_transaction_address","public_transaction_id","private_transaction_id"); diff --git a/core/go/go.mod b/core/go/go.mod index 671332f72..e039fec63 100644 --- a/core/go/go.mod +++ b/core/go/go.mod @@ -18,6 +18,7 @@ require ( github.com/stretchr/testify v1.9.0 github.com/tyler-smith/go-bip39 v1.1.0 golang.org/x/crypto v0.28.0 + golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 golang.org/x/text v0.19.0 google.golang.org/grpc v1.67.1 google.golang.org/protobuf v1.35.1 @@ -99,7 +100,6 @@ require ( gitlab.com/hfuss/mux-prometheus v0.0.5 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect golang.org/x/net v0.30.0 // indirect golang.org/x/sync v0.8.0 // indirect golang.org/x/sys v0.26.0 // indirect diff --git a/core/go/internal/componentmgr/manager.go b/core/go/internal/componentmgr/manager.go index 514cd8504..8a3f99bf8 100644 --- a/core/go/internal/componentmgr/manager.go +++ b/core/go/internal/componentmgr/manager.go @@ -17,9 +17,11 @@ package componentmgr import ( "context" + "net/http" "github.com/google/uuid" "github.com/hyperledger/firefly-common/pkg/i18n" + "github.com/kaleido-io/paladin/config/pkg/confutil" "github.com/kaleido-io/paladin/config/pkg/pldconf" "github.com/kaleido-io/paladin/core/internal/components" "github.com/kaleido-io/paladin/core/internal/domainmgr" @@ -37,7 +39,10 @@ import ( "github.com/kaleido-io/paladin/core/pkg/ethclient" "github.com/kaleido-io/paladin/core/pkg/persistence" + "github.com/kaleido-io/paladin/toolkit/pkg/httpserver" "github.com/kaleido-io/paladin/toolkit/pkg/log" + "github.com/kaleido-io/paladin/toolkit/pkg/prototk" + "github.com/kaleido-io/paladin/toolkit/pkg/retry" "github.com/kaleido-io/paladin/toolkit/pkg/rpcserver" ) @@ -55,6 +60,8 @@ type componentManager struct { bgCtx context.Context // config conf *pldconf.PaladinConfig + // debug server + debugServer httpserver.Server // pre-init keyManager components.KeyManager ethClientFactory ethclient.EthClientFactory @@ -81,6 +88,8 @@ type componentManager struct { // keep track of everything we started started map[string]stoppable opened map[string]closeable + // limited startup retry for connecting to blockchain + ethClientStartupRetry *retry.Retry } // things that have a running component that is active in the background and hence "stops" @@ -99,20 +108,45 @@ func NewComponentManager(bgCtx context.Context, grpcTarget string, instanceUUID ) ComponentManager { log.InitConfig(&conf.Log) return &componentManager{ - grpcTarget: grpcTarget, // default is a UDS path, can use tcp:127.0.0.1:12345 strings too (or tcp4:/tcp6:) - instanceUUID: instanceUUID, - bgCtx: bgCtx, - conf: conf, - additionalManagers: additionalManagers, - initResults: make(map[string]*components.ManagerInitResult), - started: make(map[string]stoppable), - opened: make(map[string]closeable), + grpcTarget: grpcTarget, // default is a UDS path, can use tcp:127.0.0.1:12345 strings too (or tcp4:/tcp6:) + instanceUUID: instanceUUID, + bgCtx: bgCtx, + conf: conf, + additionalManagers: additionalManagers, + initResults: make(map[string]*components.ManagerInitResult), + started: make(map[string]stoppable), + opened: make(map[string]closeable), + ethClientStartupRetry: retry.NewRetryLimited(&conf.Startup.BlockchainConnectRetry, &pldconf.StartupConfigDefaults.BlockchainConnectRetry), } } +func (cm *componentManager) javaDump(res http.ResponseWriter, req *http.Request) { + cm.pluginManager.SendSystemCommandToLoader(prototk.PluginLoad_THREAD_DUMP) + res.WriteHeader(202) +} + +func (cm *componentManager) startDebugServer() (httpserver.Server, error) { + cm.conf.DebugServer.Port = confutil.P(confutil.Int(cm.conf.DebugServer.Port, 0)) // if enabled with no port, we allocate one + server, err := httpserver.NewDebugServer(cm.bgCtx, &cm.conf.DebugServer.HTTPServerConfig) + if err == nil { + server.Router().PathPrefix("/debug/javadump").HandlerFunc(http.HandlerFunc(cm.javaDump)) + err = server.Start() + } + return server, err +} + func (cm *componentManager) Init() (err error) { - cm.ethClientFactory, err = ethclient.NewEthClientFactory(cm.bgCtx, &cm.conf.Blockchain) - err = cm.wrapIfErr(err, msgs.MsgComponentEthClientInitError) + // start the debug server as early as possible + if confutil.Bool(cm.conf.DebugServer.Enabled, *pldconf.DebugServerDefaults.Enabled) { + cm.debugServer, err = cm.startDebugServer() + err = cm.addIfStarted("debugServer", cm.debugServer, err, msgs.MsgComponentDebugServerStartError) + } + + if err == nil { + cm.ethClientFactory, err = ethclient.NewEthClientFactory(cm.bgCtx, &cm.conf.Blockchain) + err = cm.wrapIfErr(err, msgs.MsgComponentEthClientInitError) + } + if err == nil { cm.persistence, err = persistence.NewPersistence(cm.bgCtx, &cm.conf.DB) err = cm.addIfOpened("database", cm.persistence, err, msgs.MsgComponentDBInitError) @@ -271,10 +305,17 @@ func (cm *componentManager) startBlockIndexer() (err error) { return err } +func (cm *componentManager) startEthClient() error { + return cm.ethClientStartupRetry.Do(cm.bgCtx, func(attempt int) (retryable bool, err error) { + return true, cm.ethClientFactory.Start() + }) +} + func (cm *componentManager) StartManagers() (err error) { // start the eth client before any managers - this connects the WebSocket, and gathers the ChainID - err = cm.ethClientFactory.Start() + // We have special handling here to allow for concurrent startup of the blockchain node and Paladin + err = cm.startEthClient() err = cm.addIfStarted("eth_client", cm.ethClientFactory, err, msgs.MsgComponentEthClientStartError) // start the managers diff --git a/core/go/internal/componentmgr/manager_test.go b/core/go/internal/componentmgr/manager_test.go index b9e8a4b85..1a60c53ce 100644 --- a/core/go/internal/componentmgr/manager_test.go +++ b/core/go/internal/componentmgr/manager_test.go @@ -18,7 +18,9 @@ package componentmgr import ( "context" "errors" + "fmt" "net" + "net/http" "os" "testing" @@ -43,6 +45,11 @@ import ( func TestInitOK(t *testing.T) { + l, err := net.Listen("tcp4", ":0") + require.NoError(t, err) + debugPort := l.Addr().(*net.TCPAddr).Port + require.NoError(t, l.Close()) + // We build a config that allows us to get through init successfully, as should be possible // (anything that can't do this should have a separate Start() phase). testConfig := &pldconf.PaladinConfig{ @@ -91,6 +98,12 @@ func TestInitOK(t *testing.T) { HTTP: pldconf.RPCServerConfigHTTP{Disabled: true}, WS: pldconf.RPCServerConfigWS{Disabled: true}, }, + DebugServer: pldconf.DebugServerConfig{ + Enabled: confutil.P(true), + HTTPServerConfig: pldconf.HTTPServerConfig{ + Port: confutil.P(debugPort), + }, + }, } mockExtraManager := componentmocks.NewAdditionalManager(t) @@ -98,7 +111,7 @@ func TestInitOK(t *testing.T) { mockExtraManager.On("PreInit", mock.Anything).Return(&components.ManagerInitResult{}, nil) mockExtraManager.On("PostInit", mock.Anything).Return(nil) cm := NewComponentManager(context.Background(), tempSocketFile(t), uuid.New(), testConfig, mockExtraManager).(*componentManager) - err := cm.Init() + err = cm.Init() require.NoError(t, err) assert.NotNil(t, cm.KeyManager()) @@ -116,6 +129,11 @@ func TestInitOK(t *testing.T) { assert.NotNil(t, cm.TxManager()) assert.NotNil(t, cm.IdentityResolver()) + // Check we can send a request for a javadump - even just after init (not start) + resp, err := http.Get(fmt.Sprintf("http://localhost:%d/debug/javadump", debugPort)) + require.NoError(t, err) + assert.Equal(t, http.StatusAccepted, resp.StatusCode) + cm.Stop() } diff --git a/core/go/internal/components/domainmgr.go b/core/go/internal/components/domainmgr.go index 01d797121..7ddbb6443 100644 --- a/core/go/internal/components/domainmgr.go +++ b/core/go/internal/components/domainmgr.go @@ -41,7 +41,7 @@ type DomainManager interface { ConfiguredDomains() map[string]*pldconf.PluginConfig DomainRegistered(name string, toDomain DomainManagerToDomain) (fromDomain plugintk.DomainCallbacks, err error) GetDomainByName(ctx context.Context, name string) (Domain, error) - GetSmartContractByAddress(ctx context.Context, addr tktypes.EthAddress) (DomainSmartContract, error) + GetSmartContractByAddress(ctx context.Context, dbTX *gorm.DB, addr tktypes.EthAddress) (DomainSmartContract, error) ExecDeployAndWait(ctx context.Context, txID uuid.UUID, call func() error) (dc DomainSmartContract, err error) ExecAndWaitTransaction(ctx context.Context, txID uuid.UUID, call func() error) error GetSigner() signerapi.InMemorySigner diff --git a/core/go/internal/components/pluginmgr.go b/core/go/internal/components/pluginmgr.go index 2303ddbdf..a8dfff864 100644 --- a/core/go/internal/components/pluginmgr.go +++ b/core/go/internal/components/pluginmgr.go @@ -19,6 +19,7 @@ import ( "context" "github.com/google/uuid" + "github.com/kaleido-io/paladin/toolkit/pkg/prototk" ) type PluginManager interface { @@ -27,4 +28,5 @@ type PluginManager interface { LoaderID() uuid.UUID WaitForInit(ctx context.Context) error ReloadPluginList() error + SendSystemCommandToLoader(cmd prototk.PluginLoad_SysCommand) } diff --git a/core/go/internal/components/privatetxmgr.go b/core/go/internal/components/privatetxmgr.go index 8c5a72799..c2fba5cb0 100644 --- a/core/go/internal/components/privatetxmgr.go +++ b/core/go/internal/components/privatetxmgr.go @@ -18,6 +18,7 @@ package components import ( "context" + "github.com/google/uuid" "github.com/hyperledger/firefly-signer/pkg/abi" "gorm.io/gorm" ) @@ -34,13 +35,26 @@ type TransactionDispatchedEvent struct { SigningAddress string `json:"signingAddress"` } +type PrivateTxEndorsementStatus struct { + Party string `json:"party"` + RequestTime string `json:"requestTime,omitempty"` + EndorsementReceived bool `json:"endorsementReceived"` +} + type PrivateTxStatus struct { - TxID string `json:"transactionId"` - Status string `json:"status"` - LatestEvent string `json:"latestEvent"` - LatestError string `json:"latestError"` + TxID string `json:"transactionId"` + Status string `json:"status"` + LatestEvent string `json:"latestEvent"` + LatestError string `json:"latestError"` + Endorsements []PrivateTxEndorsementStatus `json:"endorsements"` + Transaction *PrivateTransaction `json:"transaction,omitempty"` + FailureMessage string `json:"failureMessage,omitempty"` } +// If we had lots of these we would probably want to centralize the assignment of the constants to avoid duplication +// but currently there is only 2 ( the other being IDENTITY_RESOLVER_DESTINATION ) +const PRIVATE_TX_MANAGER_DESTINATION = "private-tx-manager" + type StateDistributionSet struct { LocalNode string SenderNode string @@ -67,8 +81,8 @@ type PrivateTxManager interface { TransportClient //Synchronous functions to submit a new private transaction - HandleNewTx(ctx context.Context, tx *ValidatedTransaction) error - GetTxStatus(ctx context.Context, domainAddress string, txID string) (status PrivateTxStatus, err error) + HandleNewTx(ctx context.Context, dbTX *gorm.DB, tx *ValidatedTransaction) error + GetTxStatus(ctx context.Context, domainAddress string, txID uuid.UUID) (status PrivateTxStatus, err error) // Synchronous function to call an existing deployed smart contract CallPrivateSmartContract(ctx context.Context, call *TransactionInputs) (*abi.ComponentValue, error) diff --git a/core/go/internal/components/publictxmgr.go b/core/go/internal/components/publictxmgr.go index 51d9b6873..1435c2a5e 100644 --- a/core/go/internal/components/publictxmgr.go +++ b/core/go/internal/components/publictxmgr.go @@ -27,25 +27,8 @@ import ( "gorm.io/gorm" ) -type PublicTxAccepted interface { - Bindings() []*PaladinTXReference - PublicTx() *pldapi.PublicTx // the nonce can only be read after Submit() on the batch succeeds -} - -type PublicTxRejected interface { - Bindings() []*PaladinTXReference - RejectedError() error // non-nil if the transaction was rejected during prepare (estimate gas error), so cannot be submitted - RevertData() tktypes.HexBytes // if revert data is available for error decoding -} - -type PublicTxBatch interface { - Submit(ctx context.Context, dbTX *gorm.DB) error - Accepted() []PublicTxAccepted - Rejected() []PublicTxRejected - Completed(ctx context.Context, committed bool) // caller must ensure this is called on all code paths, and only with true after DB TX has committed -} - var PublicTxFilterFields filters.FieldSet = filters.FieldMap{ + "localId": filters.Int64Field(`"public_txns"."pub_txn_id"`), "from": filters.HexBytesField(`"from"`), "nonce": filters.Int64Field("nonce"), "created": filters.Int64Field("created"), @@ -77,7 +60,14 @@ type PublicTxManager interface { QueryPublicTxForTransactions(ctx context.Context, dbTX *gorm.DB, boundToTxns []uuid.UUID, jq *query.QueryJSON) (map[uuid.UUID][]*pldapi.PublicTx, error) QueryPublicTxWithBindings(ctx context.Context, dbTX *gorm.DB, jq *query.QueryJSON) ([]*pldapi.PublicTxWithBinding, error) GetPublicTransactionForHash(ctx context.Context, dbTX *gorm.DB, hash tktypes.Bytes32) (*pldapi.PublicTxWithBinding, error) - PrepareSubmissionBatch(ctx context.Context, transactions []*PublicTxSubmission) (batch PublicTxBatch, err error) + + // Perform (potentially expensive) transaction level validation, such as gas estimation. Call before starting a DB transaction + ValidateTransaction(ctx context.Context, dbTX *gorm.DB, transaction *PublicTxSubmission) error + // Write a set of validated transactions to the public TX mgr database, notifying the relevant orchestrator(s) to wake, assign nonces, and start the submission process + WriteNewTransactions(ctx context.Context, dbTX *gorm.DB, transactions []*PublicTxSubmission) (func(), []*pldapi.PublicTx, error) + // Convenience function that does ValidateTransaction+WriteNewTransactions for a single Tx + SingleTransactionSubmit(ctx context.Context, transaction *PublicTxSubmission) (*pldapi.PublicTx, error) + MatchUpdateConfirmedTransactions(ctx context.Context, dbTX *gorm.DB, itxs []*blockindexer.IndexedTransactionNotify) ([]*PublicTxMatch, error) NotifyConfirmPersisted(ctx context.Context, confirms []*PublicTxMatch) } diff --git a/core/go/internal/components/statemgr.go b/core/go/internal/components/statemgr.go index da04d1cf2..7113d1a45 100644 --- a/core/go/internal/components/statemgr.go +++ b/core/go/internal/components/statemgr.go @@ -96,6 +96,12 @@ type DomainContext interface { // The dbTX is passed in to allow re-use of a connection during read operations. FindAvailableStates(dbTX *gorm.DB, schemaID tktypes.Bytes32, query *query.QueryJSON) (Schema, []*pldapi.State, error) + // Return a snapshot of all currently known state locks + ExportStateLocks() ([]byte, error) + + // ImportStateLocks is used to restore the state of the domain context, by adding a set of locks + ImportStateLocks([]byte) error + // FindAvailableNullifiers is similar to FindAvailableStates, but for domains that leverage // nullifiers to record spending. // diff --git a/core/go/internal/components/transaction.go b/core/go/internal/components/transaction.go index 4c9d6eff8..65d091ce2 100644 --- a/core/go/internal/components/transaction.go +++ b/core/go/internal/components/transaction.go @@ -24,12 +24,13 @@ import ( ) type TransactionInputs struct { - Domain string `json:"domain"` - From string `json:"from"` - To tktypes.EthAddress `json:"to"` - Function *abi.Entry `json:"function"` - Inputs tktypes.RawJSON `json:"inputs"` - Intent prototk.TransactionSpecification_Intent `json:"intent"` + Domain string `json:"domain"` + From string `json:"from"` + To tktypes.EthAddress `json:"to"` + Function *abi.Entry `json:"function"` + Inputs tktypes.RawJSON `json:"inputs"` + Intent prototk.TransactionSpecification_Intent `json:"intent"` + PublicTxOptions pldapi.PublicTxOptions `json:"publicTxOptions"` } type TransactionStateRefs struct { @@ -85,6 +86,7 @@ type TransactionPostAssembly struct { Signatures []*prototk.AttestationResult `json:"signatures"` Endorsements []*prototk.AttestationResult `json:"endorsements"` DomainData *string `json:"domain_data"` + RevertReason *string `json:"revert_reason"` } // PrivateTransaction is the critical exchange object between the engine and the domain manager, @@ -108,8 +110,6 @@ type PrivateTransaction struct { PreparedPublicTransaction *pldapi.TransactionInput `json:"-"` PreparedPrivateTransaction *pldapi.TransactionInput `json:"-"` PreparedMetadata tktypes.RawJSON `json:"-"` - - PublicTxOptions pldapi.PublicTxOptions `json:"-"` } // PrivateContractDeploy is a simpler transaction type that constructs new private smart contract instances diff --git a/core/go/internal/components/txmgr.go b/core/go/internal/components/txmgr.go index 81a99d2c3..4358d3b77 100644 --- a/core/go/internal/components/txmgr.go +++ b/core/go/internal/components/txmgr.go @@ -89,8 +89,8 @@ type TXManager interface { GetTransactionDependencies(ctx context.Context, id uuid.UUID) (*pldapi.TransactionDependencies, error) GetPublicTransactionByNonce(ctx context.Context, from tktypes.EthAddress, nonce tktypes.HexUint64) (*pldapi.PublicTxWithBinding, error) GetPublicTransactionByHash(ctx context.Context, hash tktypes.Bytes32) (*pldapi.PublicTxWithBinding, error) - QueryTransactions(ctx context.Context, jq *query.QueryJSON, pending bool) ([]*pldapi.Transaction, error) - QueryTransactionsFull(ctx context.Context, jq *query.QueryJSON, pending bool) (results []*pldapi.TransactionFull, err error) + QueryTransactions(ctx context.Context, jq *query.QueryJSON, dbTX *gorm.DB, pending bool) ([]*pldapi.Transaction, error) + QueryTransactionsFull(ctx context.Context, jq *query.QueryJSON, dbTX *gorm.DB, pending bool) (results []*pldapi.TransactionFull, err error) QueryTransactionsFullTx(ctx context.Context, jq *query.QueryJSON, dbTX *gorm.DB, pending bool) ([]*pldapi.TransactionFull, error) QueryTransactionReceipts(ctx context.Context, jq *query.QueryJSON) ([]*pldapi.TransactionReceipt, error) GetTransactionReceiptByID(ctx context.Context, id uuid.UUID) (*pldapi.TransactionReceipt, error) diff --git a/core/go/internal/domainmgr/domain.go b/core/go/internal/domainmgr/domain.go index ec29b2f87..67d645470 100644 --- a/core/go/internal/domainmgr/domain.go +++ b/core/go/internal/domainmgr/domain.go @@ -52,6 +52,7 @@ type domain struct { cancelCtx context.CancelFunc conf *pldconf.DomainConfig + defaultGasLimit tktypes.HexUint64 dm *domainManager name string api components.DomainManagerToDomain @@ -83,6 +84,7 @@ func (dm *domainManager) newDomain(name string, conf *pldconf.DomainConfig, toDo d := &domain{ dm: dm, conf: conf, + defaultGasLimit: pldconf.DefaultDefaultGasLimit, // can be set by config below initRetry: retry.NewRetryIndefinite(&conf.Init.Retry), name: name, api: toDomain, @@ -94,6 +96,9 @@ func (dm *domainManager) newDomain(name string, conf *pldconf.DomainConfig, toDo inFlight: make(map[string]*inFlightDomainRequest), } + if conf.DefaultGasLimit != nil { + d.defaultGasLimit = tktypes.HexUint64(*conf.DefaultGasLimit) + } log.L(dm.bgCtx).Debugf("Domain %s configured. Config: %s", name, tktypes.JSONString(conf.Config)) d.ctx, d.cancelCtx = context.WithCancel(log.WithLogField(dm.bgCtx, "domain", d.name)) return d diff --git a/core/go/internal/domainmgr/domain_test.go b/core/go/internal/domainmgr/domain_test.go index b48fd1f99..0d9bff52e 100644 --- a/core/go/internal/domainmgr/domain_test.go +++ b/core/go/internal/domainmgr/domain_test.go @@ -192,6 +192,7 @@ func newTestDomain(t *testing.T, realDB bool, domainConfig *prototk.DomainConfig "test1": { Config: map[string]any{"some": "conf"}, RegistryAddress: tktypes.RandHex(20), + DefaultGasLimit: confutil.P(uint64(100000)), }, }, }, extraSetup...) diff --git a/core/go/internal/domainmgr/event_indexer_test.go b/core/go/internal/domainmgr/event_indexer_test.go index 01ebf77bc..0843ea271 100644 --- a/core/go/internal/domainmgr/event_indexer_test.go +++ b/core/go/internal/domainmgr/event_indexer_test.go @@ -103,7 +103,7 @@ func TestEventIndexingWithDB(t *testing.T) { } // Lookup the instance against the domain - psc, err := dm.GetSmartContractByAddress(ctx, contractAddr) + psc, err := dm.GetSmartContractByAddress(ctx, td.c.dbTX, contractAddr) require.NoError(t, err) dc := psc.(*domainContract) assert.Equal(t, &PrivateSmartContract{ @@ -117,7 +117,7 @@ func TestEventIndexingWithDB(t *testing.T) { assert.Equal(t, "0xfeedbeef", psc.(*domainContract).info.ConfigBytes.String()) // Get cached - psc2, err := dm.GetSmartContractByAddress(ctx, contractAddr) + psc2, err := dm.GetSmartContractByAddress(ctx, td.c.dbTX, contractAddr) require.NoError(t, err) assert.Equal(t, psc, psc2) } diff --git a/core/go/internal/domainmgr/manager.go b/core/go/internal/domainmgr/manager.go index b70c008ff..1049d27e4 100644 --- a/core/go/internal/domainmgr/manager.go +++ b/core/go/internal/domainmgr/manager.go @@ -139,6 +139,7 @@ func (dm *domainManager) Stop() { func (dm *domainManager) cleanupDomain(d *domain) { // must not hold the domain lock when running this + log.L(dm.bgCtx).Infof("Cleaning up domain plugin after unload name=%s address=%s", d.name, d.RegistryAddress()) d.close() delete(dm.domainsByName, d.name) delete(dm.domainsByAddress, *d.RegistryAddress()) @@ -176,6 +177,9 @@ func (dm *domainManager) DomainRegistered(name string, toDomain components.Domai // Initialize d := dm.newDomain(name, conf, toDomain) dm.domainsByName[name] = d + + log.L(dm.bgCtx).Infof("Domain plugin registered name=%s address=%s", d.name, d.RegistryAddress()) + go d.init() return d, nil } @@ -232,7 +236,7 @@ func (dm *domainManager) waitForDeploy(ctx context.Context, req *inflight.Inflig return nil, i18n.NewError(ctx, msgs.MsgDomainTransactionWasNotADeployment, receipt.TransactionID) } - return dm.GetSmartContractByAddress(ctx, *receipt.ContractAddress) + return dm.GetSmartContractByAddress(ctx, dm.persistence.DB(), *receipt.ContractAddress) } func (dm *domainManager) ExecAndWaitTransaction(ctx context.Context, txID uuid.UUID, call func() error) error { @@ -273,8 +277,8 @@ func (dm *domainManager) getDomainByAddressOrNil(addr *tktypes.EthAddress) *doma return dm.domainsByAddress[*addr] } -func (dm *domainManager) GetSmartContractByAddress(ctx context.Context, addr tktypes.EthAddress) (components.DomainSmartContract, error) { - loadResult, dc, err := dm.getSmartContractCached(ctx, dm.persistence.DB(), addr) +func (dm *domainManager) GetSmartContractByAddress(ctx context.Context, dbTX *gorm.DB, addr tktypes.EthAddress) (components.DomainSmartContract, error) { + loadResult, dc, err := dm.getSmartContractCached(ctx, dbTX, addr) if dc != nil || err != nil { return dc, err } diff --git a/core/go/internal/domainmgr/private_smart_contract.go b/core/go/internal/domainmgr/private_smart_contract.go index c85924b19..b4e5adcab 100644 --- a/core/go/internal/domainmgr/private_smart_contract.go +++ b/core/go/internal/domainmgr/private_smart_contract.go @@ -74,6 +74,7 @@ func (d *domain) initSmartContract(ctx context.Context, def *PrivateSmartContrac ContractConfig: def.ConfigBytes, }) if err != nil { + log.L(ctx).Errorf("Error initializing smart contract address: %s with config %s : %s", def.Address, def.ConfigBytes.HexString(), err.Error()) return pscInitError, nil, err } if !res.Valid { @@ -243,6 +244,7 @@ func (dc *domainContract) AssembleTransaction(dCtx components.DomainContext, rea // We need to pass the assembly result back - it needs to be assigned to a sequence // before anything interesting can happen with the result here + postAssembly.RevertReason = res.RevertReason postAssembly.AssemblyResult = res.AssemblyResult postAssembly.AttestationPlan = res.AttestationPlan tx.PostAssembly = postAssembly @@ -510,7 +512,7 @@ func (dc *domainContract) PrepareTransaction(dCtx components.DomainContext, read } if res.Transaction.Type == prototk.PreparedTransaction_PRIVATE { - psc, err := dc.dm.GetSmartContractByAddress(dCtx.Ctx(), *contractAddress) + psc, err := dc.dm.GetSmartContractByAddress(dCtx.Ctx(), readTX, *contractAddress) if err != nil { return err } @@ -529,14 +531,21 @@ func (dc *domainContract) PrepareTransaction(dCtx components.DomainContext, read } else { tx.PreparedPublicTransaction = &pldapi.TransactionInput{ TransactionBase: pldapi.TransactionBase{ - Type: pldapi.TransactionTypePublic.Enum(), - Function: functionABI.String(), - From: tx.Signer, - To: contractAddress, - Data: tktypes.RawJSON(res.Transaction.ParamsJson), + Type: pldapi.TransactionTypePublic.Enum(), + Function: functionABI.String(), + From: tx.Signer, + To: contractAddress, + Data: tktypes.RawJSON(res.Transaction.ParamsJson), + PublicTxOptions: tx.Inputs.PublicTxOptions, }, ABI: abi.ABI{&functionABI}, } + // We cannot fall back to eth_estimateGas, as we queue up multiple transactions for dispatch that chain together. + // As such our transactions are not always executable in isolation, and would revert (due to consuming non-existent UTXO states) + // if we attempted to do gas estimation or call. + if tx.PreparedPublicTransaction.PublicTxOptions.Gas == nil { + tx.PreparedPublicTransaction.PublicTxOptions.Gas = &dc.d.defaultGasLimit + } } if res.Metadata != nil { tx.PreparedMetadata = tktypes.RawJSON(*res.Metadata) diff --git a/core/go/internal/domainmgr/private_smart_contract_test.go b/core/go/internal/domainmgr/private_smart_contract_test.go index 72f52d1df..915e0800a 100644 --- a/core/go/internal/domainmgr/private_smart_contract_test.go +++ b/core/go/internal/domainmgr/private_smart_contract_test.go @@ -52,7 +52,7 @@ func TestPrivateSmartContractQueryFail(t *testing.T) { }) defer done() - _, err := td.dm.GetSmartContractByAddress(td.ctx, tktypes.EthAddress(tktypes.RandBytes(20))) + _, err := td.dm.GetSmartContractByAddress(td.ctx, td.c.dbTX, tktypes.EthAddress(tktypes.RandBytes(20))) assert.Regexp(t, "pop", err) } @@ -65,7 +65,7 @@ func TestPrivateSmartContractQueryNoResult(t *testing.T) { }) defer done() - _, err := td.dm.GetSmartContractByAddress(td.ctx, tktypes.EthAddress(tktypes.RandBytes(20))) + _, err := td.dm.GetSmartContractByAddress(td.ctx, td.c.dbTX, tktypes.EthAddress(tktypes.RandBytes(20))) assert.Regexp(t, "PD011609", err) } @@ -180,7 +180,7 @@ func doDomainInitAssembleTransactionOK(t *testing.T, td *testDomainContext) (*do }, AttestationPlan: []*prototk.AttestationRequest{ { - Name: "ensorsement1", + Name: "endorsement1", AttestationType: prototk.AttestationType_ENDORSE, Algorithm: algorithms.ECDSA_SECP256K1, PayloadType: signpayloads.OPAQUE_TO_RSV, @@ -866,6 +866,25 @@ func TestDomainAssembleTransactionLoadInputError(t *testing.T) { assert.Nil(t, tx.PostAssembly) } +func TestDomainAssembleTransactionRevert(t *testing.T) { + td, done := newTestDomain(t, false, goodDomainConf(), mockSchemas(), mockBlockHeight) + defer done() + + psc, tx := doDomainInitTransactionOK(t, td) + td.tp.Functions.AssembleTransaction = func(ctx context.Context, req *prototk.AssembleTransactionRequest) (*prototk.AssembleTransactionResponse, error) { + return &prototk.AssembleTransactionResponse{ + AssemblyResult: prototk.AssembleTransactionResponse_REVERT, + RevertReason: confutil.P("failed with error"), + }, nil + } + err := psc.AssembleTransaction(td.mdc, td.c.dbTX, tx) + require.NoError(t, err) + + assert.NotNil(t, tx.PostAssembly) + assert.Equal(t, prototk.AssembleTransactionResponse_REVERT, tx.PostAssembly.AssemblyResult) + assert.Equal(t, "failed with error", *tx.PostAssembly.RevertReason) +} + func TestDomainAssembleTransactionLoadReadError(t *testing.T) { td, done := newTestDomain(t, false, goodDomainConf(), mockSchemas(), mockBlockHeight) defer done() @@ -1393,7 +1412,7 @@ func TestGetPSCInvalidConfig(t *testing.T) { }, nil } - psc, err := td.dm.GetSmartContractByAddress(td.ctx, *addr) + psc, err := td.dm.GetSmartContractByAddress(td.ctx, td.c.dbTX, *addr) require.Regexp(t, "PD011610", err) // invalid config assert.Nil(t, psc) } @@ -1422,7 +1441,7 @@ func TestGetPSCUnknownDomain(t *testing.T) { }, nil } - psc, err := td.dm.GetSmartContractByAddress(td.ctx, *addr) + psc, err := td.dm.GetSmartContractByAddress(td.ctx, td.c.dbTX, *addr) require.Regexp(t, "PD011654", err) // domain no longer configured assert.Nil(t, psc) } @@ -1444,7 +1463,7 @@ func TestGetPSCInitError(t *testing.T) { return nil, fmt.Errorf("pop") } - psc, err := td.dm.GetSmartContractByAddress(td.ctx, *addr) + psc, err := td.dm.GetSmartContractByAddress(td.ctx, td.c.dbTX, *addr) require.Regexp(t, "pop", err) // domain no longer configured assert.Nil(t, psc) } diff --git a/core/go/internal/identityresolver/identityresolver.go b/core/go/internal/identityresolver/identityresolver.go index 83880e852..68a6431f0 100644 --- a/core/go/internal/identityresolver/identityresolver.go +++ b/core/go/internal/identityresolver/identityresolver.go @@ -49,7 +49,6 @@ type inflightRequest struct { failed func(ctx context.Context, err error) } -// As a LateBoundComponent, the identity resolver is created and initialised in a single function call func NewIdentityResolver(ctx context.Context, conf *pldconf.IdentityResolverConfig) components.IdentityResolver { return &identityResolver{ bgCtx: ctx, diff --git a/core/go/internal/msgs/en_errors.go b/core/go/internal/msgs/en_errors.go index 50ea7d6f1..039283271 100644 --- a/core/go/internal/msgs/en_errors.go +++ b/core/go/internal/msgs/en_errors.go @@ -73,6 +73,7 @@ var ( MsgComponentIdentityResolverStartError = ffe("PD010030", "Error starting identity resolver") MsgComponentAdditionalMgrInitError = ffe("PD010031", "Error initializing %s manager") MsgComponentAdditionalMgrStartError = ffe("PD010032", "Error initializing %s manager") + MsgComponentDebugServerStartError = ffe("PD010033", "Error starting debug server") // States PD0101XX MsgStateInvalidLength = ffe("PD010101", "Invalid hash len expected=%d actual=%d") @@ -104,6 +105,7 @@ var ( MsgStateHashMismatch = ffe("PD010129", "The supplied state ID '%s' does not match the state hash '%s'") MsgStateIDMissing = ffe("PD010130", "The state id must be supplied for this domain") MsgStateFlushInProgress = ffe("PD010131", "A flush is already in progress for this domain context") + MsgDomainContextImportInvalidJSON = ffe("PD010132", "Attempted to import state locks but the JSON could not be parsed") // Persistence PD0102XX MsgPersistenceInvalidType = ffe("PD010200", "Invalid persistence type: %s") @@ -272,9 +274,10 @@ var ( MsgDomainABIEncodingInlineSigningFailed = ffe("PD011656", "ABI encoding and signing failed with transaction type '%s' and key identifier '%s'") MsgDomainDomainReceiptNotAvailable = ffe("PD011657", "A domain receipt is not yet available for transaction %s as no state confirmations have been indexed for that transaction") MsgDomainDomainReceiptNoStatesAvailable = ffe("PD011658", "A domain receipt is not yet available for transaction %s as none of the private state data is available on the local node") - MsgDomainSingingKeyMustBeLocalEthSign = ffe("PD011659", "Singing key must be local for ethereum transaction signing") + MsgDomainSingingKeyMustBeLocalEthSign = ffe("PD011659", "Signing key must be local for ethereum transaction signing") MsgDomainNullifierForPartyOutsideDistro = ffe("PD011660", "A nullifier was requested for a party that is not in the distribution list") MsgDomainInvalidFromAddress = ffe("PD011661", "Invalid from identity in transaction") + MsgDomainInvalidCoordinatorSelection = ffe("PD011662", "Invalid coordinator selection of '%s' configured. valid options are: COORDINATOR_SENDER, COORDINATOR_STATIC, COORDINATOR_ENDORSER") // Entrypoint PD0117XX MsgEntrypointUnknownRunMode = ffe("PD011700", "Unknown run mode '%s'") @@ -294,7 +297,7 @@ var ( MsgContractAddressNotProvided = ffe("PD011811", "Contract address (To) not found in the transaction input") MsgPrivTxMgrPublicTxFail = ffe("PD011812", "Public transaction rejected") MsgResolveVerifierRemoteFailed = ffe("PD011813", "Failed to resolve verifier on remote node with lookup %s algorithm %s: Error %s") - MsgPrivateTxManagerAssembleRevert = ffe("PD011814", "Domain reverted transaction on assemble") + MsgPrivateTxManagerAssembleRevert = ffe("PD011814", "Domain reverted transaction on assemble: %s") MsgPrivateTxManagerResolveError = ffe("PD011815", "Failed to resolve local signer for party %s (verifier=%s,algorithm=%s): %s") MsgPrivateTxManagerSignError = ffe("PD011816", "Failed to sign for party %s (verifier=%s,algorithm=%s): %s") MsgPrivateTxManagerEndorsementRequestError = ffe("PD011817", "Failed to request endorsement from %s: %s") @@ -314,6 +317,9 @@ var ( MsgPrivateTxMgrInvalidTxStateStateDistro = ffe("PD011831", "Invalid transaction state for state distribution") MsgPrivateTxMgrDistributionNotFullyQualified = ffe("PD011832", "State distribution from domain is not fully qualified: %s") MsgPrivateTxMgrInvalidNullifierSpecInDistro = ffe("PD011833", "Invalid nullifier specification in new state instruction from domain") + MsgPrivateTxManagerNewSequencerError = ffe("PD011834", "Failed to create new sequencer") + MsgPrivateTxManagerInvalidStaticCoordinator = ffe("PD011835", "Contract was configured with invalid static coordinator '%s'. Must be of the form 'identity@node'") + MsgPrivateTxMgrFunctionNotProvided = ffe("PD011836", "Function abi not provided in transaction input") // Public Transaction Manager PD0119XX MsgInsufficientBalance = ffe("PD011900", "Balance %s of fueling source address %s is below the required amount %s") diff --git a/core/go/internal/plugins/domains_test.go b/core/go/internal/plugins/domains_test.go index 12353b1f9..faa64d25f 100644 --- a/core/go/internal/plugins/domains_test.go +++ b/core/go/internal/plugins/domains_test.go @@ -121,7 +121,7 @@ func newTestDomainPluginManager(t *testing.T, setup *testManagers) (context.Cont func TestDomainRequestsOK(t *testing.T) { - log.SetLevel("debug") + log.InitConfig(&pldconf.LogConfig{Level: confutil.P("debug")}) // test debug specific logging waitForAPI := make(chan components.DomainManagerToDomain, 1) waitForCallbacks := make(chan plugintk.DomainCallbacks, 1) diff --git a/core/go/internal/plugins/plugin_manager.go b/core/go/internal/plugins/plugin_manager.go index 3611d154a..b4231ec4a 100644 --- a/core/go/internal/plugins/plugin_manager.go +++ b/core/go/internal/plugins/plugin_manager.go @@ -65,6 +65,7 @@ type pluginManager struct { registryPlugins map[uuid.UUID]*plugin[prototk.RegistryMessage] notifyPluginsUpdated chan bool + notifySystemCommand chan prototk.PluginLoad_SysCommand pluginLoaderDone chan struct{} loadingProgressed chan *prototk.PluginLoadFailed serverDone chan error @@ -88,6 +89,7 @@ func NewPluginManager(bgCtx context.Context, serverDone: make(chan error), notifyPluginsUpdated: make(chan bool, 1), + notifySystemCommand: make(chan prototk.PluginLoad_SysCommand, 1), loadingProgressed: make(chan *prototk.PluginLoadFailed, 1), } return pc @@ -272,6 +274,14 @@ func (pm *pluginManager) InitLoader(req *prototk.PluginLoaderInit, stream protot return pm.sendPluginsToLoader(stream) } +func (pm *pluginManager) SendSystemCommandToLoader(cmd prototk.PluginLoad_SysCommand) { + select { + case pm.notifySystemCommand <- cmd: + default: + log.L(pm.bgCtx).Warnf("Unable to send system command to loader (command already queued)") + } +} + func (pm *pluginManager) LoadFailed(ctx context.Context, req *prototk.PluginLoadFailed) (*prototk.EmptyResponse, error) { log.L(ctx).Errorf("Plugin load %s (type=%s) failed: %s", req.Plugin.Name, req.Plugin.PluginType, req.ErrorMessage) select { @@ -383,6 +393,8 @@ func (pm *pluginManager) sendPluginsToLoader(stream prototk.PluginController_Ini case <-ctx.Done(): log.L(ctx).Debugf("loader stream closed") err = i18n.NewError(ctx, msgs.MsgContextCanceled) + case systemCommand := <-pm.notifySystemCommand: + _ = stream.Send(&prototk.PluginLoad{SysCommand: &systemCommand}) case <-pm.notifyPluginsUpdated: // loop and load any that need loading } diff --git a/core/go/internal/plugins/plugin_manager_test.go b/core/go/internal/plugins/plugin_manager_test.go index b3ccdc4ab..2b07cbfab 100644 --- a/core/go/internal/plugins/plugin_manager_test.go +++ b/core/go/internal/plugins/plugin_manager_test.go @@ -205,6 +205,10 @@ func TestNotifyPluginUpdateNotStarted(t *testing.T) { require.NoError(t, err) err = pc.ReloadPluginList() require.NoError(t, err) + + // System command sending should not block + pc.SendSystemCommandToLoader(prototk.PluginLoad_THREAD_DUMP) + pc.SendSystemCommandToLoader(prototk.PluginLoad_THREAD_DUMP) } func TestLoaderErrors(t *testing.T) { @@ -269,6 +273,12 @@ func TestLoaderErrors(t *testing.T) { err = pc.WaitForInit(ctx) assert.Regexp(t, "pop", err) + // Get a system command + pc.SendSystemCommandToLoader(prototk.PluginLoad_THREAD_DUMP) + loadReq, err = loaderStream.Recv() + require.NoError(t, err) + require.Equal(t, prototk.PluginLoad_THREAD_DUMP, *loadReq.SysCommand) + // then attempt double start of the loader dupLoader, err := client.InitLoader(ctx, &prototk.PluginLoaderInit{ Id: pc.LoaderID().String(), diff --git a/core/go/internal/preparedtxdistribution/prepared_transaction_receiver.go b/core/go/internal/preparedtxdistribution/prepared_transaction_receiver.go index ccbe2d2b0..7663891c2 100644 --- a/core/go/internal/preparedtxdistribution/prepared_transaction_receiver.go +++ b/core/go/internal/preparedtxdistribution/prepared_transaction_receiver.go @@ -25,7 +25,7 @@ import ( ) func (sd *preparedTransactionDistributer) sendPreparedTransactionAcknowledgement(ctx context.Context, domainName string, contractAddress string, preparedTxnId string, receivingParty string, distributingNode string, distributionID string) error { - log.L(ctx).Debugf("preparedTransactionDistributer:sendPreparedTransactionAcknowledgement %s %s %s %s %s %s", domainName, contractAddress, preparedTxnId, receivingParty, distributingNode, distributionID) + log.L(ctx).Debugf("preparedTransactionDistributer:sendPreparedTransactionAcknowledgement domainName=%s contractAddress=%s preparedTxnId=%s receivingParty=%s distributingNode=%s distributionID=%s", domainName, contractAddress, preparedTxnId, receivingParty, distributingNode, distributionID) preparedTransactionAcknowledgedMessage := &pb.PreparedTransactionAcknowledgedMessage{ DomainName: domainName, ContractAddress: contractAddress, diff --git a/core/go/internal/privatetxnmgr/assemble_and_sign.go b/core/go/internal/privatetxnmgr/assemble_and_sign.go new file mode 100644 index 000000000..49048f9fb --- /dev/null +++ b/core/go/internal/privatetxnmgr/assemble_and_sign.go @@ -0,0 +1,189 @@ +/* + * Copyright © 2024 Kaleido, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package privatetxnmgr + +import ( + "context" + "fmt" + + "github.com/google/uuid" + "github.com/hyperledger/firefly-common/pkg/i18n" + "github.com/kaleido-io/paladin/core/internal/components" + "github.com/kaleido-io/paladin/core/internal/msgs" + "github.com/kaleido-io/paladin/toolkit/pkg/log" + "github.com/kaleido-io/paladin/toolkit/pkg/prototk" + "github.com/kaleido-io/paladin/toolkit/pkg/tktypes" +) + +// assemble a transaction that we are not coordinating, using the provided state locks +// all errors are assumed to be transient and the request should be retried +// if the domain as deemed the request as invalid then it will communicate the `revert` directive via the AssembleTransactionResponse_REVERT result without any error +func (s *Sequencer) assembleForRemoteCoordinator(ctx context.Context, transactionID uuid.UUID, transactionInputs *components.TransactionInputs, preAssembly *components.TransactionPreAssembly, stateLocksJSON []byte, blockHeight int64) (*components.TransactionPostAssembly, error) { + + log.L(ctx).Debugf("assembleForRemoteCoordinator: Assembling transaction %s ", transactionID) + + log.L(ctx).Debugf("assembleForRemoteCoordinator: resetting domain context with state locks from the coordinator which assumes a block height of %d compared with local blockHeight of %d", blockHeight, s.environment.GetBlockHeight()) + //If our block height is behind the coordinator, there are some states that would otherwise be available to us but we wont see + // if our block height is ahead of the coordinator, there is a small chance that we we assemble a transaction that the coordinator will not be able to + // endorse yet but it is better to wait around on the endorsement flow than to wait around on the assemble flow which is single threaded per domain + + err := s.delegateDomainContext.ImportStateLocks(stateLocksJSON) + if err != nil { + log.L(ctx).Errorf("assembleForRemoteCoordinator: Error importing state locks: %s", err) + return nil, err + } + + postAssembly, err := s.assembleAndSign(ctx, transactionID, transactionInputs, preAssembly, s.delegateDomainContext) + + if err != nil { + log.L(ctx).Errorf("assembleForRemoteCoordinator: Error assembling and signing transaction: %s", err) + return nil, err + } + + return postAssembly, nil +} + +func (s *Sequencer) AssembleLocal(ctx context.Context, requestID string, transactionID uuid.UUID, transactionInputs *components.TransactionInputs, preAssembly *components.TransactionPreAssembly) { + + log.L(ctx).Debugf("assembleForLocalCoordinator: Assembling transaction %s ", transactionID) + + postAssembly, err := s.assembleAndSign(ctx, transactionID, transactionInputs, preAssembly, s.coordinatorDomainContext) + + if err != nil { + log.L(ctx).Errorf("assembleForLocalCoordinator: Error assembling and signing transaction: %s", err) + s.publisher.PublishTransactionAssembleFailedEvent(ctx, + transactionID.String(), + i18n.ExpandWithCode(ctx, i18n.MessageKey(msgs.MsgPrivateTxManagerAssembleError), err.Error()), + requestID, + ) + return + } + + s.publisher.PublishTransactionAssembledEvent(ctx, + transactionID.String(), + postAssembly, + requestID, + ) +} + +func (s *Sequencer) assembleAndSign(ctx context.Context, transactionID uuid.UUID, transactionInputs *components.TransactionInputs, preAssembly *components.TransactionPreAssembly, domainContext components.DomainContext) (*components.TransactionPostAssembly, error) { + //Assembles the transaction and synchronously fulfills any local signature attestation requests + // Given that the coordinator is single threading calls to assemble, there may be benefits to performance if we were to fulfill the signature request async + // but that would introduce levels of complexity that may not be justified so this is open as a potential for future optimization where we would need to think about + // whether a lost/late signature would trigger a re-assembly of the transaction ( and any transaction that come after it in the sequencer) or whether we could safely ask the assembly + // to post hoc sign an assembly + + transaction := &components.PrivateTransaction{ + ID: transactionID, + Inputs: transactionInputs, + PreAssembly: preAssembly, + } + + /* + * Assemble + */ + readTX := s.components.Persistence().DB() + err := s.domainAPI.AssembleTransaction(domainContext, readTX, transaction) + if err != nil { + log.L(ctx).Errorf("assembleAndSign: Error assembling transaction: %s", err) + return nil, err + } + if transaction.PostAssembly == nil { + log.L(ctx).Errorf("assembleForCoordinator: AssembleTransaction returned nil PostAssembly") + // This is most likely a programming error in the domain + err := i18n.NewError(ctx, msgs.MsgPrivateTxManagerInternalError, "AssembleTransaction returned nil PostAssembly") + log.L(ctx).Error(err) + return nil, err + } + + // Some validation that we are confident we can execute the given attestation plan + for _, attRequest := range transaction.PostAssembly.AttestationPlan { + switch attRequest.AttestationType { + case prototk.AttestationType_ENDORSE: + case prototk.AttestationType_SIGN: + case prototk.AttestationType_GENERATE_PROOF: + errorMessage := "AttestationType_GENERATE_PROOF is not implemented yet" + log.L(ctx).Error(errorMessage) + return nil, i18n.NewError(ctx, msgs.MsgPrivateTxManagerInternalError, errorMessage) + + default: + errorMessage := fmt.Sprintf("Unsupported attestation type: %s", attRequest.AttestationType) + log.L(ctx).Error(errorMessage) + return nil, i18n.NewError(ctx, msgs.MsgPrivateTxManagerInternalError, errorMessage) + } + } + + /* + * Sign + */ + for _, attRequest := range transaction.PostAssembly.AttestationPlan { + if attRequest.AttestationType == prototk.AttestationType_SIGN { + for _, partyName := range attRequest.Parties { + unqualifiedLookup, signerNode, err := tktypes.PrivateIdentityLocator(partyName).Validate(ctx, s.nodeName, true) + if err != nil { + log.L(ctx).Errorf("Failed to validate identity locator for signing party %s: %s", partyName, err) + return nil, err + } + if signerNode == s.nodeName { + + keyMgr := s.components.KeyManager() + resolvedKey, err := keyMgr.ResolveKeyNewDatabaseTX(ctx, unqualifiedLookup, attRequest.Algorithm, attRequest.VerifierType) + if err != nil { + log.L(ctx).Errorf("Failed to resolve local signer for %s (algorithm=%s): %s", unqualifiedLookup, attRequest.Algorithm, err) + return nil, i18n.WrapError(ctx, err, msgs.MsgPrivateTxManagerResolveError, unqualifiedLookup, attRequest.Algorithm) + } + + signaturePayload, err := keyMgr.Sign(ctx, resolvedKey, attRequest.PayloadType, attRequest.Payload) + if err != nil { + log.L(ctx).Errorf("failed to sign for party %s (verifier=%s,algorithm=%s): %s", unqualifiedLookup, resolvedKey.Verifier.Verifier, attRequest.Algorithm, err) + return nil, i18n.WrapError(ctx, err, msgs.MsgPrivateTxManagerSignError, unqualifiedLookup, resolvedKey.Verifier.Verifier, attRequest.Algorithm) + } + log.L(ctx).Debugf("payload: %x signed %x by %s (%s)", attRequest.Payload, signaturePayload, unqualifiedLookup, resolvedKey.Verifier.Verifier) + + transaction.PostAssembly.Signatures = []*prototk.AttestationResult{ + { + Name: attRequest.Name, + AttestationType: attRequest.AttestationType, + Verifier: &prototk.ResolvedVerifier{ + Lookup: partyName, + Algorithm: attRequest.Algorithm, + Verifier: resolvedKey.Verifier.Verifier, + VerifierType: attRequest.VerifierType, + }, + Payload: signaturePayload, + PayloadType: &attRequest.PayloadType, + }, + } + + } else { + log.L(ctx).Warnf("assembleAndSign: ignoring signature request of transaction %s for remote party %s ", transactionID, partyName) + + } + } + } else { + log.L(ctx).Debugf("assembleAndSign: ignoring attestationType %s for fulfillment later", attRequest.AttestationType) + } + } + + if log.IsDebugEnabled() { + stateIDs := "" + for _, state := range transaction.PostAssembly.OutputStates { + stateIDs += "," + state.ID.String() + } + log.L(ctx).Debugf("assembleAndSign: Assembled transaction %s : %s", transactionID, stateIDs) + } + return transaction.PostAssembly, nil +} diff --git a/core/go/internal/privatetxnmgr/assemble_coordinator.go b/core/go/internal/privatetxnmgr/assemble_coordinator.go new file mode 100644 index 000000000..f378ddcef --- /dev/null +++ b/core/go/internal/privatetxnmgr/assemble_coordinator.go @@ -0,0 +1,207 @@ +/* + * Copyright © 2024 Kaleido, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package privatetxnmgr + +import ( + "context" + "time" + + "github.com/google/uuid" + "github.com/kaleido-io/paladin/core/internal/components" + "github.com/kaleido-io/paladin/core/internal/privatetxnmgr/ptmgrtypes" + "github.com/kaleido-io/paladin/core/internal/statedistribution" + "github.com/kaleido-io/paladin/toolkit/pkg/log" + "github.com/kaleido-io/paladin/toolkit/pkg/tktypes" +) + +type assembleCoordinator struct { + ctx context.Context + nodeName string + requests chan *assembleRequest + stopProcess chan bool + commit chan string + components components.AllComponents + domainAPI components.DomainSmartContract + domainContext components.DomainContext + transportWriter ptmgrtypes.TransportWriter + contractAddress tktypes.EthAddress + sequencerEnvironment ptmgrtypes.SequencerEnvironment + requestTimeout time.Duration + stateDistributer statedistribution.StateDistributer + localAssembler ptmgrtypes.LocalAssembler +} + +type assembleRequest struct { + assemblingNode string + assembleCoordinator *assembleCoordinator + transactionID uuid.UUID + transactionInputs *components.TransactionInputs + transactionPreassembly *components.TransactionPreAssembly +} + +func NewAssembleCoordinator(ctx context.Context, nodeName string, maxPendingRequests int, components components.AllComponents, domainAPI components.DomainSmartContract, domainContext components.DomainContext, transportWriter ptmgrtypes.TransportWriter, contractAddress tktypes.EthAddress, sequencerEnvironment ptmgrtypes.SequencerEnvironment, requestTimeout time.Duration, stateDistributer statedistribution.StateDistributer, localAssembler ptmgrtypes.LocalAssembler) ptmgrtypes.AssembleCoordinator { + return &assembleCoordinator{ + ctx: ctx, + nodeName: nodeName, + stopProcess: make(chan bool, 1), + requests: make(chan *assembleRequest, maxPendingRequests), + commit: make(chan string, 1), + components: components, + domainAPI: domainAPI, + domainContext: domainContext, + transportWriter: transportWriter, + contractAddress: contractAddress, + sequencerEnvironment: sequencerEnvironment, + requestTimeout: requestTimeout, + stateDistributer: stateDistributer, + localAssembler: localAssembler, + } +} + +func (ac *assembleCoordinator) Complete(requestID string, stateDistributions []*components.StateDistribution) { + + log.L(ac.ctx).Debugf("AssembleCoordinator:Commit %s", requestID) + ac.stateDistributer.DistributeStates(ac.ctx, stateDistributions) + ac.commit <- requestID + +} +func (ac *assembleCoordinator) Start() { + log.L(ac.ctx).Info("Starting AssembleCoordinator") + go func() { + for { + select { + case req := <-ac.requests: + requestID := uuid.New().String() + if req.assemblingNode == "" || req.assemblingNode == ac.nodeName { + req.processLocal(ac.ctx, requestID) + } else { + err := req.processRemote(ac.ctx, req.assemblingNode, requestID) + if err != nil { + log.L(ac.ctx).Errorf("AssembleCoordinator request failed: %s", err) + //we failed sending the request so we continue to the next request + // without waiting for this one to complete + // the sequencer event loop is responsible for requesting a new assemble + continue + } + } + + //The actual response is processed on the sequencer event loop. We just need to know when it is safe to proceed + // to the next request + ac.waitForDone(requestID) + case <-ac.stopProcess: + log.L(ac.ctx).Info("assembleCoordinator loop process stopped") + return + case <-ac.ctx.Done(): + log.L(ac.ctx).Info("AssembleCoordinator loop exit due to canceled context") + return + } + } + }() +} + +func (ac *assembleCoordinator) waitForDone(requestID string) { + log.L(ac.ctx).Debugf("AssembleCoordinator:waitForDone %s", requestID) + + // wait for the response or a timeout + timeoutTimer := time.NewTimer(ac.requestTimeout) +out: + for { + select { + case response := <-ac.commit: + if response == requestID { + log.L(ac.ctx).Debugf("AssembleCoordinator:waitForDone received notification of completion %s", requestID) + break out + } else { + // the response was not for this request, must have been an old request that we have already timed out + log.L(ac.ctx).Debugf("AssembleCoordinator:waitForDone received spurious response %s. Continue to wait for %s", response, requestID) + } + case <-ac.ctx.Done(): + log.L(ac.ctx).Info("AssembleCoordinator:waitForDone loop exit due to canceled context") + return + case <-timeoutTimer.C: + log.L(ac.ctx).Errorf("AssembleCoordinator:waitForDone request timeout for request %s", requestID) + //sequencer event loop is responsible for requesting a new assemble + break + } + } + log.L(ac.ctx).Debugf("AssembleCoordinator:waitForDone done %s", requestID) + +} + +func (ac *assembleCoordinator) Stop() { + // try to send an item in `stopProcess` channel, which has a buffer of 1 + // if it already has an item in the channel, this function does nothing + select { + case ac.stopProcess <- true: + default: + } + +} + +// TODO really need to figure out the separation between PrivateTxManager and DomainManager +// to allow us to do the assemble on a separate thread and without worrying about locking the PrivateTransaction objects +// we copy the pertinent structures out of the PrivateTransaction and pass them to the assemble thread +// and then use them to create another private transaction object that is passed to the domain manager which then just unpicks it again +func (ac *assembleCoordinator) QueueAssemble(ctx context.Context, assemblingNode string, transactionID uuid.UUID, transactionInputs *components.TransactionInputs, transactionPreAssembly *components.TransactionPreAssembly) { + + ac.requests <- &assembleRequest{ + assemblingNode: assemblingNode, + assembleCoordinator: ac, + transactionID: transactionID, + transactionInputs: transactionInputs, + transactionPreassembly: transactionPreAssembly, + } + log.L(ctx).Debugf("QueueAssemble: assemble request for %s queued", transactionID) + +} + +func (req *assembleRequest) processLocal(ctx context.Context, requestID string) { + log.L(ctx).Debug("assembleRequest:processLocal") + + req.assembleCoordinator.localAssembler.AssembleLocal(ctx, requestID, req.transactionID, req.transactionInputs, req.transactionPreassembly) + + log.L(ctx).Debug("assembleRequest:processLocal complete") + +} + +func (req *assembleRequest) processRemote(ctx context.Context, assemblingNode string, requestID string) error { + + //Assemble may require a call to another node ( in the case we have been delegated to coordinate transaction for other nodes) + //Usually, they will get sent to us already assembled but there may be cases where we need to re-assemble + // so this needs to be an async step + // however, there must be only one assemble in progress at a time or else there is a risk that 2 transactions could chose to spend the same state + // (TODO - maybe in future, we could further optimize this and allow multiple assembles to be in progress if we can assert that they are not presented with the same available states) + // However, before we do that, we really need to sort out the separation of concerns between the domain manager, state store and private transaction manager and where the responsibility to single thread the assembly stream(s) lies + + log.L(ctx).Debugf("assembleRequest:processRemote requestID %s", requestID) + + stateLocksJSON, err := req.assembleCoordinator.domainContext.ExportStateLocks() + if err != nil { + return err + } + + contractAddressString := req.assembleCoordinator.contractAddress.String() + blockHeight := req.assembleCoordinator.sequencerEnvironment.GetBlockHeight() + log.L(ctx).Debugf("assembleRequest:processRemote Assembling transaction %s on node %s", req.transactionID.String(), assemblingNode) + + //send a request to the node that is responsible for assembling this transaction + err = req.assembleCoordinator.transportWriter.SendAssembleRequest(ctx, assemblingNode, requestID, req.transactionID, contractAddressString, req.transactionInputs, req.transactionPreassembly, stateLocksJSON, blockHeight) + if err != nil { + log.L(ctx).Errorf("assembleRequest:processRemote error from sendAssembleRequest: %s", err) + return err + } + return nil +} diff --git a/core/go/internal/privatetxnmgr/coordinator_selector.go b/core/go/internal/privatetxnmgr/coordinator_selector.go new file mode 100644 index 000000000..8088b3e36 --- /dev/null +++ b/core/go/internal/privatetxnmgr/coordinator_selector.go @@ -0,0 +1,211 @@ +/* + * Copyright © 2024 Kaleido, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package privatetxnmgr + +import ( + "context" + "fmt" + "hash/fnv" + "slices" + + "github.com/hyperledger/firefly-common/pkg/i18n" + "github.com/kaleido-io/paladin/config/pkg/confutil" + "github.com/kaleido-io/paladin/config/pkg/pldconf" + "github.com/kaleido-io/paladin/core/internal/components" + "github.com/kaleido-io/paladin/core/internal/msgs" + "github.com/kaleido-io/paladin/core/internal/privatetxnmgr/ptmgrtypes" + "github.com/kaleido-io/paladin/toolkit/pkg/log" + "github.com/kaleido-io/paladin/toolkit/pkg/prototk" + "github.com/kaleido-io/paladin/toolkit/pkg/tktypes" +) + +// Coordinator selector policy is either +// - coordinator node is statically configured in the contract +// - deterministic and fair rotation between a predefined set of endorsers +// - the sender of the transaction coordinates the transaction +// +// Submitter selection policy is either +// - Coordinator submits +// - Sender submits +// - 3rd party submission + +// Currently only the following combinations are implemented +// 1+1 - core option set for Noto +// 2+1 - core option set for Pente +// 3+2 - core option set for Zeto + +type CoordinatorSelectionMode int + +const ( + BlockHeightRoundRobin CoordinatorSelectionMode = iota + HashedSelection CoordinatorSelectionMode = iota +) + +// Override only intended for unit tests currently +var EndorsementCoordinatorSelectionMode CoordinatorSelectionMode = HashedSelection + +func NewCoordinatorSelector(ctx context.Context, nodeName string, contractConfig *prototk.ContractConfig, sequencerConfig pldconf.PrivateTxManagerSequencerConfig) (ptmgrtypes.CoordinatorSelector, error) { + if contractConfig.GetCoordinatorSelection() == prototk.ContractConfig_COORDINATOR_SENDER { + return &staticCoordinatorSelectorPolicy{ + nodeName: nodeName, + }, nil + } + if contractConfig.GetCoordinatorSelection() == prototk.ContractConfig_COORDINATOR_STATIC { + staticCoordinator := contractConfig.GetStaticCoordinator() + //staticCoordinator must be a fully qualified identity because it is also used to locate the signing key + // but at this point, we only need the node name + staticCoordinatorNode, err := tktypes.PrivateIdentityLocator(staticCoordinator).Node(ctx, false) + if err != nil { + log.L(ctx).Errorf("Error resolving node for static coordinator %s: %s", staticCoordinator, err) + return nil, i18n.NewError(ctx, msgs.MsgPrivateTxManagerInternalError, err) + } + + return &staticCoordinatorSelectorPolicy{ + nodeName: staticCoordinatorNode, + }, nil + } + if contractConfig.GetCoordinatorSelection() == prototk.ContractConfig_COORDINATOR_ENDORSER { + if EndorsementCoordinatorSelectionMode == BlockHeightRoundRobin { + return &roundRobinCoordinatorSelectorPolicy{ + localNode: nodeName, + rangeSize: confutil.Int(sequencerConfig.RoundRobinCoordinatorBlockRangeSize, *pldconf.PrivateTxManagerDefaults.Sequencer.RoundRobinCoordinatorBlockRangeSize), + }, nil + } + // TODO: More work is required to perform leader election of an endorser, so right now a simple hash algorithm is used. + return &endorsementSetHashSelection{ + localNode: nodeName, + }, nil + } + return nil, i18n.NewError(ctx, msgs.MsgDomainInvalidCoordinatorSelection, contractConfig.GetCoordinatorSelection()) +} + +type staticCoordinatorSelectorPolicy struct { + nodeName string +} + +type endorsementSetHashSelection struct { + localNode string + chosenNode string +} + +func (s *staticCoordinatorSelectorPolicy) SelectCoordinatorNode(ctx context.Context, _ *components.PrivateTransaction, environment ptmgrtypes.SequencerEnvironment) (int64, string, error) { + log.L(ctx).Debugf("SelectCoordinatorNode: Selecting coordinator node %s", s.nodeName) + return environment.GetBlockHeight(), s.nodeName, nil +} + +func (s *endorsementSetHashSelection) SelectCoordinatorNode(ctx context.Context, transaction *components.PrivateTransaction, environment ptmgrtypes.SequencerEnvironment) (int64, string, error) { + blockHeight := environment.GetBlockHeight() + if s.chosenNode == "" { + if transaction.PostAssembly == nil { + //if we don't know the candidate nodes, and the transaction hasn't been assembled yet, then we can't select a coordinator so just assume we are the coordinator + // until we get the transaction assembled and then re-evaluate + log.L(ctx).Debug("SelectCoordinatorNode: Assembly not yet completed - using local node for assembly") + return blockHeight, s.localNode, nil + } + //use a map to dedupe as we go + candidateNodesMap := make(map[string]struct{}) + identities := make([]string, 0, len(transaction.PostAssembly.AttestationPlan)) + for _, attestationPlan := range transaction.PostAssembly.AttestationPlan { + if attestationPlan.AttestationType == prototk.AttestationType_ENDORSE { + for _, party := range attestationPlan.Parties { + identity, node, err := tktypes.PrivateIdentityLocator(party).Validate(ctx, s.localNode, false) + if err != nil { + log.L(ctx).Errorf("SelectCoordinatorNode: Error resolving node for party %s: %s", party, err) + return -1, "", i18n.NewError(ctx, msgs.MsgPrivateTxManagerInternalError, err) + } + candidateNodesMap[node] = struct{}{} + identities = append(identities, fmt.Sprintf("%s@%s", identity, node)) + } + } + } + candidateNodes := make([]string, 0, len(candidateNodesMap)) + for candidateNode := range candidateNodesMap { + candidateNodes = append(candidateNodes, candidateNode) + } + slices.Sort(candidateNodes) + slices.Sort(identities) + if len(candidateNodes) == 0 { + log.L(ctx).Warn("SelectCoordinatorNode: No candidate nodes, assuming local node is the coordinator") + return blockHeight, s.localNode, nil + } + // Take a simple numeric hash of the identities string + h := fnv.New32a() + for _, identity := range identities { + h.Write([]byte(identity)) + } + // Use that as an index into the chosen node set + s.chosenNode = candidateNodes[int(h.Sum32())%len(candidateNodes)] + } + + return blockHeight, s.chosenNode, nil + +} + +type roundRobinCoordinatorSelectorPolicy struct { + localNode string + candidateNodes []string + rangeSize int +} + +func (s *roundRobinCoordinatorSelectorPolicy) SelectCoordinatorNode(ctx context.Context, transaction *components.PrivateTransaction, environment ptmgrtypes.SequencerEnvironment) (int64, string, error) { + blockHeight := environment.GetBlockHeight() + + if len(s.candidateNodes) == 0 { + if transaction.PostAssembly == nil { + //if we don't know the candidate nodes, and the transaction hasn't been assembled yet, then we can't select a coordinator so just assume we are the coordinator + // until we get the transaction assembled and then re-evaluate + log.L(ctx).Debug("SelectCoordinatorNode: No candidate nodes, assuming local node is the coordinator") + return blockHeight, s.localNode, nil + } else { + //use a map to dedupe as we go + candidateNodesMap := make(map[string]struct{}) + for _, attestationPlan := range transaction.PostAssembly.AttestationPlan { + if attestationPlan.AttestationType == prototk.AttestationType_ENDORSE { + for _, party := range attestationPlan.Parties { + node, err := tktypes.PrivateIdentityLocator(party).Node(ctx, true) + if err != nil { + log.L(ctx).Errorf("SelectCoordinatorNode: Error resolving node for party %s: %s", party, err) + return -1, "", i18n.NewError(ctx, msgs.MsgPrivateTxManagerInternalError, err) + } + if node == "" { + node = s.localNode + } + candidateNodesMap[node] = struct{}{} + } + } + } + for candidateNode := range candidateNodesMap { + s.candidateNodes = append(s.candidateNodes, candidateNode) + } + slices.Sort(s.candidateNodes) + } + } + + if len(s.candidateNodes) == 0 { + //if we still don't have any candidate nodes, then we can't select a coordinator so just assume we are the coordinator + log.L(ctx).Debug("SelectCoordinatorNode: No candidate nodes, assuming local node is the coordinator") + return blockHeight, s.localNode, nil + } + + rangeIndex := blockHeight / int64(s.rangeSize) + + coordinatorIndex := int(rangeIndex) % len(s.candidateNodes) + coordinatorNode := s.candidateNodes[coordinatorIndex] + log.L(ctx).Debugf("SelectCoordinatorNode: selected coordinator node %s using round robin algorithm for blockHeight: %d and rangeSize %d ", coordinatorNode, blockHeight, s.rangeSize) + + return blockHeight, coordinatorNode, nil + +} diff --git a/core/go/internal/privatetxnmgr/graph.go b/core/go/internal/privatetxnmgr/graph.go index 85f8850ee..54656826f 100644 --- a/core/go/internal/privatetxnmgr/graph.go +++ b/core/go/internal/privatetxnmgr/graph.go @@ -30,7 +30,7 @@ type Graph interface { AddTransaction(ctx context.Context, transaction ptmgrtypes.TransactionFlow) GetDispatchableTransactions(ctx context.Context) (ptmgrtypes.DispatchableTransactions, error) RemoveTransaction(ctx context.Context, txID string) - RemoveTransactions(ctx context.Context, transactionsToRemove ptmgrtypes.DispatchableTransactions) + RemoveTransactions(ctx context.Context, transactionsToRemove []string) IncludesTransaction(txID string) bool } @@ -57,8 +57,8 @@ func NewGraph() Graph { } func (g *graph) AddTransaction(ctx context.Context, transaction ptmgrtypes.TransactionFlow) { - log.L(ctx).Debugf("Adding transaction %s to graph", transaction.ID().String()) - g.allTransactions[transaction.ID().String()] = transaction + log.L(ctx).Debugf("Adding transaction %s to graph", transaction.ID(ctx).String()) + g.allTransactions[transaction.ID(ctx).String()] = transaction } @@ -81,7 +81,7 @@ func (g *graph) buildMatrix(ctx context.Context) error { //for each unique state hash, create an index of its minter and/or spender stateToSpender := make(map[string]*int) for txnIndex, txn := range g.transactions { - for _, stateID := range txn.InputStateIDs() { + for _, stateID := range txn.InputStateIDs(ctx) { if stateToSpender[stateID] != nil { //TODO this is expected in some cases and represents a contention that needs to be resolved //TBC do we assert that it is resolved before we get to this point? @@ -101,11 +101,11 @@ func (g *graph) buildMatrix(ctx context.Context) error { //TODO this is O(n^2) and could be optimised //TODO what about input states that are not output states of any transaction? Do we assume that the minter transactions are already dispatched / // or confirmed? - for _, stateID := range minter.OutputStateIDs() { + for _, stateID := range minter.OutputStateIDs(ctx) { if spenderIndex := stateToSpender[stateID]; spenderIndex != nil { //we have a dependency relationship if log.IsTraceEnabled() { - log.L(ctx).Tracef("Graph.buildMatrix Transaction %s depends on transaction %s", minter.ID().String(), g.transactions[*spenderIndex].ID().String()) + log.L(ctx).Tracef("Graph.buildMatrix Transaction %s depends on transaction %s", minter.ID(ctx).String(), g.transactions[*spenderIndex].ID(ctx).String()) } g.transactionsMatrix[minterIndex][*spenderIndex] = append(g.transactionsMatrix[minterIndex][*spenderIndex], stateID) } @@ -138,7 +138,7 @@ func (g *graph) GetDispatchableTransactions(ctx context.Context) (ptmgrtypes.Dis queue := make([]int, 0, len(g.transactionsMatrix)) //find all independent transactions - that have no input states in this graph and then do a breadth first search // of each of them to find all of its dependent transactions that are also dispatchable and recurse - dispatchable := make([]string, 0, len(g.transactionsMatrix)) + dispatchable := make([]ptmgrtypes.TransactionFlow, 0, len(g.transactionsMatrix)) //i.e. the input states are the output of transactions that are either in the dispatch stage or have been confirmed // calcaulate the number of dependencies of each transaction @@ -156,7 +156,7 @@ func (g *graph) GetDispatchableTransactions(ctx context.Context) (ptmgrtypes.Dis if indegree == 0 { if log.IsTraceEnabled() { - log.L(ctx).Tracef("Graph.GetDispatchableTransactions Transaction %s has no dependencies", g.transactions[txnIndex].ID().String()) + log.L(ctx).Tracef("Graph.GetDispatchableTransactions Transaction %s has no dependencies", g.transactions[txnIndex].ID(ctx).String()) } queue = append(queue, txnIndex) } @@ -172,17 +172,17 @@ func (g *graph) GetDispatchableTransactions(ctx context.Context) (ptmgrtypes.Dis if !g.transactions[nextTransaction].IsEndorsed(ctx) { //this transaction is not endorsed, so we cannot dispatch it if log.IsTraceEnabled() { - log.L(ctx).Tracef("Graph.GetDispatchableTransactions Transaction %s not endorsed so cannot be dispatched", g.transactions[nextTransaction].ID().String()) + log.L(ctx).Tracef("Graph.GetDispatchableTransactions Transaction %s not endorsed so cannot be dispatched", g.transactions[nextTransaction].ID(ctx).String()) } continue } else { if log.IsTraceEnabled() { - log.L(ctx).Tracef("Graph.GetDispatchableTransactions Transaction %s is endorsed and will be dispatched", g.transactions[nextTransaction].ID().String()) + log.L(ctx).Tracef("Graph.GetDispatchableTransactions Transaction %s is endorsed and will be dispatched", g.transactions[nextTransaction].ID(ctx).String()) } } //transaction can be dispatched - dispatchable = append(dispatchable, g.transactions[nextTransaction].ID().String()) + dispatchable = append(dispatchable, g.transactions[nextTransaction]) //get this transaction's dependencies dependencies := g.transactionsMatrix[nextTransaction] @@ -192,7 +192,7 @@ func (g *graph) GetDispatchableTransactions(ctx context.Context) (ptmgrtypes.Dis indegrees[dependant]-- if indegrees[dependant] == 0 { if log.IsTraceEnabled() { - log.L(ctx).Tracef("Graph.GetDispatchableTransactions Transaction %s dependencies are being dispatched", g.transactions[dependant].ID().String()) + log.L(ctx).Tracef("Graph.GetDispatchableTransactions Transaction %s dependencies are being dispatched", g.transactions[dependant].ID(ctx).String()) } // add the dependant to the queue queue = append(queue, dependant) @@ -206,22 +206,22 @@ func (g *graph) GetDispatchableTransactions(ctx context.Context) (ptmgrtypes.Dis // across signing keys if len(dispatchable) > 0 { - signingAddress := g.allTransactions[dispatchable[0]].Signer() + signingAddress := g.allTransactions[dispatchable[0].ID(ctx).String()].Signer(ctx) log.L(ctx).Debugf("Graph.GetDispatchableTransactions %d dispatchable transactions", len(dispatchable)) - return map[string][]string{ + return map[string][]ptmgrtypes.TransactionFlow{ signingAddress: dispatchable, }, nil } log.L(ctx).Debug("Graph.GetDispatchableTransactions No dispatchable transactions") - return map[string][]string{}, nil + return map[string][]ptmgrtypes.TransactionFlow{}, nil } func (g *graph) RemoveTransaction(ctx context.Context, txID string) { log.L(ctx).Debugf("Graph.RemoveTransaction Removing transaction %s from graph", txID) delete(g.allTransactions, txID) } -func (g *graph) RemoveTransactions(ctx context.Context, transactionsToRemove ptmgrtypes.DispatchableTransactions) { +func (g *graph) RemoveTransactions(ctx context.Context, transactionIDsToRemove []string) { log.L(ctx).Debugf("Graph.RemoveTransactions Removing transactions from graph") // no validation performed here // it is valid to remove transactions that have dependents. In fact that is normal. @@ -230,13 +230,11 @@ func (g *graph) RemoveTransactions(ctx context.Context, transactionsToRemove ptm // maybe they got reverted before being endorsed or whatever it is not the concern of the graph to validate this // the graph just gets redrawn based on the dependencies that remain after a transaction is removed - for _, sequence := range transactionsToRemove { - for _, txID := range sequence { - if g.allTransactions[txID] == nil { - log.L(ctx).Infof("Transaction %s already removed", txID) - } else { - delete(g.allTransactions, txID) - } + for _, transactionID := range transactionIDsToRemove { + if g.allTransactions[transactionID] == nil { + log.L(ctx).Infof("Transaction %s already removed", transactionID) + } else { + delete(g.allTransactions, transactionID) } } } diff --git a/core/go/internal/privatetxnmgr/graph_test.go b/core/go/internal/privatetxnmgr/graph_test.go index cfde3125d..4baa8c145 100644 --- a/core/go/internal/privatetxnmgr/graph_test.go +++ b/core/go/internal/privatetxnmgr/graph_test.go @@ -30,11 +30,11 @@ import ( func NewMockTransactionProcessorForTesting(t *testing.T, transactionID uuid.UUID, inputStateIDs []string, outputStateIDs []string, endorsed bool, signer string) *privatetxnmgrmocks.TransactionFlow { mockTransactionProcessor := privatetxnmgrmocks.NewTransactionFlow(t) - mockTransactionProcessor.On("ID").Return(transactionID).Maybe() - mockTransactionProcessor.On("InputStateIDs").Return(inputStateIDs).Maybe() - mockTransactionProcessor.On("OutputStateIDs").Return(outputStateIDs).Maybe() - mockTransactionProcessor.On("IsEndorsed", mock.Anything).Return(endorsed).Maybe() - mockTransactionProcessor.On("Signer").Return(signer).Maybe() + mockTransactionProcessor.On("ID", mock.Anything).Return(transactionID).Maybe() + mockTransactionProcessor.On("InputStateIDs", mock.Anything).Return(inputStateIDs).Maybe() + mockTransactionProcessor.On("OutputStateIDs", mock.Anything).Return(outputStateIDs).Maybe() + mockTransactionProcessor.On("IsEndorsed", mock.Anything, mock.Anything).Return(endorsed).Maybe() + mockTransactionProcessor.On("Signer", mock.Anything).Return(signer).Maybe() return mockTransactionProcessor } @@ -90,7 +90,7 @@ func TestRemoveTransactions(t *testing.T) { testGraph.AddTransaction(ctx, mockTransactionProcessor2) testGraph.AddTransaction(ctx, mockTransactionProcessor3) - testGraph.RemoveTransactions(ctx, map[string][]string{signer: {"tx1", "tx2"}}) + testGraph.RemoveTransactions(ctx, []string{"tx1", "tx2"}) assert.True(t, testGraph.IncludesTransaction(TxID0.String())) assert.True(t, testGraph.IncludesTransaction(TxID1.String())) @@ -148,11 +148,11 @@ func TestScenario1(t *testing.T) { //make sure they come out in the expected order // the absolute order does not matter so long as dependencies come before dependants. // so we expect either 0,1,3 or 1,0,3. - assert.True(t, ((dispatchableTransactions[0] == TxID0.String() && dispatchableTransactions[1] == TxID1.String()) || - (dispatchableTransactions[0] == TxID1.String() && dispatchableTransactions[1] == TxID0.String()))) + assert.True(t, ((dispatchableTransactions[0].ID(ctx) == TxID0 && dispatchableTransactions[1].ID(ctx) == TxID1) || + (dispatchableTransactions[0].ID(ctx) == TxID1 && dispatchableTransactions[1].ID(ctx) == TxID0))) //transaction 2 is not endorsed so should not be in the dispatchable list - assert.Equal(t, TxID3.String(), dispatchableTransactions[2]) + assert.Equal(t, TxID3, dispatchableTransactions[2].ID(ctx)) // GetDispatchableTransactions is a read only operation so we can call it again and get the same result dispatchable, err = testGraph.GetDispatchableTransactions(ctx) @@ -165,11 +165,11 @@ func TestScenario1(t *testing.T) { //make sure they come out in the expected order // the absolute order does not matter so long as dependencies come before dependants. // so we expect either 0,1,3 or 1,0,3. - assert.True(t, ((dispatchableTransactions[0] == TxID0.String() && dispatchableTransactions[1] == TxID1.String()) || - (dispatchableTransactions[0] == TxID1.String() && dispatchableTransactions[1] == TxID0.String()))) - assert.Equal(t, TxID3.String(), dispatchableTransactions[2]) + assert.True(t, ((dispatchableTransactions[0].ID(ctx) == TxID0 && dispatchableTransactions[1].ID(ctx) == TxID1) || + (dispatchableTransactions[0].ID(ctx) == TxID1 && dispatchableTransactions[1].ID(ctx) == TxID0))) + assert.Equal(t, TxID3, dispatchableTransactions[2].ID(ctx)) - testGraph.RemoveTransactions(ctx, dispatchable) + testGraph.RemoveTransactions(ctx, dispatchable.IDs(ctx)) dispatchable, err = testGraph.GetDispatchableTransactions(ctx) require.NoError(t, err) @@ -230,7 +230,7 @@ func TestScenario2(t *testing.T) { isBefore := func(tx1, tx2 string) bool { foundTx1 := false for _, tx := range dispatchableTransactions { - switch tx { + switch tx.ID(ctx).String() { case tx1: foundTx1 = true case tx2: diff --git a/core/go/internal/privatetxnmgr/private_txn_mgr.go b/core/go/internal/privatetxnmgr/private_txn_mgr.go index dd1cfa45d..b10382bb3 100644 --- a/core/go/internal/privatetxnmgr/private_txn_mgr.go +++ b/core/go/internal/privatetxnmgr/private_txn_mgr.go @@ -18,9 +18,9 @@ package privatetxnmgr import ( "context" "encoding/json" - "fmt" "sync" + "github.com/google/uuid" "github.com/hyperledger/firefly-common/pkg/i18n" "github.com/hyperledger/firefly-signer/pkg/abi" "github.com/kaleido-io/paladin/core/internal/components" @@ -32,6 +32,7 @@ import ( "github.com/kaleido-io/paladin/core/internal/msgs" + "github.com/kaleido-io/paladin/core/pkg/blockindexer" pbEngine "github.com/kaleido-io/paladin/core/pkg/proto/engine" "github.com/kaleido-io/paladin/config/pkg/confutil" @@ -59,17 +60,27 @@ type privateTxManager struct { syncPoints syncpoints.SyncPoints stateDistributer statedistribution.StateDistributer preparedTransactionDistributer preparedtxdistribution.PreparedTransactionDistributer + blockHeight int64 } // Init implements Engine. func (p *privateTxManager) PreInit(c components.PreInitComponents) (*components.ManagerInitResult, error) { - return &components.ManagerInitResult{}, nil + return &components.ManagerInitResult{ + PreCommitHandler: func(ctx context.Context, _ *gorm.DB, blocks []*pldapi.IndexedBlock, transactions []*blockindexer.IndexedTransactionNotify) (blockindexer.PostCommit, error) { + log.L(ctx).Debug("PrivateTxManager PreCommitHandler") + latestBlockNumber := blocks[len(blocks)-1].Number + return func() { + log.L(ctx).Debugf("PrivateTxManager PostCommitHandler: %d", latestBlockNumber) + p.OnNewBlockHeight(ctx, latestBlockNumber) + }, nil + }, + }, nil } func (p *privateTxManager) PostInit(c components.AllComponents) error { p.components = c p.nodeName = p.components.TransportManager().LocalNodeName() - p.syncPoints = syncpoints.NewSyncPoints(p.ctx, &p.config.Writer, c.Persistence(), c.TxManager()) + p.syncPoints = syncpoints.NewSyncPoints(p.ctx, &p.config.Writer, c.Persistence(), c.TxManager(), c.PublicTxManager()) p.stateDistributer = statedistribution.NewStateDistributer( p.ctx, p.components.TransportManager(), @@ -117,7 +128,25 @@ func NewPrivateTransactionMgr(ctx context.Context, config *pldconf.PrivateTxMana return p } -func (p *privateTxManager) getSequencerForContract(ctx context.Context, contractAddr tktypes.EthAddress, domainAPI components.DomainSmartContract) (oc *Sequencer, err error) { +func (p *privateTxManager) OnNewBlockHeight(ctx context.Context, blockHeight int64) { + p.blockHeight = blockHeight + + p.sequencersLock.RLock() + defer p.sequencersLock.RUnlock() + for _, sequencer := range p.sequencers { + sequencer.OnNewBlockHeight(ctx, blockHeight) + } +} + +func (p *privateTxManager) getSequencerForContract(ctx context.Context, dbTX *gorm.DB, contractAddr tktypes.EthAddress, domainAPI components.DomainSmartContract) (oc *Sequencer, err error) { + + if domainAPI == nil { + domainAPI, err = p.components.DomainManager().GetSmartContractByAddress(ctx, dbTX, contractAddr) + if err != nil { + log.L(ctx).Errorf("Failed to get domain smart contract for contract address %s: %s", contractAddr, err) + return nil, err + } + } readlock := true p.sequencersLock.RLock() @@ -137,30 +166,36 @@ func (p *privateTxManager) getSequencerForContract(ctx context.Context, contract transportWriter := NewTransportWriter(domainAPI.Domain().Name(), &contractAddr, p.nodeName, p.components.TransportManager()) publisher := NewPublisher(p, contractAddr.String()) - endorsementGatherer, err := p.getEndorsementGathererForContract(ctx, contractAddr) + endorsementGatherer, err := p.getEndorsementGathererForContract(ctx, dbTX, contractAddr) if err != nil { log.L(ctx).Errorf("Failed to get endorsement gatherer for contract %s: %s", contractAddr.String(), err) return nil, err } - p.sequencers[contractAddr.String()] = - NewSequencer( - p.ctx, - p, - p.nodeName, - contractAddr, - &p.config.Sequencer, - p.components, - domainAPI, - endorsementGatherer, - publisher, - p.syncPoints, - p.components.IdentityResolver(), - p.stateDistributer, - p.preparedTransactionDistributer, - transportWriter, - confutil.DurationMin(p.config.RequestTimeout, 0, *pldconf.PrivateTxManagerDefaults.RequestTimeout), - ) + newSequencer, err := NewSequencer( + p.ctx, + p, + p.nodeName, + contractAddr, + &p.config.Sequencer, + p.components, + domainAPI, + endorsementGatherer, + publisher, + p.syncPoints, + p.components.IdentityResolver(), + p.stateDistributer, + p.preparedTransactionDistributer, + transportWriter, + confutil.DurationMin(p.config.RequestTimeout, 0, *pldconf.PrivateTxManagerDefaults.RequestTimeout), + p.blockHeight, + ) + if err != nil { + log.L(ctx).Errorf("Failed to create sequencer for contract %s: %s", contractAddr.String(), err) + return nil, err + } + p.sequencers[contractAddr.String()] = newSequencer + sequencerDone, err := p.sequencers[contractAddr.String()].Start(ctx) if err != nil { log.L(ctx).Errorf("Failed to start sequencer for contract %s: %s", contractAddr.String(), err) @@ -180,9 +215,11 @@ func (p *privateTxManager) getSequencerForContract(ctx context.Context, contract return p.sequencers[contractAddr.String()], nil } -func (p *privateTxManager) getEndorsementGathererForContract(ctx context.Context, contractAddr tktypes.EthAddress) (ptmgrtypes.EndorsementGatherer, error) { - - domainSmartContract, err := p.components.DomainManager().GetSmartContractByAddress(ctx, contractAddr) +func (p *privateTxManager) getEndorsementGathererForContract(ctx context.Context, dbTX *gorm.DB, contractAddr tktypes.EthAddress) (ptmgrtypes.EndorsementGatherer, error) { + // We need to have this as a function of the PrivateTransactionManager rather than a function of the sequencer because the endorsement gatherer is needed + // even if we don't have a sequencer. e.g. maybe the transaction is being coordinated by another node and this node has just been asked to endorse it + // in that case, we need to make sure that we are using the domainContext provided by the endorsement request + domainSmartContract, err := p.components.DomainManager().GetSmartContractByAddress(ctx, dbTX, contractAddr) if err != nil { return nil, err } @@ -195,7 +232,7 @@ func (p *privateTxManager) getEndorsementGathererForContract(ctx context.Context return p.endorsementGatherers[contractAddr.String()], nil } -func (p *privateTxManager) HandleNewTx(ctx context.Context, txi *components.ValidatedTransaction) error { +func (p *privateTxManager) HandleNewTx(ctx context.Context, dbTX *gorm.DB, txi *components.ValidatedTransaction) error { tx := txi.Transaction if tx.To == nil { if txi.Transaction.SubmitMode.V() != pldapi.SubmitModeAuto { @@ -212,17 +249,20 @@ func (p *privateTxManager) HandleNewTx(ctx context.Context, txi *components.Vali if txi.Transaction.SubmitMode.V() == pldapi.SubmitModeExternal { intent = prototk.TransactionSpecification_PREPARE_TRANSACTION } - return p.handleNewTx(ctx, &components.PrivateTransaction{ + if txi.Function == nil || txi.Function.Definition == nil { + return i18n.NewError(ctx, msgs.MsgPrivateTxMgrFunctionNotProvided) + } + return p.handleNewTx(ctx, dbTX, &components.PrivateTransaction{ ID: *tx.ID, Inputs: &components.TransactionInputs{ - Domain: tx.Domain, - From: tx.From, - To: *tx.To, - Function: txi.Function.Definition, - Inputs: txi.Inputs, - Intent: intent, + Domain: tx.Domain, + From: tx.From, + To: *tx.To, + Function: txi.Function.Definition, + Inputs: txi.Inputs, + Intent: intent, + PublicTxOptions: txi.Transaction.PublicTxOptions, }, - PublicTxOptions: tx.PublicTxOptions, }) } @@ -234,11 +274,8 @@ func (p *privateTxManager) HandleNewTx(ctx context.Context, txi *components.Vali // // We are currently proving out this pattern on the boundary of the private transaction manager and the public transaction manager and once that has settled, we will implement the same pattern here. // In the meantime, we a single function to submit a transaction and there is currently no persistence of the submission record. It is all held in memory only -func (p *privateTxManager) handleNewTx(ctx context.Context, tx *components.PrivateTransaction) error { +func (p *privateTxManager) handleNewTx(ctx context.Context, dbTX *gorm.DB, tx *components.PrivateTransaction) error { log.L(ctx).Debugf("Handling new transaction: %v", tx) - if tx.Inputs == nil { - return i18n.NewError(ctx, msgs.MsgDomainNotProvided) - } emptyAddress := tktypes.EthAddress{} if tx.Inputs.To == emptyAddress { @@ -246,7 +283,7 @@ func (p *privateTxManager) handleNewTx(ctx context.Context, tx *components.Priva } contractAddr := tx.Inputs.To - domainAPI, err := p.components.DomainManager().GetSmartContractByAddress(ctx, contractAddr) + domainAPI, err := p.components.DomainManager().GetSmartContractByAddress(ctx, dbTX, contractAddr) if err != nil { return err } @@ -266,7 +303,7 @@ func (p *privateTxManager) handleNewTx(ctx context.Context, tx *components.Priva return i18n.NewError(ctx, msgs.MsgPrivateTxManagerInternalError, "PreAssembly is nil") } - oc, err := p.getSequencerForContract(ctx, contractAddr, domainAPI) + oc, err := p.getSequencerForContract(ctx, dbTX, contractAddr, domainAPI) if err != nil { return err } @@ -295,22 +332,28 @@ func (p *privateTxManager) validateDelegatedTransaction(ctx context.Context, tx } -func (p *privateTxManager) handleDelegatedTransaction(ctx context.Context, tx *components.PrivateTransaction) error { +func (p *privateTxManager) handleDelegatedTransaction(ctx context.Context, dbTX *gorm.DB, delegationBlockHeight int64, delegatingNodeName string, delegationId string, tx *components.PrivateTransaction) error { log.L(ctx).Debugf("Handling delegated transaction: %v", tx) contractAddr := tx.Inputs.To - domainAPI, err := p.components.DomainManager().GetSmartContractByAddress(ctx, contractAddr) + domainAPI, err := p.components.DomainManager().GetSmartContractByAddress(ctx, dbTX, contractAddr) if err != nil { + log.L(ctx).Errorf("handleDelegatedTransaction: Failed to get domain smart contract for contract address %s: %s", contractAddr, err) return err } - oc, err := p.getSequencerForContract(ctx, contractAddr, domainAPI) + sequencer, err := p.getSequencerForContract(ctx, dbTX, contractAddr, domainAPI) if err != nil { return err } - queued := oc.ProcessInFlightTransaction(ctx, tx) + queued := sequencer.ProcessInFlightTransaction(ctx, tx, &delegationBlockHeight) if queued { log.L(ctx).Debugf("Delegated Transaction with ID %s queued in database", tx.ID) } + err = sequencer.transportWriter.SendDelegationRequestAcknowledgment(ctx, delegatingNodeName, delegationId, p.nodeName, tx.ID.String()) + if err != nil { + log.L(ctx).Errorf("Failed to send delegation request acknowledgment: %s", err) + // if we can't send the acknowledgment, the sender will retry + } return nil } @@ -333,16 +376,28 @@ func (p *privateTxManager) handleDeployTx(ctx context.Context, tx *components.Pr return i18n.WrapError(ctx, err, msgs.MsgDeployInitFailed) } - // NOTE unlike private transactions, we assume that all verifiers are resolved locally + // this is a transaction that will confirm just like invoke transactions + // unlike invoke transactions, we don't yet have the sequencer thread to dispatch to so we start a new go routine for each deployment + // TODO - should have a pool of deployment threads? Maybe size of pool should be one? Or at least one per domain? + go p.deploymentLoop(log.WithLogField(p.ctx, "role", "deploy-loop"), domain, tx) + + return nil +} + +func (p *privateTxManager) deploymentLoop(ctx context.Context, domain components.Domain, tx *components.PrivateContractDeploy) { + log.L(ctx).Info("Starting deployment loop") + + var err error - //Resolve keys synchronously so that we can return an error if any key resolution fails + // Resolve keys synchronously on this go routine so that we can return an error if any key resolution fails tx.Verifiers = make([]*prototk.ResolvedVerifier, len(tx.RequiredVerifiers)) for i, v := range tx.RequiredVerifiers { // TODO: This is a synchronous cross-node exchange, done sequentially for each verifier. // Potentially needs to move to an event-driven model like on invocation. - verifier, err := p.components.IdentityResolver().ResolveVerifier(ctx, v.Lookup, v.Algorithm, v.VerifierType) - if err != nil { - return i18n.WrapError(ctx, err, msgs.MsgKeyResolutionFailed, v.Lookup, v.Algorithm, v.VerifierType) + verifier, resolveErr := p.components.IdentityResolver().ResolveVerifier(ctx, v.Lookup, v.Algorithm, v.VerifierType) + if resolveErr != nil { + err = i18n.WrapError(ctx, resolveErr, msgs.MsgKeyResolutionFailed, v.Lookup, v.Algorithm, v.VerifierType) + break } tx.Verifiers[i] = &prototk.ResolvedVerifier{ Lookup: v.Lookup, @@ -352,16 +407,9 @@ func (p *privateTxManager) handleDeployTx(ctx context.Context, tx *components.Pr } } - // this is a transaction that will confirm just like invoke transactions - // unlike invoke transactions, we don't yet have the sequencer thread to dispatch to so we start a new go routine for each deployment - // TODO - should have a pool of deployment threads? Maybe size of pool should be one? Or at least one per domain? - go p.deploymentLoop(log.WithLogField(p.ctx, "role", "deploy-loop"), domain, tx) - - return nil -} -func (p *privateTxManager) deploymentLoop(ctx context.Context, domain components.Domain, tx *components.PrivateContractDeploy) { - log.L(ctx).Info("Starting deployment loop") - err := p.evaluateDeployment(ctx, domain, tx) + if err == nil { + err = p.evaluateDeployment(ctx, domain, tx) + } if err != nil { log.L(ctx).Errorf("Error evaluating deployment: %s", err) return @@ -442,19 +490,11 @@ func (p *privateTxManager) evaluateDeployment(ctx context.Context, domain compon return p.revertDeploy(ctx, tx, i18n.NewError(ctx, msgs.MsgPrivateTxManagerInternalError, "Neither InvokeTransaction nor DeployTransaction set")) } - pubBatch, err := publicTransactionEngine.PrepareSubmissionBatch(ctx, publicTXs) - if err != nil { - return p.revertDeploy(ctx, tx, i18n.WrapError(ctx, err, msgs.MsgPrivateTxManagerInternalError, "PrepareSubmissionBatch failed")) - } - - // Must make sure from this point we return the nonces - completed := false // and include whether we committed the DB transaction or not - defer func() { - pubBatch.Completed(ctx, completed) - }() - if len(pubBatch.Rejected()) > 0 { - // We do not handle partial success - roll everything back - return p.revertDeploy(ctx, tx, i18n.WrapError(ctx, pubBatch.Rejected()[0].RejectedError(), msgs.MsgPrivateTxManagerInternalError, "Submission batch rejected ")) + for _, pubTx := range publicTXs { + err := publicTransactionEngine.ValidateTransaction(ctx, p.components.Persistence().DB(), pubTx) + if err != nil { + return p.revertDeploy(ctx, tx, i18n.WrapError(ctx, err, msgs.MsgPrivateTxManagerInternalError, "PrepareSubmissionBatch failed")) + } } //transactions are always dispatched as a sequence, even if only a sequence of one @@ -465,7 +505,7 @@ func (p *privateTxManager) evaluateDeployment(ctx context.Context, domain compon }, }, } - sequence.PublicTxBatch = pubBatch + sequence.PublicTxs = publicTXs dispatchBatch := &syncpoints.DispatchBatch{ PublicDispatches: []*syncpoints.PublicDispatch{ sequence, @@ -479,8 +519,6 @@ func (p *privateTxManager) evaluateDeployment(ctx context.Context, domain compon return p.revertDeploy(ctx, tx, err) } - completed = true - p.publishToSubscribers(ctx, &components.TransactionDispatchedEvent{ TransactionID: tx.ID.String(), Nonce: uint64(0), /*TODO*/ @@ -491,16 +529,18 @@ func (p *privateTxManager) evaluateDeployment(ctx context.Context, domain compon } -func (p *privateTxManager) GetTxStatus(ctx context.Context, domainAddress string, txID string) (status components.PrivateTxStatus, err error) { +func (p *privateTxManager) GetTxStatus(ctx context.Context, domainAddress string, txID uuid.UUID) (status components.PrivateTxStatus, err error) { // this returns status that we happen to have in memory at the moment and might be useful for debugging p.sequencersLock.RLock() defer p.sequencersLock.RUnlock() targetSequencer := p.sequencers[domainAddress] if targetSequencer == nil { - //TODO should be valid to query the status of a transaction that belongs to a domain instance that is not currently active - errorMessage := fmt.Sprintf("Sequencer not found for domain address %s", domainAddress) - return components.PrivateTxStatus{}, i18n.NewError(ctx, msgs.MsgPrivateTxManagerInternalError, errorMessage) + return components.PrivateTxStatus{ + TxID: txID.String(), + Status: "unknown", + }, nil + } else { return targetSequencer.GetTxStatus(ctx, txID) } @@ -532,7 +572,7 @@ func (p *privateTxManager) handleEndorsementRequest(ctx context.Context, message return } - endorsementGatherer, err := p.getEndorsementGathererForContract(ctx, *contractAddress) + endorsementGatherer, err := p.getEndorsementGathererForContract(ctx, p.components.Persistence().DB(), *contractAddress) if err != nil { log.L(ctx).Errorf("Failed to get endorsement gatherer for contract address %s: %s", contractAddressString, err) return @@ -641,10 +681,13 @@ func (p *privateTxManager) handleEndorsementRequest(ctx context.Context, message } endorsementResponse := &pbEngine.EndorsementResponse{ - ContractAddress: contractAddressString, - TransactionId: endorsementRequest.TransactionId, - Endorsement: endorsementAny, - RevertReason: revertReason, + IdempotencyKey: endorsementRequest.IdempotencyKey, + ContractAddress: contractAddressString, + TransactionId: endorsementRequest.TransactionId, + Endorsement: endorsementAny, + RevertReason: revertReason, + Party: endorsementRequest.Party, + AttestationRequestName: attestationRequest.Name, } endorsementResponseBytes, err := proto.Marshal(endorsementResponse) if err != nil { @@ -657,7 +700,7 @@ func (p *privateTxManager) handleEndorsementRequest(ctx context.Context, message ReplyTo: p.nodeName, Payload: endorsementResponseBytes, Node: replyTo, - Component: PRIVATE_TX_MANAGER_DESTINATION, + Component: components.PRIVATE_TX_MANAGER_DESTINATION, }) if err != nil { log.L(ctx).Errorf("Failed to send endorsement response: %s", err) @@ -665,7 +708,7 @@ func (p *privateTxManager) handleEndorsementRequest(ctx context.Context, message } } -func (p *privateTxManager) handleDelegationRequest(ctx context.Context, messagePayload []byte) { +func (p *privateTxManager) handleDelegationRequest(ctx context.Context, messagePayload []byte, replyTo string) { delegationRequest := &pbEngine.DelegationRequest{} err := proto.Unmarshal(messagePayload, delegationRequest) if err != nil { @@ -686,16 +729,34 @@ func (p *privateTxManager) handleDelegationRequest(ctx context.Context, messageP return } - //TODO persist the delegated transaction and only continue once it has been persisted - - err = p.handleDelegatedTransaction(ctx, transaction) + //TODO not quite figured out how to receive an assembled transaction because it will have been assembled + // in the domain context of the sender. In some cases, it will be using committed states so that will be ok. + // for now, in the interest of simplicity, we just trash the PostAssembly and start again + transaction.PostAssembly = nil + err = p.handleDelegatedTransaction(ctx, p.components.Persistence().DB(), delegationRequest.BlockHeight, replyTo, delegationRequest.DelegationId, transaction) if err != nil { log.L(ctx).Errorf("Failed to handle delegated transaction: %s", err) // do not send an ack and let the sender retry return } +} + +func (p *privateTxManager) handleDelegationRequestAcknowledgment(ctx context.Context, messagePayload []byte) { + delegationRequestAcknowledgment := &pbEngine.DelegationRequestAcknowledgment{} + err := proto.Unmarshal(messagePayload, delegationRequestAcknowledgment) + if err != nil { + log.L(ctx).Errorf("Failed to unmarshal delegation request acknowledgment: %s", err) + return + } + + p.HandleNewEvent(ctx, &ptmgrtypes.TransactionDelegationAcknowledgedEvent{ + PrivateTransactionEventBase: ptmgrtypes.PrivateTransactionEventBase{ + TransactionID: delegationRequestAcknowledgment.TransactionId, + ContractAddress: delegationRequestAcknowledgment.ContractAddress, + }, + DelegationRequestID: delegationRequestAcknowledgment.DelegationId, + }) - //TODO send an ack } func (p *privateTxManager) handleEndorsementResponse(ctx context.Context, messagePayload []byte) { @@ -725,12 +786,203 @@ func (p *privateTxManager) handleEndorsementResponse(ctx context.Context, messag TransactionID: endorsementResponse.TransactionId, ContractAddress: contractAddressString, }, - RevertReason: revertReason, - Endorsement: endorsement, + RevertReason: revertReason, + Endorsement: endorsement, + Party: endorsementResponse.Party, + AttestationRequestName: endorsementResponse.AttestationRequestName, + IdempotencyKey: endorsementResponse.IdempotencyKey, }) } +func (p *privateTxManager) sendAssembleError(ctx context.Context, node string, assembleRequestId string, contractAddress string, transactionID string, err error) { + + assembleError := &pbEngine.AssembleError{ + ContractAddress: contractAddress, + AssembleRequestId: assembleRequestId, + TransactionId: transactionID, + ErrorMessage: err.Error(), + } + assembleErrorBytes, err := proto.Marshal(assembleError) + if err != nil { + log.L(ctx).Errorf("Failed to marshal assemble error: %s", err) + return + } + + log.L(ctx).Infof("Sending Assemble Error: ContractAddress: %s, TransactionId: %s, AssembleRequestId %s, Error: %s", contractAddress, transactionID, assembleRequestId, assembleError.ErrorMessage) + + err = p.components.TransportManager().Send(ctx, &components.TransportMessage{ + MessageType: "AssembleError", + ReplyTo: p.nodeName, + Payload: assembleErrorBytes, + Node: node, + Component: components.PRIVATE_TX_MANAGER_DESTINATION, + }) + if err != nil { + log.L(ctx).Errorf("Failed to send assemble error: %s", err) + return + } +} + +func (p *privateTxManager) handleAssembleRequest(ctx context.Context, messagePayload []byte, replyTo string) { + + assembleRequest := &pbEngine.AssembleRequest{} + err := proto.Unmarshal(messagePayload, assembleRequest) + if err != nil { + log.L(ctx).Errorf("Failed to unmarshal assembleRequest: %s", err) + return + } + + transactionIDString := assembleRequest.TransactionId + transactionID, err := uuid.Parse(transactionIDString) + if err != nil { + log.L(ctx).Errorf("Failed to parse transaction ID: %s", err) + return + } + + contractAddressString := assembleRequest.ContractAddress + contractAddress, err := tktypes.ParseEthAddress(contractAddressString) + if err != nil { + log.L(ctx).Errorf("Failed to parse contract address: %s", err) + return + } + + // now we have enough info from the request, at least to send an error if we can't proceed + // but until this point any errors result in a silent failure and we assume the coordinator will eventually timeout + // and retry the request + + transactionInputs := &components.TransactionInputs{} + err = json.Unmarshal(assembleRequest.TransactionInputs, transactionInputs) + if err != nil { + log.L(ctx).Errorf("Failed to unmarshal transaction inputs: %s", err) + p.sendAssembleError(ctx, replyTo, assembleRequest.AssembleRequestId, assembleRequest.ContractAddress, assembleRequest.TransactionId, err) + return + } + + preAssembly := &components.TransactionPreAssembly{} + err = json.Unmarshal(assembleRequest.PreAssembly, preAssembly) + if err != nil { + log.L(ctx).Errorf("Failed to unmarshal preAssembly: %s", err) + p.sendAssembleError(ctx, replyTo, assembleRequest.AssembleRequestId, assembleRequest.ContractAddress, assembleRequest.TransactionId, err) + return + } + + sequencer, err := p.getSequencerForContract(ctx, p.components.Persistence().DB(), *contractAddress, nil) // this is just to make sure the sequencer is running + if err != nil { + log.L(ctx).Errorf("Failed to get sequencer for contract address %s: %s", contractAddressString, err) + p.sendAssembleError(ctx, replyTo, assembleRequest.AssembleRequestId, assembleRequest.ContractAddress, assembleRequest.TransactionId, err) + return + } + + postAssembly, err := sequencer.assembleForRemoteCoordinator(ctx, transactionID, transactionInputs, preAssembly, assembleRequest.StateLocks, assembleRequest.BlockHeight) + if err != nil { + log.L(ctx).Errorf("Failed to assemble for coordinator: %s", err) + p.sendAssembleError(ctx, replyTo, assembleRequest.AssembleRequestId, assembleRequest.ContractAddress, assembleRequest.TransactionId, err) + return + } + + postAssemblyBytes, err := json.Marshal(postAssembly) + if err != nil { + log.L(ctx).Errorf("Failed to marshal post assembly: %s", err) + p.sendAssembleError(ctx, replyTo, assembleRequest.AssembleRequestId, assembleRequest.ContractAddress, assembleRequest.TransactionId, err) + return + } + + //Send success assemble response. This is a best can do effort, and no attempt to make the response delivery reliable + // in worst case scenario, the coordinator will time out and retry the request + + assembleResponse := &pbEngine.AssembleResponse{ + ContractAddress: contractAddressString, + AssembleRequestId: assembleRequest.AssembleRequestId, + TransactionId: transactionIDString, + PostAssembly: postAssemblyBytes, + } + assembleResponseBytes, err := proto.Marshal(assembleResponse) + if err != nil { + log.L(ctx).Errorf("Failed to marshal assemble response: %s", err) + return + } + + err = p.components.TransportManager().Send(ctx, &components.TransportMessage{ + MessageType: "AssembleResponse", + ReplyTo: p.nodeName, + Payload: assembleResponseBytes, + Node: replyTo, + Component: components.PRIVATE_TX_MANAGER_DESTINATION, + }) + if err != nil { + log.L(ctx).Errorf("Failed to send assemble response: %s", err) + //Try to send an error to at least free up the coordinator but it is very possible the error fails to send for the same reason + // and we will need to rely on timeout and retry on the coordinator side + p.sendAssembleError(ctx, replyTo, assembleRequest.AssembleRequestId, assembleRequest.ContractAddress, assembleRequest.TransactionId, err) + return + } + + log.L(ctx).Debug("handleAssembleRequest sent assemble response") + +} + +func (p *privateTxManager) handleAssembleResponse(ctx context.Context, messagePayload []byte) { + log.L(ctx).Debug("handleAssembleResponse") + assembleResponse := &pbEngine.AssembleResponse{} + err := proto.Unmarshal(messagePayload, assembleResponse) + if err != nil { + log.L(ctx).Errorf("Failed to unmarshal assembleResponse: %s", err) + return + } + contractAddressString := assembleResponse.ContractAddress + transactionIDString := assembleResponse.TransactionId + + postAssemblyJSON := assembleResponse.PostAssembly + postAssembly := &components.TransactionPostAssembly{} + err = json.Unmarshal(postAssemblyJSON, postAssembly) + if err != nil { + log.L(ctx).Errorf("Failed to unmarshal postAssembly: %s", err) + //we at least know the transaction ID and contract address so we can communicate + // this as a failed assemble to let the coordinator know to stop waiting + p.HandleNewEvent(ctx, &ptmgrtypes.TransactionAssembleFailedEvent{ + PrivateTransactionEventBase: ptmgrtypes.PrivateTransactionEventBase{ + TransactionID: transactionIDString, + ContractAddress: contractAddressString, + }, + Error: err.Error(), + AssembleRequestID: assembleResponse.AssembleRequestId, + }) + return + } + + p.HandleNewEvent(ctx, &ptmgrtypes.TransactionAssembledEvent{ + PrivateTransactionEventBase: ptmgrtypes.PrivateTransactionEventBase{ + TransactionID: transactionIDString, + ContractAddress: contractAddressString, + }, + PostAssembly: postAssembly, + AssembleRequestID: assembleResponse.AssembleRequestId, + }) +} + +func (p *privateTxManager) handleAssembleError(ctx context.Context, messagePayload []byte) { + assembleError := &pbEngine.AssembleError{} + err := proto.Unmarshal(messagePayload, assembleError) + if err != nil { + log.L(ctx).Errorf("Failed to unmarshal assembleError: %s", err) + return + } + contractAddressString := assembleError.ContractAddress + transactionIDString := assembleError.TransactionId + + log.L(ctx).Infof("Received Assemble Error: ContractAddress: %s, TransactionId: %s, AssembleRequestId %s, Error: %s", contractAddressString, transactionIDString, assembleError.AssembleRequestId, assembleError.ErrorMessage) + + p.HandleNewEvent(ctx, &ptmgrtypes.TransactionAssembleFailedEvent{ + PrivateTransactionEventBase: ptmgrtypes.PrivateTransactionEventBase{ + TransactionID: transactionIDString, + ContractAddress: contractAddressString, + }, + AssembleRequestID: assembleError.AssembleRequestId, + Error: assembleError.ErrorMessage, + }) +} + // For now, this is here to help with testing but it seems like it could be useful thing to have // in the future if we want to have an eventing interface but at such time we would need to put more effort // into the reliability of the event delivery or maybe there is only a consumer of the event and it is responsible @@ -778,7 +1030,7 @@ func (p *privateTxManager) PrivateTransactionConfirmed(ctx context.Context, rece log.L(ctx).Infof("private TX manager notified of transaction confirmation %s deploy=%t", receipt.TransactionID, receipt.PSC == nil) if receipt.PSC != nil { - seq, err := p.getSequencerForContract(ctx, receipt.PSC.Address(), receipt.PSC) + seq, err := p.getSequencerForContract(ctx, p.components.Persistence().DB(), receipt.PSC.Address(), receipt.PSC) if err != nil { log.L(ctx).Errorf("failed to obtain sequence to process receipts on contract %s: %s", receipt.PSC.Address(), err) return @@ -787,9 +1039,34 @@ func (p *privateTxManager) PrivateTransactionConfirmed(ctx context.Context, rece } } +func (p *privateTxManager) handleStateProducedEvent(ctx context.Context, messagePayload []byte, replyToDestination string) { + + //in the meantime, we share with the sequencer in memory in case that state is needed to assemble in flight transactions + stateProducedEvent := &pbEngine.StateProducedEvent{} + err := proto.Unmarshal(messagePayload, stateProducedEvent) + if err != nil { + log.L(ctx).Errorf("Failed to unmarshal StateProducedEvent: %s", err) + return + } + + //State distributer deals with the reliable delivery e.g. sending acks etc + go p.stateDistributer.HandleStateProducedEvent(ctx, stateProducedEvent, replyToDestination) + + contractAddressString := stateProducedEvent.ContractAddress + contractAddress := tktypes.MustEthAddress(contractAddressString) + + sequencer, err := p.getSequencerForContract(ctx, p.components.Persistence().DB(), *contractAddress, nil) + if err != nil { + log.L(ctx).Errorf("Failed to get sequencer for contract address %s: %s", contractAddress, err) + return + } + sequencer.HandleStateProducedEvent(ctx, stateProducedEvent) + +} + func (p *privateTxManager) CallPrivateSmartContract(ctx context.Context, call *components.TransactionInputs) (*abi.ComponentValue, error) { - psc, err := p.components.DomainManager().GetSmartContractByAddress(ctx, call.To) + psc, err := p.components.DomainManager().GetSmartContractByAddress(ctx, p.components.Persistence().DB(), call.To) if err != nil { return nil, err } diff --git a/core/go/internal/privatetxnmgr/private_txn_mgr_test.go b/core/go/internal/privatetxnmgr/private_txn_mgr_test.go index d320b4147..21d49fccb 100644 --- a/core/go/internal/privatetxnmgr/private_txn_mgr_test.go +++ b/core/go/internal/privatetxnmgr/private_txn_mgr_test.go @@ -19,10 +19,7 @@ import ( "context" "errors" "fmt" - "math/rand" - "reflect" "regexp" - "sync" "testing" "time" @@ -34,8 +31,6 @@ import ( "github.com/kaleido-io/paladin/toolkit/pkg/log" "github.com/kaleido-io/paladin/toolkit/pkg/pldapi" "github.com/kaleido-io/paladin/toolkit/pkg/prototk" - "github.com/kaleido-io/paladin/toolkit/pkg/query" - "github.com/kaleido-io/paladin/toolkit/pkg/signerapi" "github.com/kaleido-io/paladin/toolkit/pkg/signpayloads" "github.com/kaleido-io/paladin/toolkit/pkg/tktypes" "github.com/kaleido-io/paladin/toolkit/pkg/verifiers" @@ -54,69 +49,152 @@ import ( pbEngine "github.com/kaleido-io/paladin/core/pkg/proto/engine" ) -// Attempt to assert the behaviour of the private transaction manager as a whole component in isolation from the rest of the system -// Tests in this file do not mock anything else in this package or sub packages but does mock other components and managers in paladin as per their interfaces +/* + * There are 2 flavours of test in this file + * 1. Package level tests: Tests that assert the behavior of the private transaction manager package as a whole component in isolation from the rest of the system. + * . None of the code from this package is mocked in these tests and the only interfaces that the test calls are public interfaces, defined on the components package. + * 2. Unit tests: Tests that assert the nuanced behavior of the functions in the private_txn_mgr.go file to provide more granular coverage of the codebase. + * . These tests are more white box in nature and mock other functions and interfaces within this package (and it `ptmgrtypes` subpackage) + */ -var testABI = abi.ABI{ - { - Name: "execute", - Type: abi.Function, - Inputs: abi.ParameterArray{ - { - Name: "inputs", - Type: "bytes32[]", - }, - { - Name: "outputs", - Type: "bytes32[]", - }, - { - Name: "data", - Type: "bytes", - }, - }, - }, +/* Package level tests */ + +func mockWritePublicTxsOk(mocks *dependencyMocks) chan struct{} { + mockPublicTxManager := mocks.publicTxManager + mockPublicTxManager.On("ValidateTransaction", mock.Anything, mock.Anything, mock.Anything).Return(nil) + dispatched := make(chan struct{}) + mwtx := mockPublicTxManager.On("WriteNewTransactions", mock.Anything, mock.Anything, mock.Anything) + mwtx.Run(func(args mock.Arguments) { + txs := args[2].([]*components.PublicTxSubmission) + res := make([]*pldapi.PublicTx, len(txs)) + for i, tx := range txs { + res[i] = &pldapi.PublicTx{ + LocalID: confutil.P(uint64(1000 + i)), + From: *tx.From, + To: tx.To, + Data: tx.Data, + PublicTxOptions: tx.PublicTxOptions, + } + } + mwtx.Return(func() {}, res, nil) + if dispatched != nil { + close(dispatched) + dispatched = nil + } + }) + return dispatched } func TestPrivateTxManagerInit(t *testing.T) { - privateTxManager, mocks := NewPrivateTransactionMgrForTesting(t, "node1") + privateTxManager, mocks := NewPrivateTransactionMgrForPackageTesting(t, "node1") err := privateTxManager.PostInit(mocks.allComponents) require.NoError(t, err) } -func TestPrivateTxManagerInvalidTransaction(t *testing.T) { +func TestPrivateTxManagerInvalidTransactionMissingDomain(t *testing.T) { + t.Skip("This test is not valid because the code accepts empty domain. TODO: remove this test or change the code and migrate any consumers") ctx := context.Background() - privateTxManager, mocks := NewPrivateTransactionMgrForTesting(t, "node1") + privateTxManager, mocks := NewPrivateTransactionMgrForPackageTesting(t, "node1") + domainAddress := tktypes.MustEthAddress(tktypes.RandHex(20)) + mocks.domainMgr.On("GetSmartContractByAddress", mock.Anything, mock.Anything, *domainAddress).Return(mocks.domainSmartContract, nil) + err := privateTxManager.PostInit(mocks.allComponents) require.NoError(t, err) err = privateTxManager.Start() require.NoError(t, err) - err = privateTxManager.handleNewTx(ctx, &components.PrivateTransaction{}) + err = privateTxManager.HandleNewTx(ctx, privateTxManager.DB(), &components.ValidatedTransaction{ + Function: &components.ResolvedFunction{ + Definition: testABI[0], + }, + Transaction: &pldapi.Transaction{ + ID: confutil.P(uuid.New()), + TransactionBase: pldapi.TransactionBase{ + To: domainAddress, + From: "alice@node1", + }, + }, + }) // no input domain should err assert.Regexp(t, "PD011800", err) } +func TestPrivateTxManagerInvalidTransactionMismatchedDomain(t *testing.T) { + ctx := context.Background() + + privateTxManager, mocks := NewPrivateTransactionMgrForPackageTesting(t, "node1") + domainAddress := tktypes.MustEthAddress(tktypes.RandHex(20)) + mocks.domainMgr.On("GetSmartContractByAddress", mock.Anything, mock.Anything, *domainAddress).Return(mocks.domainSmartContract, nil) + mocks.domainSmartContract.On("Address").Return(*domainAddress) + + err := privateTxManager.PostInit(mocks.allComponents) + require.NoError(t, err) + + err = privateTxManager.Start() + require.NoError(t, err) + + err = privateTxManager.HandleNewTx(ctx, privateTxManager.DB(), &components.ValidatedTransaction{ + Function: &components.ResolvedFunction{ + Definition: testABI[0], + }, + Transaction: &pldapi.Transaction{ + ID: confutil.P(uuid.New()), + TransactionBase: pldapi.TransactionBase{ + To: domainAddress, + Domain: "domain2", + From: "alice@node1", + }, + }, + }) + // no input domain should err + assert.Regexp(t, "PD011825", err) +} + +func TestPrivateTxManagerInvalidTransactionEmptyAddress(t *testing.T) { + ctx := context.Background() + + privateTxManager, mocks := NewPrivateTransactionMgrForPackageTesting(t, "node1") + domainAddress := &tktypes.EthAddress{} + + err := privateTxManager.PostInit(mocks.allComponents) + require.NoError(t, err) + + err = privateTxManager.Start() + require.NoError(t, err) + + err = privateTxManager.HandleNewTx(ctx, privateTxManager.DB(), &components.ValidatedTransaction{ + Function: &components.ResolvedFunction{ + Definition: testABI[0], + }, + Transaction: &pldapi.Transaction{ + ID: confutil.P(uuid.New()), + TransactionBase: pldapi.TransactionBase{ + To: domainAddress, + Domain: "domain1", + From: "alice@node1", + }, + }, + }) + // no input domain should err + assert.Regexp(t, "PD011811", err) +} + func TestPrivateTxManagerSimpleTransaction(t *testing.T) { //Submit a transaction that gets assembled with an attestation plan for a local endorser to sign the transaction ctx := context.Background() domainAddress := tktypes.MustEthAddress(tktypes.RandHex(20)) - privateTxManager, mocks := NewPrivateTransactionMgrForTesting(t, "node1") + privateTxManager, mocks := NewPrivateTransactionMgrForPackageTesting(t, "node1") mocks.mockDomain(domainAddress) domainAddressString := domainAddress.String() // unqualified lookup string because everything is local - aliceIdentity := "alice@node1" - aliceVerifier := tktypes.RandAddress().String() - notaryIdentityLocal := "domain1.contract1.notary" - notaryIdentity := notaryIdentityLocal + "@node1" - notaryVerifier := tktypes.RandAddress().String() - notaryKeyHandle := "notaryKeyHandle" + alice := newPartyForTesting(ctx, "alice", "node1", mocks) + notary := newPartyForTesting(ctx, "notary", "node1", mocks) initialised := make(chan struct{}, 1) mocks.domainSmartContract.On("InitTransaction", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { @@ -124,12 +202,12 @@ func TestPrivateTxManagerSimpleTransaction(t *testing.T) { tx.PreAssembly = &components.TransactionPreAssembly{ RequiredVerifiers: []*prototk.ResolveVerifierRequest{ { - Lookup: aliceIdentity, + Lookup: alice.identityLocator, Algorithm: algorithms.ECDSA_SECP256K1, VerifierType: verifiers.ETH_ADDRESS, }, { - Lookup: notaryIdentity, + Lookup: notary.identityLocator, Algorithm: algorithms.ECDSA_SECP256K1, VerifierType: verifiers.ETH_ADDRESS, }, @@ -138,13 +216,13 @@ func TestPrivateTxManagerSimpleTransaction(t *testing.T) { initialised <- struct{}{} }).Return(nil) - mocks.identityResolver.On("ResolveVerifierAsync", mock.Anything, aliceIdentity, algorithms.ECDSA_SECP256K1, verifiers.ETH_ADDRESS, mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + mocks.identityResolver.On("ResolveVerifierAsync", mock.Anything, alice.identityLocator, algorithms.ECDSA_SECP256K1, verifiers.ETH_ADDRESS, mock.Anything, mock.Anything).Run(func(args mock.Arguments) { resovleFn := args.Get(4).(func(context.Context, string)) - resovleFn(ctx, aliceVerifier) + resovleFn(ctx, alice.verifier) }).Return(nil) - mocks.identityResolver.On("ResolveVerifierAsync", mock.Anything, notaryIdentity, algorithms.ECDSA_SECP256K1, verifiers.ETH_ADDRESS, mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + mocks.identityResolver.On("ResolveVerifierAsync", mock.Anything, notary.identityLocator, algorithms.ECDSA_SECP256K1, verifiers.ETH_ADDRESS, mock.Anything, mock.Anything).Run(func(args mock.Arguments) { resovleFn := args.Get(4).(func(context.Context, string)) - resovleFn(ctx, notaryVerifier) + resovleFn(ctx, notary.verifier) }).Return(nil) // TODO check that the transaction is signed with this key @@ -152,6 +230,7 @@ func TestPrivateTxManagerSimpleTransaction(t *testing.T) { mocks.domainSmartContract.On("ContractConfig").Return(&prototk.ContractConfig{ CoordinatorSelection: prototk.ContractConfig_COORDINATOR_ENDORSER, }) + endorsePayload := []byte("some-endorsement-bytes") mocks.domainSmartContract.On("AssembleTransaction", mock.Anything, mock.Anything, mock.Anything).Run(func(args mock.Arguments) { tx := args.Get(2).(*components.PrivateTransaction) @@ -172,7 +251,7 @@ func TestPrivateTxManagerSimpleTransaction(t *testing.T) { VerifierType: verifiers.ETH_ADDRESS, PayloadType: signpayloads.OPAQUE_TO_RSV, Parties: []string{ - notaryIdentity, + notary.identityLocator, }, }, }, @@ -183,20 +262,20 @@ func TestPrivateTxManagerSimpleTransaction(t *testing.T) { notaryKeyMapping := &pldapi.KeyMappingAndVerifier{ KeyMappingWithPath: &pldapi.KeyMappingWithPath{KeyMapping: &pldapi.KeyMapping{ - Identifier: notaryIdentityLocal, - KeyHandle: notaryKeyHandle, + Identifier: notary.identity, + KeyHandle: notary.keyHandle, }}, - Verifier: &pldapi.KeyVerifier{Verifier: notaryVerifier}, + Verifier: &pldapi.KeyVerifier{Verifier: notary.verifier}, } - mocks.keyManager.On("ResolveKeyNewDatabaseTX", mock.Anything, notaryIdentityLocal, algorithms.ECDSA_SECP256K1, verifiers.ETH_ADDRESS).Return(notaryKeyMapping, nil) + mocks.keyManager.On("ResolveKeyNewDatabaseTX", mock.Anything, notary.identity, algorithms.ECDSA_SECP256K1, verifiers.ETH_ADDRESS).Return(notaryKeyMapping, nil) //TODO match endorsement request and verifier args mocks.domainSmartContract.On("EndorseTransaction", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&components.EndorsementResult{ Result: prototk.EndorseTransactionResponse_SIGN, - Payload: []byte("some-endorsement-bytes"), + Payload: endorsePayload, Endorser: &prototk.ResolvedVerifier{ - Lookup: notaryIdentity, - Verifier: notaryVerifier, + Lookup: notary.identityLocator, + Verifier: notary.verifier, Algorithm: algorithms.ECDSA_SECP256K1, VerifierType: verifiers.ETH_ADDRESS, }, @@ -219,45 +298,20 @@ func TestPrivateTxManagerSimpleTransaction(t *testing.T) { tx.PreparedPublicTransaction = &pldapi.TransactionInput{ ABI: abi.ABI{testABI[0]}, TransactionBase: pldapi.TransactionBase{ - To: domainAddress, - Data: tktypes.RawJSON(jsonData), + To: domainAddress, + Data: tktypes.RawJSON(jsonData), + PublicTxOptions: pldapi.PublicTxOptions{Gas: confutil.P(tktypes.HexUint64(100000))}, }, } }, ) - - tx := &components.PrivateTransaction{ - ID: uuid.New(), - Inputs: &components.TransactionInputs{ - Domain: "domain1", - To: *domainAddress, - From: aliceIdentity, - }, - } - - mockPublicTxBatch := componentmocks.NewPublicTxBatch(t) - mockPublicTxBatch.On("Finalize", mock.Anything).Return().Maybe() - mockPublicTxBatch.On("CleanUp", mock.Anything).Return().Maybe() - - mockPublicTxManager := mocks.publicTxManager.(*componentmocks.PublicTxManager) - mockPublicTxManager.On("PrepareSubmissionBatch", mock.Anything, mock.Anything).Return(mockPublicTxBatch, nil) + testTransactionID := confutil.P(uuid.New()) signingAddr := tktypes.RandAddress() mocks.keyManager.On("ResolveEthAddressBatchNewDatabaseTX", mock.Anything, []string{"signer1"}). Return([]*tktypes.EthAddress{signingAddr}, nil) - publicTransactions := []components.PublicTxAccepted{ - newFakePublicTx(&components.PublicTxSubmission{ - Bindings: []*components.PaladinTXReference{{TransactionID: tx.ID, TransactionType: pldapi.TransactionTypePrivate.Enum()}}, - PublicTxInput: pldapi.PublicTxInput{ - From: signingAddr, - }, - }, nil), - } - mockPublicTxBatch.On("Submit", mock.Anything, mock.Anything).Return(nil) - mockPublicTxBatch.On("Rejected").Return([]components.PublicTxRejected{}) - mockPublicTxBatch.On("Accepted").Return(publicTransactions) - mockPublicTxBatch.On("Completed", mock.Anything, true).Return() + _ = mockWritePublicTxsOk(mocks) dcFlushed := make(chan error, 1) mocks.domainContext.On("Flush", mock.Anything).Return(func(err error) { @@ -266,113 +320,63 @@ func TestPrivateTxManagerSimpleTransaction(t *testing.T) { err := privateTxManager.Start() require.NoError(t, err) - err = privateTxManager.handleNewTx(ctx, tx) + + tx := &components.ValidatedTransaction{ + Function: &components.ResolvedFunction{ + Definition: testABI[0], + }, + Transaction: &pldapi.Transaction{ + ID: testTransactionID, + TransactionBase: pldapi.TransactionBase{ + Domain: "domain1", + To: domainAddress, + From: alice.identityLocator, + }, + }, + } + err = privateTxManager.HandleNewTx(ctx, privateTxManager.DB(), tx) require.NoError(t, err) // testTimeout := 2 * time.Second testTimeout := 100 * time.Minute - status := pollForStatus(ctx, t, "dispatched", privateTxManager, domainAddressString, tx.ID.String(), testTimeout) + status := pollForStatus(ctx, t, "dispatched", privateTxManager, domainAddressString, testTransactionID.String(), testTimeout) assert.Equal(t, "dispatched", status) require.NoError(t, <-dcFlushed) -} - -func TestPrivateTxManagerLocalEndorserSubmits(t *testing.T) { -} - -func TestPrivateTxManagerRevertFromLocalEndorsement(t *testing.T) { -} - -type identityForTesting struct { - identity string - identityLocator string - verifier string - keyHandle string - mocks *dependencyMocks - mockSign func(signature []byte) -} - -func (i *identityForTesting) mockResolve(ctx context.Context, other identityForTesting) { - // in addition to the default mocks set up in newPartyForTesting, we can set up mocks to resolve remote identitys - // we could have used a real IdentityResolver here but we are testing the private transaction manager in isolation and so we mock the IdentityResolver as we do with all other tests in this file - i.mocks.identityResolver.On( - "ResolveVerifierAsync", - mock.Anything, - other.identityLocator, - algorithms.ECDSA_SECP256K1, - verifiers.ETH_ADDRESS, - mock.Anything, - mock.Anything). - Run(func(args mock.Arguments) { - resolveFn := args.Get(4).(func(context.Context, string)) - resolveFn(ctx, other.verifier) - }).Return(nil).Maybe() -} - -func newPartyForTesting(ctx context.Context, name, node string, mocks *dependencyMocks) identityForTesting { - party := identityForTesting{ - identity: name, - identityLocator: name + "@" + node, - verifier: tktypes.RandAddress().String(), - keyHandle: name + "KeyHandle", - mocks: mocks, - } - - mocks.identityResolver.On("ResolveVerifierAsync", mock.Anything, party.identity, algorithms.ECDSA_SECP256K1, verifiers.ETH_ADDRESS, mock.Anything, mock.Anything).Run(func(args mock.Arguments) { - resolveFn := args.Get(4).(func(context.Context, string)) - resolveFn(ctx, party.verifier) - }).Return(nil).Maybe() - - mocks.identityResolver.On("ResolveVerifierAsync", mock.Anything, party.identityLocator, algorithms.ECDSA_SECP256K1, verifiers.ETH_ADDRESS, mock.Anything, mock.Anything).Run(func(args mock.Arguments) { - resolveFn := args.Get(4).(func(context.Context, string)) - resolveFn(ctx, party.verifier) - }).Return(nil).Maybe() - - mocks.identityResolver.On("ResolveVerifier", mock.Anything, party.identityLocator, algorithms.ECDSA_SECP256K1, verifiers.ETH_ADDRESS). - Return(party.verifier, nil).Maybe() - - keyMapping := &pldapi.KeyMappingAndVerifier{ - KeyMappingWithPath: &pldapi.KeyMappingWithPath{KeyMapping: &pldapi.KeyMapping{ - Identifier: party.identity, - KeyHandle: party.keyHandle, - }}, - Verifier: &pldapi.KeyVerifier{Verifier: party.verifier}, - } - mocks.keyManager.On("ResolveKeyNewDatabaseTX", mock.Anything, party.identity, algorithms.ECDSA_SECP256K1, verifiers.ETH_ADDRESS).Return(keyMapping, nil).Maybe() - party.mockSign = func(signature []byte) { - mocks.keyManager.On("Sign", mock.Anything, keyMapping, signpayloads.OPAQUE_TO_RSV, mock.Anything). - Return(signature, nil) - } + privateTxManager.Stop() - return party } -func TestPrivateTxManagerRemoteNotaryEndorser(t *testing.T) { +func TestPrivateTxManagerSimplePreparedTransaction(t *testing.T) { + //Prepare a transaction that gets assembled with an attestation plan for a local endorser to sign the transaction + // submit mode external means the transaction does not get dispatched to the public tx manager + // but should be distributed to the sender + + t.Skip("Test incomplete") + //TODO: this is a challenging test to write because the nature of these tests is that we mock most things outside the privatetxnmgr package (and its subpackages) + // however, in this use case, the integration between privatetxnmgr code and the TxManager includes database integration + // specifically, privatetxnmgr writes prepared transaction distribtion records to the database which have a foreign key to the prepared transaction + // we have a few options here + // 1. In the mock for WritePreparedTransactions do a DB insert to the prepared_txns table. This should be trivial because we receive the DB transaction + // 2. Given that the privatetxnmgr package only deals with in memory objects, and all DB persistence is delegated to its subpackage `syncpoints` we could mock the syncpoints package + // . in these tests and rely on component tests for the full integration between packages and the database + // Currently leaning towards option 2 but will need some refactoring of the code to allow us to inject the syncpoints mocks into the privatetxnmgr package. We also call the real state distribution + // package which really should be changed to be a mock to align with this testing strategy ctx := context.Background() - // A transaction that requires exactly one endorsement from a notary (as per noto) and therefore delegates coordination of the transaction to that node domainAddress := tktypes.MustEthAddress(tktypes.RandHex(20)) - localNodeName := "localNode" - remoteNodeName := "remoteNode" - privateTxManager, localNodeMocks := NewPrivateTransactionMgrForTesting(t, localNodeName) - localNodeMocks.mockDomain(domainAddress) + privateTxManager, mocks := NewPrivateTransactionMgrForPackageTesting(t, "node1") + mocks.mockDomain(domainAddress) domainAddressString := domainAddress.String() - remoteEngine, remoteEngineMocks := NewPrivateTransactionMgrForTesting(t, remoteNodeName) - remoteEngineMocks.mockDomain(domainAddress) - - alice := newPartyForTesting(ctx, "alice", localNodeName, localNodeMocks) - notary := newPartyForTesting(ctx, "notary", remoteNodeName, remoteEngineMocks) - - alice.mockResolve(ctx, notary) + // unqualified lookup string because everything is local + alice := newPartyForTesting(ctx, "alice", "node1", mocks) + notary := newPartyForTesting(ctx, "notary", "node1", mocks) initialised := make(chan struct{}, 1) - localNodeMocks.domainSmartContract.On("ContractConfig").Return(&prototk.ContractConfig{ - CoordinatorSelection: prototk.ContractConfig_COORDINATOR_ENDORSER, - }) - localNodeMocks.domainSmartContract.On("InitTransaction", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + mocks.domainSmartContract.On("InitTransaction", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { tx := args.Get(1).(*components.PrivateTransaction) tx.PreAssembly = &components.TransactionPreAssembly{ RequiredVerifiers: []*prototk.ResolveVerifierRequest{ @@ -391,9 +395,22 @@ func TestPrivateTxManagerRemoteNotaryEndorser(t *testing.T) { initialised <- struct{}{} }).Return(nil) - assembled := make(chan struct{}, 1) + mocks.identityResolver.On("ResolveVerifierAsync", mock.Anything, alice.identityLocator, algorithms.ECDSA_SECP256K1, verifiers.ETH_ADDRESS, mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + resovleFn := args.Get(4).(func(context.Context, string)) + resovleFn(ctx, alice.verifier) + }).Return(nil) + mocks.identityResolver.On("ResolveVerifierAsync", mock.Anything, notary.identityLocator, algorithms.ECDSA_SECP256K1, verifiers.ETH_ADDRESS, mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + resovleFn := args.Get(4).(func(context.Context, string)) + resovleFn(ctx, notary.verifier) + }).Return(nil) + // TODO check that the transaction is signed with this key - localNodeMocks.domainSmartContract.On("AssembleTransaction", mock.Anything, mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + assembled := make(chan struct{}, 1) + mocks.domainSmartContract.On("ContractConfig").Return(&prototk.ContractConfig{ + CoordinatorSelection: prototk.ContractConfig_COORDINATOR_ENDORSER, + }) + endorsePayload := []byte("some-endorsement-bytes") + mocks.domainSmartContract.On("AssembleTransaction", mock.Anything, mock.Anything, mock.Anything).Run(func(args mock.Arguments) { tx := args.Get(2).(*components.PrivateTransaction) tx.PostAssembly = &components.TransactionPostAssembly{ @@ -422,41 +439,31 @@ func TestPrivateTxManagerRemoteNotaryEndorser(t *testing.T) { }).Return(nil) - localNodeMocks.transportManager.On("Send", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { - go func() { - assert.Equal(t, remoteNodeName, args.Get(1).(*components.TransportMessage).Node) - transportMessage := args.Get(1).(*components.TransportMessage) - remoteEngine.ReceiveTransportMessage(ctx, transportMessage) - }() - }).Return(nil).Maybe() - - remoteEngineMocks.transportManager.On("Send", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { - go func() { - transportMessage := args.Get(1).(*components.TransportMessage) - privateTxManager.ReceiveTransportMessage(ctx, transportMessage) - }() - }).Return(nil).Maybe() - - remoteEngineMocks.domainMgr.On("GetSmartContractByAddress", mock.Anything, *domainAddress).Return(remoteEngineMocks.domainSmartContract, nil) + notaryKeyMapping := &pldapi.KeyMappingAndVerifier{ + KeyMappingWithPath: &pldapi.KeyMappingWithPath{KeyMapping: &pldapi.KeyMapping{ + Identifier: notary.identity, + KeyHandle: notary.keyHandle, + }}, + Verifier: &pldapi.KeyVerifier{Verifier: notary.verifier}, + } + mocks.keyManager.On("ResolveKeyNewDatabaseTX", mock.Anything, notary.identity, algorithms.ECDSA_SECP256K1, verifiers.ETH_ADDRESS).Return(notaryKeyMapping, nil) //TODO match endorsement request and verifier args - remoteEngineMocks.domainSmartContract.On("ContractConfig").Return(&prototk.ContractConfig{ - CoordinatorSelection: prototk.ContractConfig_COORDINATOR_ENDORSER, - }) - remoteEngineMocks.domainSmartContract.On("EndorseTransaction", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&components.EndorsementResult{ + mocks.domainSmartContract.On("EndorseTransaction", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&components.EndorsementResult{ Result: prototk.EndorseTransactionResponse_SIGN, - Payload: []byte("some-endorsement-bytes"), + Payload: endorsePayload, Endorser: &prototk.ResolvedVerifier{ - Lookup: notary.identityLocator, //matches whatever was specified in PreAssembly.RequiredVerifiers + Lookup: notary.identityLocator, Verifier: notary.verifier, Algorithm: algorithms.ECDSA_SECP256K1, VerifierType: verifiers.ETH_ADDRESS, }, }, nil) - notary.mockSign([]byte("some-signature-bytes")) + mocks.keyManager.On("Sign", mock.Anything, notaryKeyMapping, signpayloads.OPAQUE_TO_RSV, mock.Anything). + Return([]byte("notary-signature-bytes"), nil) - remoteEngineMocks.domainSmartContract.On("PrepareTransaction", mock.Anything, mock.Anything, mock.Anything).Return(nil).Run( + mocks.domainSmartContract.On("PrepareTransaction", mock.Anything, mock.Anything, mock.Anything).Return(nil).Run( func(args mock.Arguments) { cv, err := testABI[0].Inputs.ParseExternalData(map[string]any{ "inputs": []any{tktypes.Bytes32(tktypes.RandBytes(32))}, @@ -470,101 +477,88 @@ func TestPrivateTxManagerRemoteNotaryEndorser(t *testing.T) { tx.PreparedPublicTransaction = &pldapi.TransactionInput{ ABI: abi.ABI{testABI[0]}, TransactionBase: pldapi.TransactionBase{ - To: domainAddress, - Data: tktypes.RawJSON(jsonData), + To: domainAddress, + Data: tktypes.RawJSON(jsonData), + PublicTxOptions: pldapi.PublicTxOptions{Gas: confutil.P(tktypes.HexUint64(100000))}, }, } }, ) + testTransactionID := confutil.P(uuid.New()) - tx := &components.PrivateTransaction{ - ID: uuid.New(), - Inputs: &components.TransactionInputs{ - Domain: "domain1", - To: *domainAddress, - From: alice.identityLocator, // domain manager would insert the local node name for us, but we've mocked that - }, - } - - mockPublicTxBatch := componentmocks.NewPublicTxBatch(t) - mockPublicTxBatch.On("Finalize", mock.Anything).Return().Maybe() - mockPublicTxBatch.On("CleanUp", mock.Anything).Return().Maybe() - - mockPublicTxManager := remoteEngineMocks.publicTxManager.(*componentmocks.PublicTxManager) - mockPublicTxManager.On("PrepareSubmissionBatch", mock.Anything, mock.Anything).Return(mockPublicTxBatch, nil) - - signingAddr := tktypes.RandAddress() - remoteEngineMocks.keyManager.On("ResolveEthAddressBatchNewDatabaseTX", mock.Anything, []string{"signer1"}). - Return([]*tktypes.EthAddress{signingAddr}, nil) + _ = mockWritePublicTxsOk(mocks) - publicTransactions := []components.PublicTxAccepted{ - newFakePublicTx(&components.PublicTxSubmission{ - Bindings: []*components.PaladinTXReference{{TransactionID: tx.ID, TransactionType: pldapi.TransactionTypePrivate.Enum()}}, - PublicTxInput: pldapi.PublicTxInput{ - From: signingAddr, - }, - }, nil), - } - mockPublicTxBatch.On("Submit", mock.Anything, mock.Anything).Return(nil) - mockPublicTxBatch.On("Rejected").Return([]components.PublicTxRejected{}) - mockPublicTxBatch.On("Accepted").Return(publicTransactions) - mockPublicTxBatch.On("Completed", mock.Anything, true).Return() + mocks.txManager.On("WritePreparedTransactions", mock.Anything, mock.Anything, mock.Anything).Return(nil) - // Flush of domain context happens on the remote node (the notary) dcFlushed := make(chan error, 1) - remoteEngineMocks.domainContext.On("Flush", mock.Anything).Return(func(err error) { + mocks.domainContext.On("Flush", mock.Anything).Return(func(err error) { dcFlushed <- err }, nil) err := privateTxManager.Start() - assert.NoError(t, err) - - err = privateTxManager.handleNewTx(ctx, tx) - assert.NoError(t, err) + require.NoError(t, err) - status := pollForStatus(ctx, t, "delegating", privateTxManager, domainAddressString, tx.ID.String(), 200*time.Second) - assert.Equal(t, "delegating", status) + tx := &components.ValidatedTransaction{ + Function: &components.ResolvedFunction{ + Definition: testABI[0], + }, + Transaction: &pldapi.Transaction{ + ID: testTransactionID, + SubmitMode: pldapi.SubmitModeExternal.Enum(), + TransactionBase: pldapi.TransactionBase{ + Domain: "domain1", + To: domainAddress, + From: alice.identityLocator, + }, + }, + } + err = privateTxManager.HandleNewTx(ctx, privateTxManager.DB(), tx) + require.NoError(t, err) - status = pollForStatus(ctx, t, "dispatched", remoteEngine, domainAddressString, tx.ID.String(), 200*time.Second) - assert.Equal(t, "dispatched", status) + // testTimeout := 2 * time.Second + testTimeout := 100 * time.Minute + status := pollForStatus(ctx, t, "prepared", privateTxManager, domainAddressString, testTransactionID.String(), testTimeout) + assert.Equal(t, "prepared", status) require.NoError(t, <-dcFlushed) + privateTxManager.Stop() + } -func TestPrivateTxManagerEndorsementGroup(t *testing.T) { +func TestPrivateTxManagerLocalEndorserSubmits(t *testing.T) { +} +func TestPrivateTxManagerRevertFromLocalEndorsement(t *testing.T) { +} + +func TestPrivateTxManagerRemoteNotaryEndorser(t *testing.T) { ctx := context.Background() - // A transaction that requires endorsement from a group of remote endorsers (as per pente and its 100% endorsement policy) - // In this scenario there is only one active transaction and therefore no risk of contention so the transactions is coordinated - // and dispatched locally. The only expected interaction with the remote nodes is to request endorsements and to distribute the new states + // A transaction that requires exactly one endorsement from a notary (as per noto) and therefore delegates coordination of the transaction to that node domainAddress := tktypes.MustEthAddress(tktypes.RandHex(20)) + localNodeName := "localNode" + remoteNodeName := "remoteNode" + privateTxManager, localNodeMocks := NewPrivateTransactionMgrForPackageTesting(t, localNodeName) + localNodeMocks.mockDomain(domainAddress) + domainAddressString := domainAddress.String() - aliceNodeName := "aliceNode" - bobNodeName := "bobNode" - carolNodeName := "carolNode" + remoteEngine, remoteEngineMocks := NewPrivateTransactionMgrForPackageTesting(t, remoteNodeName) + remoteEngineMocks.mockDomain(domainAddress) - aliceEngine, aliceEngineMocks := NewPrivateTransactionMgrForTesting(t, aliceNodeName) - aliceEngineMocks.mockDomain(domainAddress) - - bobEngine, bobEngineMocks := NewPrivateTransactionMgrForTesting(t, bobNodeName) - bobEngineMocks.mockDomain(domainAddress) - - carolEngine, carolEngineMocks := NewPrivateTransactionMgrForTesting(t, carolNodeName) - carolEngineMocks.mockDomain(domainAddress) - - alice := newPartyForTesting(ctx, "alice", aliceNodeName, aliceEngineMocks) - bob := newPartyForTesting(ctx, "bob", bobNodeName, bobEngineMocks) - carol := newPartyForTesting(ctx, "carol", carolNodeName, carolEngineMocks) + alice := newPartyForTesting(ctx, "alice", localNodeName, localNodeMocks) + notary := newPartyForTesting(ctx, "notary", remoteNodeName, remoteEngineMocks) - alice.mockResolve(ctx, bob) - alice.mockResolve(ctx, carol) + alice.mockResolve(ctx, notary) + notary.mockResolve(ctx, alice) initialised := make(chan struct{}, 1) - - aliceEngineMocks.domainSmartContract.On("InitTransaction", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + localNodeMocks.domainSmartContract.On("ContractConfig").Return(&prototk.ContractConfig{ + CoordinatorSelection: prototk.ContractConfig_COORDINATOR_STATIC, + StaticCoordinator: ¬ary.identityLocator, + }) + localNodeMocks.domainSmartContract.On("InitTransaction", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { tx := args.Get(1).(*components.PrivateTransaction) tx.PreAssembly = &components.TransactionPreAssembly{ RequiredVerifiers: []*prototk.ResolveVerifierRequest{ @@ -574,12 +568,7 @@ func TestPrivateTxManagerEndorsementGroup(t *testing.T) { VerifierType: verifiers.ETH_ADDRESS, }, { - Lookup: bob.identityLocator, - Algorithm: algorithms.ECDSA_SECP256K1, - VerifierType: verifiers.ETH_ADDRESS, - }, - { - Lookup: carol.identityLocator, + Lookup: notary.identityLocator, Algorithm: algorithms.ECDSA_SECP256K1, VerifierType: verifiers.ETH_ADDRESS, }, @@ -588,23 +577,9 @@ func TestPrivateTxManagerEndorsementGroup(t *testing.T) { initialised <- struct{}{} }).Return(nil) - aliceEngineMocks.identityResolver.On("ResolveVerifierAsync", mock.Anything, alice.identityLocator, algorithms.ECDSA_SECP256K1, verifiers.ETH_ADDRESS, mock.Anything, mock.Anything).Run(func(args mock.Arguments) { - resovleFn := args.Get(4).(func(context.Context, string)) - resovleFn(ctx, alice.verifier) - }).Return(nil) - - aliceEngineMocks.identityResolver.On("ResolveVerifierAsync", mock.Anything, bob.identityLocator, algorithms.ECDSA_SECP256K1, verifiers.ETH_ADDRESS, mock.Anything, mock.Anything).Run(func(args mock.Arguments) { - resovleFn := args.Get(4).(func(context.Context, string)) - resovleFn(ctx, bob.verifier) - }).Return(nil) - - aliceEngineMocks.identityResolver.On("ResolveVerifierAsync", mock.Anything, carol.identityLocator, algorithms.ECDSA_SECP256K1, verifiers.ETH_ADDRESS, mock.Anything, mock.Anything).Run(func(args mock.Arguments) { - resovleFn := args.Get(4).(func(context.Context, string)) - resovleFn(ctx, carol.verifier) - }).Return(nil) - assembled := make(chan struct{}, 1) - aliceEngineMocks.domainSmartContract.On("AssembleTransaction", mock.Anything, mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + + localNodeMocks.domainSmartContract.On("AssembleTransaction", mock.Anything, mock.Anything, mock.Anything).Run(func(args mock.Arguments) { tx := args.Get(2).(*components.PrivateTransaction) tx.PostAssembly = &components.TransactionPostAssembly{ @@ -618,15 +593,13 @@ func TestPrivateTxManagerEndorsementGroup(t *testing.T) { }, AttestationPlan: []*prototk.AttestationRequest{ { - Name: "endorsers", + Name: "notary", AttestationType: prototk.AttestationType_ENDORSE, Algorithm: algorithms.ECDSA_SECP256K1, VerifierType: verifiers.ETH_ADDRESS, PayloadType: signpayloads.OPAQUE_TO_RSV, Parties: []string{ - alice.identityLocator, - bob.identityLocator, - carol.identityLocator, + notary.identityLocator, }, }, }, @@ -635,73 +608,41 @@ func TestPrivateTxManagerEndorsementGroup(t *testing.T) { }).Return(nil) - routeToNode := func(args mock.Arguments) { + localNodeMocks.transportManager.On("Send", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { go func() { + assert.Equal(t, remoteNodeName, args.Get(1).(*components.TransportMessage).Node) transportMessage := args.Get(1).(*components.TransportMessage) - switch transportMessage.Node { - case aliceNodeName: - aliceEngine.ReceiveTransportMessage(ctx, transportMessage) - case bobNodeName: - bobEngine.ReceiveTransportMessage(ctx, transportMessage) - case carolNodeName: - carolEngine.ReceiveTransportMessage(ctx, transportMessage) - } + remoteEngine.ReceiveTransportMessage(ctx, transportMessage) }() - } - - aliceEngineMocks.transportManager.On("Send", mock.Anything, mock.Anything).Run(routeToNode).Return(nil).Maybe() - bobEngineMocks.transportManager.On("Send", mock.Anything, mock.Anything).Run(routeToNode).Return(nil).Maybe() - carolEngineMocks.transportManager.On("Send", mock.Anything, mock.Anything).Run(routeToNode).Return(nil).Maybe() + }).Return(nil).Maybe() - //set up the mocks on bob and carols engines that are need on the endorse code path (and of course also on alice's engine because she is an endorser too) + remoteEngineMocks.transportManager.On("Send", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + go func() { + transportMessage := args.Get(1).(*components.TransportMessage) + privateTxManager.ReceiveTransportMessage(ctx, transportMessage) + }() + }).Return(nil).Maybe() - bobEngineMocks.domainMgr.On("GetSmartContractByAddress", mock.Anything, *domainAddress).Return(bobEngineMocks.domainSmartContract, nil) - carolEngineMocks.domainMgr.On("GetSmartContractByAddress", mock.Anything, *domainAddress).Return(carolEngineMocks.domainSmartContract, nil) + remoteEngineMocks.domainMgr.On("GetSmartContractByAddress", mock.Anything, mock.Anything, *domainAddress).Return(remoteEngineMocks.domainSmartContract, nil) //TODO match endorsement request and verifier args - aliceEngineMocks.domainSmartContract.On("ContractConfig").Return(&prototk.ContractConfig{ + remoteEngineMocks.domainSmartContract.On("ContractConfig").Return(&prototk.ContractConfig{ CoordinatorSelection: prototk.ContractConfig_COORDINATOR_ENDORSER, }) - aliceEngineMocks.domainSmartContract.On("EndorseTransaction", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&components.EndorsementResult{ - Result: prototk.EndorseTransactionResponse_SIGN, - Payload: []byte("alice-endorsement-bytes"), - Endorser: &prototk.ResolvedVerifier{ - Lookup: alice.identityLocator, - Verifier: alice.verifier, - Algorithm: algorithms.ECDSA_SECP256K1, - VerifierType: verifiers.ETH_ADDRESS, - }, - }, nil) - - alice.mockSign([]byte("alice-signature-bytes")) - - bobEngineMocks.domainSmartContract.On("EndorseTransaction", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&components.EndorsementResult{ - Result: prototk.EndorseTransactionResponse_SIGN, - Payload: []byte("bob-endorsement-bytes"), - Endorser: &prototk.ResolvedVerifier{ - Lookup: bob.identityLocator, - Verifier: bob.verifier, - Algorithm: algorithms.ECDSA_SECP256K1, - VerifierType: verifiers.ETH_ADDRESS, - }, - }, nil) - - bob.mockSign([]byte("bob-signature-bytes")) - - carolEngineMocks.domainSmartContract.On("EndorseTransaction", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&components.EndorsementResult{ + remoteEngineMocks.domainSmartContract.On("EndorseTransaction", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&components.EndorsementResult{ Result: prototk.EndorseTransactionResponse_SIGN, - Payload: []byte("carol-endorsement-bytes"), + Payload: []byte("some-endorsement-bytes"), Endorser: &prototk.ResolvedVerifier{ - Lookup: carol.identityLocator, - Verifier: carol.verifier, + Lookup: notary.identityLocator, //matches whatever was specified in PreAssembly.RequiredVerifiers + Verifier: notary.verifier, Algorithm: algorithms.ECDSA_SECP256K1, VerifierType: verifiers.ETH_ADDRESS, }, }, nil) - carol.mockSign([]byte("carol-signature-bytes")) + notary.mockSign([]byte("some-endorsement-bytes"), []byte("some-signature-bytes")) - aliceEngineMocks.domainSmartContract.On("PrepareTransaction", mock.Anything, mock.Anything, mock.Anything).Return(nil).Run( + remoteEngineMocks.domainSmartContract.On("PrepareTransaction", mock.Anything, mock.Anything, mock.Anything).Return(nil).Run( func(args mock.Arguments) { cv, err := testABI[0].Inputs.ParseExternalData(map[string]any{ "inputs": []any{tktypes.Bytes32(tktypes.RandBytes(32))}, @@ -715,113 +656,89 @@ func TestPrivateTxManagerEndorsementGroup(t *testing.T) { tx.PreparedPublicTransaction = &pldapi.TransactionInput{ ABI: abi.ABI{testABI[0]}, TransactionBase: pldapi.TransactionBase{ - To: domainAddress, - Data: tktypes.RawJSON(jsonData), + To: domainAddress, + Data: tktypes.RawJSON(jsonData), + PublicTxOptions: pldapi.PublicTxOptions{Gas: confutil.P(tktypes.HexUint64(100000))}, }, } - aliceEndorsed, bobEndorsed, carolEndorsed := false, false, false - for _, endorsement := range tx.PostAssembly.Endorsements { - switch endorsement.Verifier.Verifier { - case alice.verifier: - if reflect.DeepEqual(endorsement.Payload, []byte("alice-signature-bytes")) { - aliceEndorsed = true - } - case bob.verifier: - if reflect.DeepEqual(endorsement.Payload, []byte("bob-signature-bytes")) { - bobEndorsed = true - } - case carol.verifier: - if reflect.DeepEqual(endorsement.Payload, []byte("carol-signature-bytes")) { - carolEndorsed = true - } - } - } - assert.True(t, aliceEndorsed) - assert.True(t, bobEndorsed) - assert.True(t, carolEndorsed) }, ) + testTransactionID := confutil.P(uuid.New()) - tx := &components.PrivateTransaction{ - ID: uuid.New(), - Inputs: &components.TransactionInputs{ - Domain: "domain1", - To: *domainAddress, - From: alice.identityLocator, - }, - } - - mockPublicTxBatch := componentmocks.NewPublicTxBatch(t) - mockPublicTxBatch.On("Finalize", mock.Anything).Return().Maybe() - mockPublicTxBatch.On("CleanUp", mock.Anything).Return().Maybe() - - mockPublicTxManager := aliceEngineMocks.publicTxManager.(*componentmocks.PublicTxManager) - mockPublicTxManager.On("PrepareSubmissionBatch", mock.Anything, mock.Anything).Return(mockPublicTxBatch, nil) + _ = mockWritePublicTxsOk(remoteEngineMocks) signingAddr := tktypes.RandAddress() - aliceEngineMocks.keyManager.On("ResolveEthAddressBatchNewDatabaseTX", mock.Anything, []string{"signer1"}). + remoteEngineMocks.keyManager.On("ResolveEthAddressBatchNewDatabaseTX", mock.Anything, []string{"signer1"}). Return([]*tktypes.EthAddress{signingAddr}, nil) - publicTransactions := []components.PublicTxAccepted{ - newFakePublicTx(&components.PublicTxSubmission{ - Bindings: []*components.PaladinTXReference{{TransactionID: tx.ID, TransactionType: pldapi.TransactionTypePrivate.Enum()}}, - PublicTxInput: pldapi.PublicTxInput{ - From: signingAddr, - }, - }, nil), - } - mockPublicTxBatch.On("Submit", mock.Anything, mock.Anything).Return(nil) - mockPublicTxBatch.On("Rejected").Return([]components.PublicTxRejected{}) - mockPublicTxBatch.On("Accepted").Return(publicTransactions) - mockPublicTxBatch.On("Completed", mock.Anything, true).Return() - + // Flush of domain context happens on the remote node (the notary) dcFlushed := make(chan error, 1) - aliceEngineMocks.domainContext.On("Flush", mock.Anything).Return(func(err error) { + remoteEngineMocks.domainContext.On("Flush", mock.Anything).Return(func(err error) { dcFlushed <- err }, nil) - err := aliceEngine.Start() + err := privateTxManager.Start() assert.NoError(t, err) - err = aliceEngine.handleNewTx(ctx, tx) + tx := &components.ValidatedTransaction{ + Function: &components.ResolvedFunction{ + Definition: testABI[0], + }, + Transaction: &pldapi.Transaction{ + ID: testTransactionID, + TransactionBase: pldapi.TransactionBase{ + Domain: "domain1", + To: domainAddress, + From: alice.identityLocator, + }, + }, + } + + err = privateTxManager.HandleNewTx(ctx, privateTxManager.DB(), tx) assert.NoError(t, err) - status := pollForStatus(ctx, t, "dispatched", aliceEngine, domainAddressString, tx.ID.String(), 200*time.Second) + status := pollForStatus(ctx, t, "delegated", privateTxManager, domainAddressString, testTransactionID.String(), 200*time.Second) + assert.Equal(t, "delegated", status) + + status = pollForStatus(ctx, t, "dispatched", remoteEngine, domainAddressString, testTransactionID.String(), 200*time.Second) assert.Equal(t, "dispatched", status) require.NoError(t, <-dcFlushed) -} -func TestPrivateTxManagerDependantTransactionEndorsedOutOfOrder(t *testing.T) { - // extension to the TestPrivateTxManagerEndorsementGroup test - // 2 transactions, one dependant on the other - // we purposely endorse the first transaction late to ensure that the 2nd transaction - // is still sequenced behind the first +} - log.SetLevel("trace") +func TestPrivateTxManagerRemoteNotaryEndorserRetry(t *testing.T) { + if testing.Short() { + // test test takes a second for the timeout to fire + // TODO investigate ways to mock the time.AfterFunc function to avoid this delay + t.Skip("skipping test in short mode.") + } ctx := context.Background() + // A transaction that requires exactly one endorsement from a notary (as per noto) and therefore delegates coordination of the transaction to that node domainAddress := tktypes.MustEthAddress(tktypes.RandHex(20)) - domainAddressString := domainAddress.String() + localNodeName := "localNode" + remoteNodeName := "remoteNode" + privateTxManager, localNodeMocks := NewPrivateTransactionMgrForPackageTesting(t, localNodeName) + localNodeMocks.mockDomain(domainAddress) - aliceNodeName := "aliceNode" - bobNodeName := "bobNode" - aliceEngine, aliceEngineMocks := NewPrivateTransactionMgrForTesting(t, aliceNodeName) - aliceEngineMocks.mockDomain(domainAddress) + domainAddressString := domainAddress.String() - _, bobEngineMocks := NewPrivateTransactionMgrForTesting(t, bobNodeName) - bobEngineMocks.mockDomain(domainAddress) + remoteEngine, remoteEngineMocks := NewPrivateTransactionMgrForPackageTesting(t, remoteNodeName) + remoteEngineMocks.mockDomain(domainAddress) - alice := newPartyForTesting(ctx, "alice", aliceNodeName, aliceEngineMocks) - bob := newPartyForTesting(ctx, "bob", bobNodeName, bobEngineMocks) + alice := newPartyForTesting(ctx, "alice", localNodeName, localNodeMocks) + notary := newPartyForTesting(ctx, "notary", remoteNodeName, remoteEngineMocks) - alice.mockResolve(ctx, bob) + alice.mockResolve(ctx, notary) + notary.mockResolve(ctx, alice) - aliceEngineMocks.domainSmartContract.On("ContractConfig").Return(&prototk.ContractConfig{ - CoordinatorSelection: prototk.ContractConfig_COORDINATOR_ENDORSER, + initialised := make(chan struct{}, 1) + localNodeMocks.domainSmartContract.On("ContractConfig").Return(&prototk.ContractConfig{ + CoordinatorSelection: prototk.ContractConfig_COORDINATOR_STATIC, + StaticCoordinator: ¬ary.identityLocator, }) - - aliceEngineMocks.domainSmartContract.On("InitTransaction", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + localNodeMocks.domainSmartContract.On("InitTransaction", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { tx := args.Get(1).(*components.PrivateTransaction) tx.PreAssembly = &components.TransactionPreAssembly{ RequiredVerifiers: []*prototk.ResolveVerifierRequest{ @@ -831,114 +748,870 @@ func TestPrivateTxManagerDependantTransactionEndorsedOutOfOrder(t *testing.T) { VerifierType: verifiers.ETH_ADDRESS, }, { - Lookup: bob.identityLocator, + Lookup: notary.identityLocator, Algorithm: algorithms.ECDSA_SECP256K1, VerifierType: verifiers.ETH_ADDRESS, }, }, } + initialised <- struct{}{} }).Return(nil) - // TODO check that the transaction is signed with this key - - states := []*components.FullState{ - { - ID: tktypes.RandBytes(32), - Schema: tktypes.Bytes32(tktypes.RandBytes(32)), - Data: tktypes.JSONString("foo"), - }, - } - - potentialStates := []*prototk.NewState{ - { - SchemaId: states[0].Schema.String(), - StateDataJson: states[0].Data.String(), - }, - } - - tx1 := &components.PrivateTransaction{ - ID: uuid.New(), - Inputs: &components.TransactionInputs{ - Domain: "domain1", - To: *domainAddress, - From: alice.identityLocator, - }, - } - - tx2 := &components.PrivateTransaction{ - ID: uuid.New(), - Inputs: &components.TransactionInputs{ - Domain: "domain1", - To: *domainAddress, - From: alice.identityLocator, - }, - } + assembled := make(chan struct{}, 1) - aliceEngineMocks.domainSmartContract.On("AssembleTransaction", mock.Anything, mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + localNodeMocks.domainSmartContract.On("AssembleTransaction", mock.Anything, mock.Anything, mock.Anything).Run(func(args mock.Arguments) { tx := args.Get(2).(*components.PrivateTransaction) - switch tx.ID.String() { - case tx1.ID.String(): - tx.PostAssembly = &components.TransactionPostAssembly{ - AssemblyResult: prototk.AssembleTransactionResponse_OK, - OutputStates: states, - OutputStatesPotential: potentialStates, - AttestationPlan: []*prototk.AttestationRequest{ - { - Name: "notary", - AttestationType: prototk.AttestationType_ENDORSE, - Algorithm: algorithms.ECDSA_SECP256K1, - VerifierType: verifiers.ETH_ADDRESS, - PayloadType: signpayloads.OPAQUE_TO_RSV, - Parties: []string{ - alice.identityLocator, - bob.identityLocator, - }, - }, + + tx.PostAssembly = &components.TransactionPostAssembly{ + AssemblyResult: prototk.AssembleTransactionResponse_OK, + InputStates: []*components.FullState{ + { + ID: tktypes.RandBytes(32), + Schema: tktypes.Bytes32(tktypes.RandBytes(32)), + Data: tktypes.JSONString("foo"), }, - } - case tx2.ID.String(): - tx.PostAssembly = &components.TransactionPostAssembly{ - AssemblyResult: prototk.AssembleTransactionResponse_OK, - InputStates: states, - AttestationPlan: []*prototk.AttestationRequest{ - { - Name: "notary", - AttestationType: prototk.AttestationType_ENDORSE, - Algorithm: algorithms.ECDSA_SECP256K1, - VerifierType: verifiers.ETH_ADDRESS, - PayloadType: signpayloads.OPAQUE_TO_RSV, - Parties: []string{ - alice.identityLocator, - bob.identityLocator, - }, + }, + AttestationPlan: []*prototk.AttestationRequest{ + { + Name: "notary", + AttestationType: prototk.AttestationType_ENDORSE, + Algorithm: algorithms.ECDSA_SECP256K1, + VerifierType: verifiers.ETH_ADDRESS, + PayloadType: signpayloads.OPAQUE_TO_RSV, + Parties: []string{ + notary.identityLocator, }, }, - } - default: - assert.Fail(t, "Unexpected transaction ID") + }, } - }).Times(2).Return(nil) + assembled <- struct{}{} - sentEndorsementRequest := make(chan struct{}, 1) - aliceEngineMocks.transportManager.On("Send", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Run(func(args mock.Arguments) { - sentEndorsementRequest <- struct{}{} + }).Return(nil) + + ignoredDelegateRequest := false + + localNodeMocks.transportManager.On("Send", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + go func() { + assert.Equal(t, remoteNodeName, args.Get(1).(*components.TransportMessage).Node) + transportMessage := args.Get(1).(*components.TransportMessage) + if transportMessage.MessageType == "DelegationRequest" && !ignoredDelegateRequest { + //ignore the first delegate request and force a retry + ignoredDelegateRequest = true + } else { + remoteEngine.ReceiveTransportMessage(ctx, transportMessage) + } + }() }).Return(nil).Maybe() - aliceEngineMocks.domainSmartContract.On("EndorseTransaction", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&components.EndorsementResult{ - Result: prototk.EndorseTransactionResponse_SIGN, - Payload: []byte("alice-endorsement-bytes"), - Endorser: &prototk.ResolvedVerifier{ - Lookup: alice.identityLocator, - Verifier: alice.verifier, - Algorithm: algorithms.ECDSA_SECP256K1, - VerifierType: verifiers.ETH_ADDRESS, - }, - }, nil) + remoteEngineMocks.transportManager.On("Send", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + go func() { + transportMessage := args.Get(1).(*components.TransportMessage) + privateTxManager.ReceiveTransportMessage(ctx, transportMessage) + }() + }).Return(nil).Maybe() - alice.mockSign([]byte("alice-signature-bytes")) + remoteEngineMocks.domainMgr.On("GetSmartContractByAddress", mock.Anything, mock.Anything, *domainAddress).Return(remoteEngineMocks.domainSmartContract, nil) - aliceEngineMocks.domainSmartContract.On("PrepareTransaction", mock.Anything, mock.Anything, mock.Anything).Return(nil).Run( - func(args mock.Arguments) { + //TODO match endorsement request and verifier args + remoteEngineMocks.domainSmartContract.On("ContractConfig").Return(&prototk.ContractConfig{ + CoordinatorSelection: prototk.ContractConfig_COORDINATOR_ENDORSER, + }) + remoteEngineMocks.domainSmartContract.On("EndorseTransaction", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&components.EndorsementResult{ + Result: prototk.EndorseTransactionResponse_SIGN, + Payload: []byte("some-endorsement-bytes"), + Endorser: &prototk.ResolvedVerifier{ + Lookup: notary.identityLocator, //matches whatever was specified in PreAssembly.RequiredVerifiers + Verifier: notary.verifier, + Algorithm: algorithms.ECDSA_SECP256K1, + VerifierType: verifiers.ETH_ADDRESS, + }, + }, nil) + + notary.mockSign([]byte("some-endorsement-bytes"), []byte("some-signature-bytes")) + + remoteEngineMocks.domainSmartContract.On("PrepareTransaction", mock.Anything, mock.Anything, mock.Anything).Return(nil).Run( + func(args mock.Arguments) { + cv, err := testABI[0].Inputs.ParseExternalData(map[string]any{ + "inputs": []any{tktypes.Bytes32(tktypes.RandBytes(32))}, + "outputs": []any{tktypes.Bytes32(tktypes.RandBytes(32))}, + "data": "0xfeedbeef", + }) + require.NoError(t, err) + tx := args[2].(*components.PrivateTransaction) + tx.Signer = "signer1" + jsonData, _ := cv.JSON() + tx.PreparedPublicTransaction = &pldapi.TransactionInput{ + ABI: abi.ABI{testABI[0]}, + TransactionBase: pldapi.TransactionBase{ + To: domainAddress, + Data: tktypes.RawJSON(jsonData), + PublicTxOptions: pldapi.PublicTxOptions{Gas: confutil.P(tktypes.HexUint64(100000))}, + }, + } + }, + ) + testTransactionID := confutil.P(uuid.New()) + + signingAddr := tktypes.RandAddress() + remoteEngineMocks.keyManager.On("ResolveEthAddressBatchNewDatabaseTX", mock.Anything, []string{"signer1"}). + Return([]*tktypes.EthAddress{signingAddr}, nil) + + _ = mockWritePublicTxsOk(remoteEngineMocks) + + // Flush of domain context happens on the remote node (the notary) + dcFlushed := make(chan error, 1) + remoteEngineMocks.domainContext.On("Flush", mock.Anything).Return(func(err error) { + dcFlushed <- err + }, nil) + + err := privateTxManager.Start() + assert.NoError(t, err) + + tx := &components.ValidatedTransaction{ + Function: &components.ResolvedFunction{ + Definition: testABI[0], + }, + Transaction: &pldapi.Transaction{ + ID: testTransactionID, + TransactionBase: pldapi.TransactionBase{ + Domain: "domain1", + To: domainAddress, + From: alice.identityLocator, + }, + }, + } + + err = privateTxManager.HandleNewTx(ctx, privateTxManager.DB(), tx) + assert.NoError(t, err) + + status := pollForStatus(ctx, t, "delegated", privateTxManager, domainAddressString, testTransactionID.String(), 200*time.Second) + assert.Equal(t, "delegated", status) + + status = pollForStatus(ctx, t, "dispatched", remoteEngine, domainAddressString, testTransactionID.String(), 200*time.Second) + assert.Equal(t, "dispatched", status) + + require.NoError(t, <-dcFlushed) + +} + +func TestPrivateTxManagerEndorsementGroup(t *testing.T) { + + ctx := context.Background() + // A transaction that requires endorsement from a group of remote endorsers (as per pente and its 100% endorsement policy) + // In this scenario there is only one active transaction and therefore no risk of contention so the transactions is coordinated + // and dispatched locally. The only expected interaction with the remote nodes is to request endorsements and to distribute the new states + + domainAddress := tktypes.MustEthAddress(tktypes.RandHex(20)) + domainAddressString := domainAddress.String() + + aliceNodeName := "aliceNode" + bobNodeName := "bobNode" + carolNodeName := "carolNode" + + aliceEngine, aliceEngineMocks := NewPrivateTransactionMgrForPackageTesting(t, aliceNodeName) + aliceEngineMocks.mockDomain(domainAddress) + + bobEngine, bobEngineMocks := NewPrivateTransactionMgrForPackageTesting(t, bobNodeName) + bobEngineMocks.mockDomain(domainAddress) + + carolEngine, carolEngineMocks := NewPrivateTransactionMgrForPackageTesting(t, carolNodeName) + carolEngineMocks.mockDomain(domainAddress) + + alice := newPartyForTesting(ctx, "alice", aliceNodeName, aliceEngineMocks) + bob := newPartyForTesting(ctx, "bob", bobNodeName, bobEngineMocks) + carol := newPartyForTesting(ctx, "carol", carolNodeName, carolEngineMocks) + + alice.mockResolve(ctx, bob) + alice.mockResolve(ctx, carol) + + testTransactionID := confutil.P(uuid.New()) + + aliceEngineMocks.domainSmartContract.On("InitTransaction", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + tx := args.Get(1).(*components.PrivateTransaction) + tx.PreAssembly = &components.TransactionPreAssembly{ + TransactionSpecification: &prototk.TransactionSpecification{ + TransactionId: testTransactionID.String(), + }, + RequiredVerifiers: []*prototk.ResolveVerifierRequest{ + { + Lookup: alice.identityLocator, + Algorithm: algorithms.ECDSA_SECP256K1, + VerifierType: verifiers.ETH_ADDRESS, + }, + { + Lookup: bob.identityLocator, + Algorithm: algorithms.ECDSA_SECP256K1, + VerifierType: verifiers.ETH_ADDRESS, + }, + { + Lookup: carol.identityLocator, + Algorithm: algorithms.ECDSA_SECP256K1, + VerifierType: verifiers.ETH_ADDRESS, + }, + }, + } + }).Return(nil) + + aliceEngineMocks.domainSmartContract.On("AssembleTransaction", mock.Anything, mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + tx := args.Get(2).(*components.PrivateTransaction) + + tx.PostAssembly = &components.TransactionPostAssembly{ + AssemblyResult: prototk.AssembleTransactionResponse_OK, + InputStates: []*components.FullState{ + { + ID: tktypes.RandBytes(32), + Schema: tktypes.Bytes32(tktypes.RandBytes(32)), + Data: tktypes.JSONString("foo"), + }, + }, + AttestationPlan: []*prototk.AttestationRequest{ + { + Name: "endorsers", + AttestationType: prototk.AttestationType_ENDORSE, + Algorithm: algorithms.ECDSA_SECP256K1, + VerifierType: verifiers.ETH_ADDRESS, + PayloadType: signpayloads.OPAQUE_TO_RSV, + Parties: []string{ + alice.identityLocator, + bob.identityLocator, + carol.identityLocator, + }, + }, + }, + } + + }).Return(nil) + + //Set up mocks that allow nodes to exchange messages with each other + mockNetwork(t, []privateTransactionMgrForPackageTesting{ + aliceEngine, + bobEngine, + carolEngine, + }) + + //set up the mocks on bob and carols engines that are need on the endorse code path (and of course also on alice's engine because she is an endorser too) + + bobEngineMocks.domainMgr.On("GetSmartContractByAddress", mock.Anything, mock.Anything, *domainAddress).Return(bobEngineMocks.domainSmartContract, nil) + carolEngineMocks.domainMgr.On("GetSmartContractByAddress", mock.Anything, mock.Anything, *domainAddress).Return(carolEngineMocks.domainSmartContract, nil) + + //TODO match endorsement request and verifier args + aliceEngineMocks.domainSmartContract.On("ContractConfig").Return(&prototk.ContractConfig{ + CoordinatorSelection: prototk.ContractConfig_COORDINATOR_ENDORSER, + }) + + aliceEngineMocks.mockForEndorsement(t, *testTransactionID, &alice, []byte("alice-endorsement-bytes"), []byte("alice-signature-bytes")) + bobEngineMocks.mockForEndorsement(t, *testTransactionID, &bob, []byte("bob-endorsement-bytes"), []byte("bob-signature-bytes")) + carolEngineMocks.mockForEndorsement(t, *testTransactionID, &carol, []byte("carol-endorsement-bytes"), []byte("carol-signature-bytes")) + + dcFlushed := make(chan error, 1) + + //Set up mocks on alice's engine that are needed for alice to be the submitter of the transaction + aliceEngineMocks.mockForSubmitter(t, testTransactionID, domainAddress, + map[string][]byte{ //expected endorsement signatures + alice.verifier: []byte("alice-signature-bytes"), + bob.verifier: []byte("bob-signature-bytes"), + carol.verifier: []byte("carol-signature-bytes"), + }, + dcFlushed, + ) + + err := aliceEngine.Start() + assert.NoError(t, err) + + tx := &components.ValidatedTransaction{ + Function: &components.ResolvedFunction{ + Definition: testABI[0], + }, + Transaction: &pldapi.Transaction{ + ID: testTransactionID, + TransactionBase: pldapi.TransactionBase{ + Domain: "domain1", + To: domainAddress, + From: alice.identityLocator, + }, + }, + } + + err = aliceEngine.HandleNewTx(ctx, aliceEngine.DB(), tx) + assert.NoError(t, err) + + status := pollForStatus(ctx, t, "dispatched", aliceEngine, domainAddressString, testTransactionID.String(), 200*time.Second) + assert.Equal(t, "dispatched", status) + + require.NoError(t, <-dcFlushed) +} + +func TestPrivateTxManagerEndorsementGroupDynamicCoordinator(t *testing.T) { + + // Extension to TestPrivateTxManagerEndorsementGroup with the addition of emulating the progression of block height + // beyond the range boundaries so that nodes switch coordinator roles + ctx := context.Background() + + domainAddress := tktypes.MustEthAddress(tktypes.RandHex(20)) + domainAddressString := domainAddress.String() + + testTransactionID1 := confutil.P(uuid.New()) + testTransactionID2 := confutil.P(uuid.New()) + + aliceNodeName := "aliceNode" + bobNodeName := "bobNode" + carolNodeName := "carolNode" + + aliceEngine, aliceEngineMocks := NewPrivateTransactionMgrForPackageTesting(t, aliceNodeName) + aliceEngineMocks.mockDomain(domainAddress) + + bobEngine, bobEngineMocks := NewPrivateTransactionMgrForPackageTesting(t, bobNodeName) + bobEngineMocks.mockDomain(domainAddress) + + carolEngine, carolEngineMocks := NewPrivateTransactionMgrForPackageTesting(t, carolNodeName) + carolEngineMocks.mockDomain(domainAddress) + + alice := newPartyForTesting(ctx, "alice", aliceNodeName, aliceEngineMocks) + bob := newPartyForTesting(ctx, "bob", bobNodeName, bobEngineMocks) + carol := newPartyForTesting(ctx, "carol", carolNodeName, carolEngineMocks) + + alice.mockResolve(ctx, bob) + alice.mockResolve(ctx, carol) + + bob.mockResolve(ctx, alice) + bob.mockResolve(ctx, carol) + + carol.mockResolve(ctx, bob) + carol.mockResolve(ctx, alice) + + //Set up mocks on alice's transaction manager that are needed for it to be the sender (aka assembler) of transaction 1 + + aliceEngineMocks.domainSmartContract.On("InitTransaction", mock.Anything, mock.MatchedBy(privateTransactionMatcher(*testTransactionID1, *testTransactionID2))).Run(func(args mock.Arguments) { + tx := args.Get(1).(*components.PrivateTransaction) + tx.PreAssembly = &components.TransactionPreAssembly{ + TransactionSpecification: &prototk.TransactionSpecification{ + TransactionId: tx.ID.String(), + }, + RequiredVerifiers: []*prototk.ResolveVerifierRequest{ + { + Lookup: alice.identityLocator, + Algorithm: algorithms.ECDSA_SECP256K1, + VerifierType: verifiers.ETH_ADDRESS, + }, + { + Lookup: bob.identityLocator, + Algorithm: algorithms.ECDSA_SECP256K1, + VerifierType: verifiers.ETH_ADDRESS, + }, + { + Lookup: carol.identityLocator, + Algorithm: algorithms.ECDSA_SECP256K1, + VerifierType: verifiers.ETH_ADDRESS, + }, + }, + } + }).Return(nil) + + aliceEngineMocks.domainSmartContract.On("AssembleTransaction", mock.Anything, mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + tx := args.Get(2).(*components.PrivateTransaction) + + tx.PostAssembly = &components.TransactionPostAssembly{ + AssemblyResult: prototk.AssembleTransactionResponse_OK, + InputStates: []*components.FullState{ + { + ID: tktypes.RandBytes(32), + Schema: tktypes.Bytes32(tktypes.RandBytes(32)), + Data: tktypes.JSONString("foo"), + }, + }, + AttestationPlan: []*prototk.AttestationRequest{ + { + Name: "endorsers", + AttestationType: prototk.AttestationType_ENDORSE, + Algorithm: algorithms.ECDSA_SECP256K1, + VerifierType: verifiers.ETH_ADDRESS, + PayloadType: signpayloads.OPAQUE_TO_RSV, + Parties: []string{ + alice.identityLocator, + bob.identityLocator, + carol.identityLocator, + }, + }, + }, + } + + }).Return(nil) + + //Set up mocks that allow nodes to exchange messages with each other + mockNetwork(t, []privateTransactionMgrForPackageTesting{ + aliceEngine, + bobEngine, + carolEngine, + }) + + //Set up mocks on alice's engine that are needed for alice to be the coordinator of the first transaction + aliceEngineMocks.domainSmartContract.On("ContractConfig").Return(&prototk.ContractConfig{ + CoordinatorSelection: prototk.ContractConfig_COORDINATOR_ENDORSER, + }) + + //set up the mocks on all 3 engines that are need on the endorse code path + bobEngineMocks.domainMgr.On("GetSmartContractByAddress", mock.Anything, mock.Anything, *domainAddress).Return(bobEngineMocks.domainSmartContract, nil) + carolEngineMocks.domainMgr.On("GetSmartContractByAddress", mock.Anything, mock.Anything, *domainAddress).Return(carolEngineMocks.domainSmartContract, nil) + + aliceEngineMocks.mockForEndorsement(t, *testTransactionID1, &alice, []byte("alice-endorsement-bytes1"), []byte("alice-signature-bytes1")) + bobEngineMocks.mockForEndorsement(t, *testTransactionID1, &bob, []byte("bob-endorsement-bytes1"), []byte("bob-signature-bytes1")) + carolEngineMocks.mockForEndorsement(t, *testTransactionID1, &carol, []byte("carol-endorsement-bytes1"), []byte("carol-signature-bytes1")) + + dcFlushed := make(chan error, 1) + //Set up mocks on alice's engine that are needed for alice to be the submitter of the first transaction + aliceEngineMocks.mockForSubmitter(t, testTransactionID1, domainAddress, + map[string][]byte{ //expected endorsement signatures + alice.verifier: []byte("alice-signature-bytes1"), + bob.verifier: []byte("bob-signature-bytes1"), + carol.verifier: []byte("carol-signature-bytes1"), + }, + dcFlushed, + ) + + err := aliceEngine.Start() + assert.NoError(t, err) + + tx1 := &components.ValidatedTransaction{ + Function: &components.ResolvedFunction{ + Definition: testABI[0], + }, + Transaction: &pldapi.Transaction{ + ID: testTransactionID1, + TransactionBase: pldapi.TransactionBase{ + Domain: "domain1", + To: domainAddress, + From: alice.identityLocator, + }, + }, + } + + tx2 := &components.ValidatedTransaction{ + Function: &components.ResolvedFunction{ + Definition: testABI[0], + }, + Transaction: &pldapi.Transaction{ + ID: testTransactionID2, + TransactionBase: pldapi.TransactionBase{ + Domain: "domain1", + To: domainAddress, + From: alice.identityLocator, + }, + }, + } + + //Start off on block 99 where alice should be coordinator + + aliceEngine.SetBlockHeight(ctx, 99) + bobEngine.SetBlockHeight(ctx, 99) + carolEngine.SetBlockHeight(ctx, 99) + + err = aliceEngine.HandleNewTx(ctx, aliceEngine.DB(), tx1) + assert.NoError(t, err) + + status := pollForStatus(ctx, t, "dispatched", aliceEngine, domainAddressString, testTransactionID1.String(), 200*time.Second) + assert.Equal(t, "dispatched", status) + + // Setup mocks on bob to be coordinator + bobEngineMocks.domainSmartContract.On("ContractConfig").Return(&prototk.ContractConfig{ + CoordinatorSelection: prototk.ContractConfig_COORDINATOR_ENDORSER, + }) + + aliceEngineMocks.mockForEndorsement(t, *testTransactionID2, &alice, []byte("alice-endorsement-bytes2"), []byte("alice-signature-bytes2")) + bobEngineMocks.mockForEndorsement(t, *testTransactionID2, &bob, []byte("bob-endorsement-bytes2"), []byte("bob-signature-bytes2")) + carolEngineMocks.mockForEndorsement(t, *testTransactionID2, &carol, []byte("carol-endorsement-bytes2"), []byte("carol-signature-bytes2")) + + //Set up mocks on bob's engine that are needed for bob to be the submitter of the second transaction + bobEngineMocks.mockForSubmitter(t, testTransactionID2, domainAddress, + map[string][]byte{ //expected endorsement signatures + alice.verifier: []byte("alice-signature-bytes2"), + bob.verifier: []byte("bob-signature-bytes2"), + carol.verifier: []byte("carol-signature-bytes2"), + }, + dcFlushed, + ) + + aliceEngine.SetBlockHeight(ctx, 100) + bobEngine.SetBlockHeight(ctx, 100) + carolEngine.SetBlockHeight(ctx, 100) + err = aliceEngine.HandleNewTx(ctx, aliceEngine.DB(), tx2) + assert.NoError(t, err) + + status = pollForStatus(ctx, t, "delegated", aliceEngine, domainAddressString, testTransactionID2.String(), 200*time.Second) + assert.Equal(t, "delegated", status) + + status = pollForStatus(ctx, t, "dispatched", bobEngine, domainAddressString, testTransactionID2.String(), 200*time.Second) + assert.Equal(t, "dispatched", status) + + require.NoError(t, <-dcFlushed) +} + +func TestPrivateTxManagerEndorsementGroupDynamicCoordinatorRangeBoundaryHandover(t *testing.T) { + + // Extension to TestPrivateTxManagerEndorsementGroupDynamicCoordinatorRangeBoundary where we simulate the case where + // there are still some transactions in flight when the coordinator role switches + // and assert that transactions that are not yet passed the point of no return (i.e. are sequenced but not dispatched ) are transferred to, and eventually submitted by, the new coordinator + ctx := context.Background() + + domainAddress := tktypes.MustEthAddress(tktypes.RandHex(20)) + domainAddressString := domainAddress.String() + + testTransactionID1 := confutil.P(uuid.New()) + testTransactionID2 := confutil.P(uuid.New()) + + aliceNodeName := "aliceNode" + bobNodeName := "bobNode" + carolNodeName := "carolNode" + + aliceEngine, aliceEngineMocks := NewPrivateTransactionMgrForPackageTesting(t, aliceNodeName) + aliceEngineMocks.mockDomain(domainAddress) + + bobEngine, bobEngineMocks := NewPrivateTransactionMgrForPackageTesting(t, bobNodeName) + bobEngineMocks.mockDomain(domainAddress) + + carolEngine, carolEngineMocks := NewPrivateTransactionMgrForPackageTesting(t, carolNodeName) + carolEngineMocks.mockDomain(domainAddress) + + alice := newPartyForTesting(ctx, "alice", aliceNodeName, aliceEngineMocks) + bob := newPartyForTesting(ctx, "bob", bobNodeName, bobEngineMocks) + carol := newPartyForTesting(ctx, "carol", carolNodeName, carolEngineMocks) + + alice.mockResolve(ctx, bob) + alice.mockResolve(ctx, carol) + + bob.mockResolve(ctx, alice) + bob.mockResolve(ctx, carol) + + carol.mockResolve(ctx, bob) + carol.mockResolve(ctx, alice) + + //Set up mocks on alice's transaction manager that are needed for it to be the sender (aka assembler) of transaction 1 + aliceEngineMocks.domainSmartContract.On("ContractConfig").Return(&prototk.ContractConfig{ + CoordinatorSelection: prototk.ContractConfig_COORDINATOR_ENDORSER, + }) + + aliceEngineMocks.domainSmartContract.On("InitTransaction", mock.Anything, mock.MatchedBy(privateTransactionMatcher(*testTransactionID1, *testTransactionID2))).Run(func(args mock.Arguments) { + tx := args.Get(1).(*components.PrivateTransaction) + tx.PreAssembly = &components.TransactionPreAssembly{ + TransactionSpecification: &prototk.TransactionSpecification{ + TransactionId: tx.ID.String(), + }, + RequiredVerifiers: []*prototk.ResolveVerifierRequest{ + { + Lookup: alice.identityLocator, + Algorithm: algorithms.ECDSA_SECP256K1, + VerifierType: verifiers.ETH_ADDRESS, + }, + { + Lookup: bob.identityLocator, + Algorithm: algorithms.ECDSA_SECP256K1, + VerifierType: verifiers.ETH_ADDRESS, + }, + { + Lookup: carol.identityLocator, + Algorithm: algorithms.ECDSA_SECP256K1, + VerifierType: verifiers.ETH_ADDRESS, + }, + }, + } + }).Return(nil) + + aliceEngineMocks.domainSmartContract.On("AssembleTransaction", mock.Anything, mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + tx := args.Get(2).(*components.PrivateTransaction) + + tx.PostAssembly = &components.TransactionPostAssembly{ + AssemblyResult: prototk.AssembleTransactionResponse_OK, + InputStates: []*components.FullState{ + { + ID: tktypes.RandBytes(32), + Schema: tktypes.Bytes32(tktypes.RandBytes(32)), + Data: tktypes.JSONString("foo"), + }, + }, + AttestationPlan: []*prototk.AttestationRequest{ + { + Name: "endorsers", + AttestationType: prototk.AttestationType_ENDORSE, + Algorithm: algorithms.ECDSA_SECP256K1, + VerifierType: verifiers.ETH_ADDRESS, + PayloadType: signpayloads.OPAQUE_TO_RSV, + Parties: []string{ + alice.identityLocator, + bob.identityLocator, + carol.identityLocator, + }, + }, + }, + } + + }).Return(nil) + + //Set up mocks that allow nodes to exchange messages with each other + mockNetwork(t, []privateTransactionMgrForPackageTesting{ + aliceEngine, + bobEngine, + carolEngine, + }) + + //Set up mocks on bob's engine that are needed for alice to be the coordinator of the first transaction + bobEngineMocks.domainSmartContract.On("ContractConfig").Return(&prototk.ContractConfig{ + CoordinatorSelection: prototk.ContractConfig_COORDINATOR_ENDORSER, + }) + + //set up the mocks on all 3 engines that are need on the endorse code path + bobEngineMocks.domainMgr.On("GetSmartContractByAddress", mock.Anything, mock.Anything, *domainAddress).Return(bobEngineMocks.domainSmartContract, nil) + carolEngineMocks.domainMgr.On("GetSmartContractByAddress", mock.Anything, mock.Anything, *domainAddress).Return(carolEngineMocks.domainSmartContract, nil) + + aliceEngineMocks.mockForEndorsement(t, *testTransactionID1, &alice, []byte("alice-endorsement-bytes1"), []byte("alice-signature-bytes1")) + bobEngineMocks.mockForEndorsement(t, *testTransactionID1, &bob, []byte("bob-endorsement-bytes1"), []byte("bob-signature-bytes1")) + carolEngineMocks.mockForEndorsement(t, *testTransactionID1, &carol, []byte("carol-endorsement-bytes1"), []byte("carol-signature-bytes1")) + + dcFlushed := make(chan error, 1) + //Set up mocks on bobs's engine that are needed to be the submitter of the first transaction + bobEngineMocks.mockForSubmitter(t, testTransactionID1, domainAddress, + map[string][]byte{ //expected endorsement signatures + alice.verifier: []byte("alice-signature-bytes1"), + bob.verifier: []byte("bob-signature-bytes1"), + carol.verifier: []byte("carol-signature-bytes1"), + }, + dcFlushed, + ) + + err := aliceEngine.Start() + assert.NoError(t, err) + + tx1 := &components.ValidatedTransaction{ + Function: &components.ResolvedFunction{ + Definition: testABI[0], + }, + Transaction: &pldapi.Transaction{ + ID: testTransactionID1, + TransactionBase: pldapi.TransactionBase{ + Domain: "domain1", + To: domainAddress, + From: alice.identityLocator, + }, + }, + } + + tx2 := &components.ValidatedTransaction{ + Function: &components.ResolvedFunction{ + Definition: testABI[0], + }, + Transaction: &pldapi.Transaction{ + ID: testTransactionID2, + TransactionBase: pldapi.TransactionBase{ + Domain: "domain1", + To: domainAddress, + From: alice.identityLocator, + }, + }, + } + + aliceEngine.SetBlockHeight(ctx, 199) + bobEngine.SetBlockHeight(ctx, 199) + carolEngine.SetBlockHeight(ctx, 199) + + err = aliceEngine.HandleNewTx(ctx, aliceEngine.DB(), tx1) + assert.NoError(t, err) + + //wait until alice had delegated to bob and bob has dispatched + status := pollForStatus(ctx, t, "delegated", aliceEngine, domainAddressString, testTransactionID1.String(), 200*time.Second) + assert.Equal(t, "delegated", status) + + status = pollForStatus(ctx, t, "dispatched", bobEngine, domainAddressString, testTransactionID1.String(), 200*time.Second) + assert.Equal(t, "dispatched", status) + + // tx1 is now past the point of no return + + // set up mocks for second transaction to be partially endorsed + aliceEngineMocks.mockForEndorsement(t, *testTransactionID2, &alice, []byte("alice-endorsement-bytes2"), []byte("alice-signature-bytes2")) + bobEngineMocks.mockForEndorsement(t, *testTransactionID2, &bob, []byte("bob-endorsement-bytes2"), []byte("bob-signature-bytes2")) + + //prepare carol to endorse transaction 2 but not until after the block height has been incremented + carolEndorsementTrigger := carolEngineMocks.mockForEndorsementOnTrigger(t, *testTransactionID2, &carol, []byte("carol-endorsement-bytes2"), []byte("carol-signature-bytes2")) + + //send transaction 2 and it should get delegated to bob, because we have not moved the block height but it should not get dispatched yet because carol's endorsement is delayed + err = aliceEngine.HandleNewTx(ctx, aliceEngine.DB(), tx2) + assert.NoError(t, err) + + status = pollForStatus(ctx, t, "delegated", aliceEngine, domainAddressString, testTransactionID2.String(), 200*time.Second) + assert.Equal(t, "delegated", status) + + //wait until bob has sequenced the transaction and is waiting for endorsement + status = pollForStatus(ctx, t, "signed", bobEngine, domainAddressString, testTransactionID2.String(), 200*time.Second) + assert.Equal(t, "signed", status) + + // Setup mocks on carol to be coordinator + carolEngineMocks.domainSmartContract.On("ContractConfig").Return(&prototk.ContractConfig{ + CoordinatorSelection: prototk.ContractConfig_COORDINATOR_ENDORSER, + }) + + //Set up mocks on carol's engine that are needed to be the submitter of the second transaction + carolEngineMocks.mockForSubmitter(t, testTransactionID2, domainAddress, + map[string][]byte{ //expected endorsement signatures + alice.verifier: []byte("alice-signature-bytes2"), + bob.verifier: []byte("bob-signature-bytes2"), + carol.verifier: []byte("carol-signature-bytes2"), + }, + dcFlushed, + ) + + aliceEngine.SetBlockHeight(ctx, 200) + bobEngine.SetBlockHeight(ctx, 200) + carolEngine.SetBlockHeight(ctx, 200) + + carolEndorsementTrigger() + + status = pollForStatus(ctx, t, "delegated", aliceEngine, domainAddressString, testTransactionID2.String(), 200*time.Second) + assert.Equal(t, "delegated", status) + + status = pollForStatus(ctx, t, "delegated", bobEngine, domainAddressString, testTransactionID2.String(), 200*time.Second) + assert.Equal(t, "delegated", status) + + status = pollForStatus(ctx, t, "dispatched", carolEngine, domainAddressString, testTransactionID2.String(), 200*time.Second) + assert.Equal(t, "dispatched", status) + + require.NoError(t, <-dcFlushed) +} + +func TestPrivateTxManagerDependantTransactionEndorsedOutOfOrder(t *testing.T) { + // extension to the TestPrivateTxManagerEndorsementGroup test + // 2 transactions, one dependant on the other + // we purposely endorse the first transaction late to ensure that the 2nd transaction + // is still sequenced behind the first + + ctx := context.Background() + + domainAddress := tktypes.MustEthAddress(tktypes.RandHex(20)) + domainAddressString := domainAddress.String() + + aliceNodeName := "aliceNode" + bobNodeName := "bobNode" + aliceEngine, aliceEngineMocks := NewPrivateTransactionMgrForPackageTesting(t, aliceNodeName) + aliceEngineMocks.mockDomain(domainAddress) + + _, bobEngineMocks := NewPrivateTransactionMgrForPackageTesting(t, bobNodeName) + bobEngineMocks.mockDomain(domainAddress) + + alice := newPartyForTesting(ctx, "alice", aliceNodeName, aliceEngineMocks) + bob := newPartyForTesting(ctx, "bob", bobNodeName, bobEngineMocks) + + alice.mockResolve(ctx, bob) + + aliceEngineMocks.domainSmartContract.On("ContractConfig").Return(&prototk.ContractConfig{ + CoordinatorSelection: prototk.ContractConfig_COORDINATOR_ENDORSER, + }) + + aliceEngineMocks.domainSmartContract.On("InitTransaction", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + tx := args.Get(1).(*components.PrivateTransaction) + tx.PreAssembly = &components.TransactionPreAssembly{ + RequiredVerifiers: []*prototk.ResolveVerifierRequest{ + { + Lookup: alice.identityLocator, + Algorithm: algorithms.ECDSA_SECP256K1, + VerifierType: verifiers.ETH_ADDRESS, + }, + { + Lookup: bob.identityLocator, + Algorithm: algorithms.ECDSA_SECP256K1, + VerifierType: verifiers.ETH_ADDRESS, + }, + }, + } + }).Return(nil) + + // TODO check that the transaction is signed with this key + + states := []*components.FullState{ + { + ID: tktypes.RandBytes(32), + Schema: tktypes.Bytes32(tktypes.RandBytes(32)), + Data: tktypes.JSONString("foo"), + }, + } + + potentialStates := []*prototk.NewState{ + { + SchemaId: states[0].Schema.String(), + StateDataJson: states[0].Data.String(), + }, + } + testTransactionID1 := confutil.P(uuid.New()) + testTransactionID2 := confutil.P(uuid.New()) + + aliceEngineMocks.domainSmartContract.On("AssembleTransaction", mock.Anything, mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + tx := args.Get(2).(*components.PrivateTransaction) + switch tx.ID.String() { + case testTransactionID1.String(): + tx.PostAssembly = &components.TransactionPostAssembly{ + AssemblyResult: prototk.AssembleTransactionResponse_OK, + OutputStates: states, + OutputStatesPotential: potentialStates, + AttestationPlan: []*prototk.AttestationRequest{ + { + Name: "notary", + AttestationType: prototk.AttestationType_ENDORSE, + Algorithm: algorithms.ECDSA_SECP256K1, + VerifierType: verifiers.ETH_ADDRESS, + PayloadType: signpayloads.OPAQUE_TO_RSV, + Parties: []string{ + alice.identityLocator, + bob.identityLocator, + }, + }, + }, + } + case testTransactionID2.String(): + tx.PostAssembly = &components.TransactionPostAssembly{ + AssemblyResult: prototk.AssembleTransactionResponse_OK, + InputStates: states, + AttestationPlan: []*prototk.AttestationRequest{ + { + Name: "notary", + AttestationType: prototk.AttestationType_ENDORSE, + Algorithm: algorithms.ECDSA_SECP256K1, + VerifierType: verifiers.ETH_ADDRESS, + PayloadType: signpayloads.OPAQUE_TO_RSV, + Parties: []string{ + alice.identityLocator, + bob.identityLocator, + }, + }, + }, + } + default: + assert.Fail(t, "Unexpected transaction ID") + } + }).Times(2).Return(nil) + + sentEndorsementRequest := make(chan string, 1) + aliceEngineMocks.transportManager.On("Send", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + message := args.Get(1).(*components.TransportMessage) + endorsementRequest := &pbEngine.EndorsementRequest{} + err := proto.Unmarshal(message.Payload, endorsementRequest) + if err != nil { + log.L(ctx).Errorf("Failed to unmarshal endorsement request: %s", err) + return + } + + sentEndorsementRequest <- endorsementRequest.IdempotencyKey + }).Return(nil).Maybe() + + aliceEngineMocks.domainSmartContract.On("EndorseTransaction", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&components.EndorsementResult{ + Result: prototk.EndorseTransactionResponse_SIGN, + Payload: []byte("alice-endorsement-bytes"), + Endorser: &prototk.ResolvedVerifier{ + Lookup: alice.identityLocator, + Verifier: alice.verifier, + Algorithm: algorithms.ECDSA_SECP256K1, + VerifierType: verifiers.ETH_ADDRESS, + }, + }, nil) + + alice.mockSign([]byte("alice-endorsement-bytes"), []byte("alice-signature-bytes")) + + aliceEngineMocks.domainSmartContract.On("PrepareTransaction", mock.Anything, mock.Anything, mock.Anything).Return(nil).Run( + func(args mock.Arguments) { cv, err := testABI[0].Inputs.ParseExternalData(map[string]any{ "inputs": []any{tktypes.Bytes32(tktypes.RandBytes(32))}, "outputs": []any{tktypes.Bytes32(tktypes.RandBytes(32))}, @@ -951,61 +1624,63 @@ func TestPrivateTxManagerDependantTransactionEndorsedOutOfOrder(t *testing.T) { tx.PreparedPublicTransaction = &pldapi.TransactionInput{ ABI: abi.ABI{testABI[0]}, TransactionBase: pldapi.TransactionBase{ - To: domainAddress, - Data: tktypes.RawJSON(jsonData), + To: domainAddress, + Data: tktypes.RawJSON(jsonData), + PublicTxOptions: pldapi.PublicTxOptions{Gas: confutil.P(tktypes.HexUint64(100000))}, }, } }, ) - tx := &components.PrivateTransaction{ - ID: uuid.New(), - } - mockPublicTxBatch := componentmocks.NewPublicTxBatch(t) - mockPublicTxBatch.On("Finalize", mock.Anything).Return().Maybe() - mockPublicTxBatch.On("CleanUp", mock.Anything).Return().Maybe() - - mockPublicTxManager := aliceEngineMocks.publicTxManager.(*componentmocks.PublicTxManager) - mockPublicTxManager.On("PrepareSubmissionBatch", mock.Anything, mock.Anything).Return(mockPublicTxBatch, nil) + _ = mockWritePublicTxsOk(aliceEngineMocks) signingAddr := tktypes.RandAddress() aliceEngineMocks.keyManager.On("ResolveEthAddressBatchNewDatabaseTX", mock.Anything, []string{"signer1", "signer1"}). Return([]*tktypes.EthAddress{signingAddr, signingAddr}, nil) - publicTransactions := []components.PublicTxAccepted{ - newFakePublicTx(&components.PublicTxSubmission{ - Bindings: []*components.PaladinTXReference{{TransactionID: tx.ID, TransactionType: pldapi.TransactionTypePrivate.Enum()}}, - PublicTxInput: pldapi.PublicTxInput{ - From: signingAddr, - }, - }, nil), - newFakePublicTx(&components.PublicTxSubmission{ - Bindings: []*components.PaladinTXReference{{TransactionID: tx.ID, TransactionType: pldapi.TransactionTypePrivate.Enum()}}, - PublicTxInput: pldapi.PublicTxInput{ - From: signingAddr, + err := aliceEngine.Start() + require.NoError(t, err) + + tx1 := &components.ValidatedTransaction{ + Function: &components.ResolvedFunction{ + Definition: testABI[0], + }, + Transaction: &pldapi.Transaction{ + ID: testTransactionID1, + TransactionBase: pldapi.TransactionBase{ + Domain: "domain1", + To: domainAddress, + From: alice.identityLocator, }, - }, nil), + }, } - mockPublicTxBatch.On("Submit", mock.Anything, mock.Anything).Return(nil) - mockPublicTxBatch.On("Rejected").Return([]components.PublicTxRejected{}) - mockPublicTxBatch.On("Accepted").Return(publicTransactions) - mockPublicTxBatch.On("Completed", mock.Anything, true).Return() - err := aliceEngine.Start() + err = aliceEngine.HandleNewTx(ctx, aliceEngine.DB(), tx1) require.NoError(t, err) - err = aliceEngine.handleNewTx(ctx, tx1) - require.NoError(t, err) + tx2 := &components.ValidatedTransaction{ + Function: &components.ResolvedFunction{ + Definition: testABI[0], + }, + Transaction: &pldapi.Transaction{ + ID: testTransactionID2, + TransactionBase: pldapi.TransactionBase{ + Domain: "domain1", + To: domainAddress, + From: alice.identityLocator, + }, + }, + } - err = aliceEngine.handleNewTx(ctx, tx2) + err = aliceEngine.HandleNewTx(ctx, aliceEngine.DB(), tx2) require.NoError(t, err) // Neither transaction should be dispatched yet - s, err := aliceEngine.GetTxStatus(ctx, domainAddressString, tx1.ID.String()) + s, err := aliceEngine.GetTxStatus(ctx, domainAddressString, *testTransactionID1) require.NoError(t, err) assert.NotEqual(t, "dispatch", s.Status) - s, err = aliceEngine.GetTxStatus(ctx, domainAddressString, tx2.ID.String()) + s, err = aliceEngine.GetTxStatus(ctx, domainAddressString, *testTransactionID2) require.NoError(t, err) assert.NotEqual(t, "dispatch", s.Status) @@ -1025,14 +1700,17 @@ func TestPrivateTxManagerDependantTransactionEndorsedOutOfOrder(t *testing.T) { require.NoError(t, err) //wait for both transactions to send an endorsement request each - <-sentEndorsementRequest - <-sentEndorsementRequest + idempotencyKey1 := <-sentEndorsementRequest + idempotencyKey2 := <-sentEndorsementRequest // endorse transaction 2 before 1 and check that 2 is not dispatched before 1 endorsementResponse2 := &pbEngine.EndorsementResponse{ - ContractAddress: domainAddressString, - TransactionId: tx2.ID.String(), - Endorsement: attestationResultAny, + ContractAddress: domainAddressString, + TransactionId: testTransactionID2.String(), + Endorsement: attestationResultAny, + IdempotencyKey: idempotencyKey2, + Party: bob.identityLocator, + AttestationRequestName: "notary", } endorsementResponse2bytes, err := proto.Marshal(endorsementResponse2) require.NoError(t, err) @@ -1047,19 +1725,22 @@ func TestPrivateTxManagerDependantTransactionEndorsedOutOfOrder(t *testing.T) { if !testing.Short() { time.Sleep(1 * time.Second) } - s, err = aliceEngine.GetTxStatus(ctx, domainAddressString, tx1.ID.String()) + s, err = aliceEngine.GetTxStatus(ctx, domainAddressString, *testTransactionID1) require.NoError(t, err) assert.NotEqual(t, "dispatch", s.Status) - s, err = aliceEngine.GetTxStatus(ctx, domainAddressString, tx2.ID.String()) + s, err = aliceEngine.GetTxStatus(ctx, domainAddressString, *testTransactionID2) require.NoError(t, err) assert.NotEqual(t, "dispatch", s.Status) // endorse transaction 1 and check that both it and 2 are dispatched endorsementResponse1 := &pbEngine.EndorsementResponse{ - ContractAddress: domainAddressString, - TransactionId: tx1.ID.String(), - Endorsement: attestationResultAny, + ContractAddress: domainAddressString, + TransactionId: testTransactionID1.String(), + Endorsement: attestationResultAny, + IdempotencyKey: idempotencyKey1, + Party: bob.identityLocator, + AttestationRequestName: "notary", } endorsementResponse1Bytes, err := proto.Marshal(endorsementResponse1) require.NoError(t, err) @@ -1076,10 +1757,10 @@ func TestPrivateTxManagerDependantTransactionEndorsedOutOfOrder(t *testing.T) { dcFlushed <- err }, nil) - status := pollForStatus(ctx, t, "dispatched", aliceEngine, domainAddressString, tx1.ID.String(), 200*time.Second) + status := pollForStatus(ctx, t, "dispatched", aliceEngine, domainAddressString, testTransactionID1.String(), 200*time.Second) assert.Equal(t, "dispatched", status) - status = pollForStatus(ctx, t, "dispatched", aliceEngine, domainAddressString, tx2.ID.String(), 200*time.Second) + status = pollForStatus(ctx, t, "dispatched", aliceEngine, domainAddressString, testTransactionID2.String(), 200*time.Second) assert.Equal(t, "dispatched", status) require.NoError(t, <-dcFlushed) @@ -1097,16 +1778,13 @@ func TestPrivateTxManagerLocalBlockedTransaction(t *testing.T) { func TestPrivateTxManagerDeploy(t *testing.T) { ctx := context.Background() - privateTxManager, mocks := NewPrivateTransactionMgrForTesting(t, "node1") + privateTxManager, mocks := NewPrivateTransactionMgrForPackageTesting(t, "node1") notary := newPartyForTesting(ctx, "notary", "node1", mocks) - tx := &components.PrivateContractDeploy{ - ID: uuid.New(), - Domain: "domain1", - Inputs: tktypes.JSONString(`{"inputs": ["0xfeedbeef"]}`), - } + testTransactionID := confutil.P(uuid.New()) - mocks.domain.On("InitDeploy", mock.Anything, tx).Run(func(args mock.Arguments) { + mocks.domain.On("InitDeploy", mock.Anything, mock.MatchedBy(privateDeployTransactionMatcher(*testTransactionID))).Run(func(args mock.Arguments) { + tx := args.Get(1).(*components.PrivateContractDeploy) tx.RequiredVerifiers = []*prototk.ResolveVerifierRequest{ { Lookup: notary.identityLocator, @@ -1125,7 +1803,8 @@ func TestPrivateTxManagerDeploy(t *testing.T) { testConstructorParameters, err := testConstructorABI.Inputs.ParseJSON([]byte(`{"foo": "42"}`)) require.NoError(t, err) - mocks.domain.On("PrepareDeploy", mock.Anything, tx).Run(func(args mock.Arguments) { + mocks.domain.On("PrepareDeploy", mock.Anything, mock.MatchedBy(privateDeployTransactionMatcher(*testTransactionID))).Run(func(args mock.Arguments) { + tx := args.Get(1).(*components.PrivateContractDeploy) tx.InvokeTransaction = &components.EthTransaction{ FunctionABI: testConstructorABI, To: *domainRegistryAddress, @@ -1138,34 +1817,27 @@ func TestPrivateTxManagerDeploy(t *testing.T) { mocks.keyManager.On("ResolveEthAddressBatchNewDatabaseTX", mock.Anything, []string{"signer1"}). Return([]*tktypes.EthAddress{signingAddr}, nil) - publicTransactions := []components.PublicTxAccepted{ - newFakePublicTx(&components.PublicTxSubmission{ - Bindings: []*components.PaladinTXReference{{TransactionID: tx.ID, TransactionType: pldapi.TransactionTypePrivate.Enum()}}, - PublicTxInput: pldapi.PublicTxInput{ - From: signingAddr, - }, - }, nil), - } - - mockPublicTxBatch := componentmocks.NewPublicTxBatch(t) - - mockPublicTxManager := mocks.publicTxManager.(*componentmocks.PublicTxManager) - mockPublicTxManager.On("PrepareSubmissionBatch", mock.Anything, mock.Anything).Return(mockPublicTxBatch, nil) - - dispatched := make(chan struct{}, 1) - mockPublicTxBatch.On("Submit", mock.Anything, mock.Anything).Return(nil) - mockPublicTxBatch.On("Rejected").Return([]components.PublicTxRejected{}) - mockPublicTxBatch.On("Accepted").Return(publicTransactions) - mockPublicTxBatch.On("Completed", mock.Anything, true).Run(func(args mock.Arguments) { - dispatched <- struct{}{} - }).Return() + dispatched := mockWritePublicTxsOk(mocks) mocks.txManager.On("FinalizeTransactions", mock.Anything, mock.Anything, mock.Anything).Return(nil).Panic("did not expect transaction to be reverted").Maybe() err = privateTxManager.Start() require.NoError(t, err) - err = privateTxManager.handleDeployTx(ctx, tx) + deployTx := &components.ValidatedTransaction{ + Function: &components.ResolvedFunction{ + Definition: testABI[0], + }, + Transaction: &pldapi.Transaction{ + ID: testTransactionID, + SubmitMode: pldapi.SubmitModeAuto.Enum(), + TransactionBase: pldapi.TransactionBase{ + Domain: "domain1", + From: "alice", + }, + }, + } + err = privateTxManager.HandleNewTx(ctx, privateTxManager.DB(), deployTx) require.NoError(t, err) deadlineTimer := time.NewTimer(timeTillDeadline(t)) @@ -1177,24 +1849,66 @@ func TestPrivateTxManagerDeploy(t *testing.T) { } } +func TestPrivateTxManagerDeployErrorInvalidSubmitMode(t *testing.T) { + ctx := context.Background() + + privateTxManager, mocks := NewPrivateTransactionMgrForPackageTesting(t, "node1") + notary := newPartyForTesting(ctx, "notary", "node1", mocks) + + testTransactionID := confutil.P(uuid.New()) + + err := privateTxManager.Start() + require.NoError(t, err) + + deployTx := &components.ValidatedTransaction{ + Function: &components.ResolvedFunction{ + Definition: testABI[0], + }, + Transaction: &pldapi.Transaction{ + ID: testTransactionID, + SubmitMode: pldapi.SubmitModeExternal.Enum(), + TransactionBase: pldapi.TransactionBase{ + Domain: "domain1", + From: notary.identityLocator, + }, + }, + Inputs: tktypes.JSONString(`{"inputs": ["0xfeedbeef"]}`), + } + err = privateTxManager.HandleNewTx(ctx, privateTxManager.DB(), deployTx) + assert.Error(t, err) + assert.Regexp(t, "PD011827", err.Error()) + +} + func TestPrivateTxManagerDeployFailInit(t *testing.T) { // Init errors should fail synchronously ctx := context.Background() - privateTxManager, mocks := NewPrivateTransactionMgrForTesting(t, "node1") + privateTxManager, mocks := NewPrivateTransactionMgrForPackageTesting(t, "node1") - tx := &components.PrivateContractDeploy{ - ID: uuid.New(), - Domain: "domain1", - Inputs: tktypes.JSONString(`{"inputs": ["0xfeedbeef"]}`), - } + testTransactionID := uuid.New() - mocks.domain.On("InitDeploy", mock.Anything, tx).Return(errors.New("failed to init")) + mocks.domain.On("InitDeploy", mock.Anything, mock.MatchedBy(privateDeployTransactionMatcher(testTransactionID))).Return(errors.New("failed to init")) err := privateTxManager.Start() require.NoError(t, err) - err = privateTxManager.handleDeployTx(ctx, tx) + tx := &components.ValidatedTransaction{ + Function: &components.ResolvedFunction{ + Definition: testABI[0], + }, + Transaction: &pldapi.Transaction{ + ID: &testTransactionID, + SubmitMode: pldapi.SubmitModeAuto.Enum(), + TransactionBase: pldapi.TransactionBase{ + Domain: "domain1", + From: "alice@node1", + }, + }, + Inputs: tktypes.JSONString(`{"inputs": ["0xfeedbeef"]}`), + } + + err = privateTxManager.HandleNewTx(ctx, privateTxManager.DB(), tx) assert.Error(t, err) assert.Regexp(t, regexp.MustCompile(".*failed to init.*"), err.Error()) } @@ -1202,16 +1916,26 @@ func TestPrivateTxManagerDeployFailInit(t *testing.T) { func TestPrivateTxManagerDeployFailPrepare(t *testing.T) { ctx := context.Background() - privateTxManager, mocks := NewPrivateTransactionMgrForTesting(t, "node1") + privateTxManager, mocks := NewPrivateTransactionMgrForPackageTesting(t, "node1") notary := newPartyForTesting(ctx, "notary", "node1", mocks) - tx := &components.PrivateContractDeploy{ - ID: uuid.New(), - Domain: "domain1", + testTransactionID := uuid.New() + vtx := &components.ValidatedTransaction{ + Function: &components.ResolvedFunction{ + Definition: testABI[0], + }, + Transaction: &pldapi.Transaction{ + ID: &testTransactionID, + SubmitMode: pldapi.SubmitModeAuto.Enum(), + TransactionBase: pldapi.TransactionBase{ + Domain: "domain1", + From: "alice@node1", + }, + }, Inputs: tktypes.JSONString(`{"inputs": ["0xfeedbeef"]}`), } - - mocks.domain.On("InitDeploy", mock.Anything, tx).Run(func(args mock.Arguments) { + mocks.domain.On("InitDeploy", mock.Anything, mock.MatchedBy(privateDeployTransactionMatcher(testTransactionID))).Run(func(args mock.Arguments) { + tx := args.Get(1).(*components.PrivateContractDeploy) tx.RequiredVerifiers = []*prototk.ResolveVerifierRequest{ { Lookup: notary.identityLocator, @@ -1221,7 +1945,7 @@ func TestPrivateTxManagerDeployFailPrepare(t *testing.T) { } }).Return(nil) - mocks.domain.On("PrepareDeploy", mock.Anything, tx).Return(errors.New("failed to prepare")) + mocks.domain.On("PrepareDeploy", mock.Anything, mock.MatchedBy(privateDeployTransactionMatcher(testTransactionID))).Return(errors.New("failed to prepare")) reverted := make(chan []*components.ReceiptInput, 1) @@ -1236,14 +1960,14 @@ func TestPrivateTxManagerDeployFailPrepare(t *testing.T) { err := privateTxManager.Start() require.NoError(t, err) - err = privateTxManager.handleDeployTx(ctx, tx) + err = privateTxManager.HandleNewTx(ctx, privateTxManager.DB(), vtx) require.NoError(t, err) deadlineTimer := time.NewTimer(timeTillDeadline(t)) select { case receipts := <-reverted: assert.Len(t, receipts, 1) - assert.Equal(t, tx.ID, receipts[0].TransactionID) + assert.Equal(t, testTransactionID, receipts[0].TransactionID) assert.Regexp(t, regexp.MustCompile(".*failed to prepare.*"), receipts[0].FailureMessage) case <-deadlineTimer.C: @@ -1255,16 +1979,27 @@ func TestPrivateTxManagerDeployFailPrepare(t *testing.T) { func TestPrivateTxManagerFailSignerResolve(t *testing.T) { ctx := context.Background() - privateTxManager, mocks := NewPrivateTransactionMgrForTesting(t, "node1") + privateTxManager, mocks := NewPrivateTransactionMgrForPackageTesting(t, "node1") notary := newPartyForTesting(ctx, "notary", "node1", mocks) - tx := &components.PrivateContractDeploy{ - ID: uuid.New(), - Domain: "domain1", + testTransactionID := uuid.New() + vtx := &components.ValidatedTransaction{ + Function: &components.ResolvedFunction{ + Definition: testABI[0], + }, + Transaction: &pldapi.Transaction{ + ID: &testTransactionID, + SubmitMode: pldapi.SubmitModeAuto.Enum(), + TransactionBase: pldapi.TransactionBase{ + Domain: "domain1", + From: "alice@node1", + }, + }, Inputs: tktypes.JSONString(`{"inputs": ["0xfeedbeef"]}`), } - mocks.domain.On("InitDeploy", mock.Anything, tx).Run(func(args mock.Arguments) { + mocks.domain.On("InitDeploy", mock.Anything, mock.MatchedBy(privateDeployTransactionMatcher(testTransactionID))).Run(func(args mock.Arguments) { + tx := args.Get(1).(*components.PrivateContractDeploy) tx.RequiredVerifiers = []*prototk.ResolveVerifierRequest{ { Lookup: notary.identityLocator, @@ -1283,7 +2018,8 @@ func TestPrivateTxManagerFailSignerResolve(t *testing.T) { testConstructorParameters, err := testConstructorABI.Inputs.ParseJSON([]byte(`{"foo": "42"}`)) require.NoError(t, err) - mocks.domain.On("PrepareDeploy", mock.Anything, tx).Run(func(args mock.Arguments) { + mocks.domain.On("PrepareDeploy", mock.Anything, mock.MatchedBy(privateDeployTransactionMatcher(testTransactionID))).Run(func(args mock.Arguments) { + tx := args.Get(1).(*components.PrivateContractDeploy) tx.InvokeTransaction = &components.EthTransaction{ FunctionABI: testConstructorABI, To: *domainRegistryAddress, @@ -1308,14 +2044,14 @@ func TestPrivateTxManagerFailSignerResolve(t *testing.T) { err = privateTxManager.Start() require.NoError(t, err) - err = privateTxManager.handleDeployTx(ctx, tx) + err = privateTxManager.HandleNewTx(ctx, privateTxManager.DB(), vtx) require.NoError(t, err) deadlineTimer := time.NewTimer(timeTillDeadline(t)) select { case receipts := <-reverted: assert.Len(t, receipts, 1) - assert.Equal(t, tx.ID, receipts[0].TransactionID) + assert.Equal(t, testTransactionID, receipts[0].TransactionID) assert.Regexp(t, regexp.MustCompile(".*failed to resolve.*"), receipts[0].FailureMessage) case <-deadlineTimer.C: @@ -1326,16 +2062,28 @@ func TestPrivateTxManagerFailSignerResolve(t *testing.T) { func TestPrivateTxManagerDeployFailNoInvokeOrDeploy(t *testing.T) { ctx := context.Background() - privateTxManager, mocks := NewPrivateTransactionMgrForTesting(t, "node1") + privateTxManager, mocks := NewPrivateTransactionMgrForPackageTesting(t, "node1") notary := newPartyForTesting(ctx, "notary", "node1", mocks) - tx := &components.PrivateContractDeploy{ - ID: uuid.New(), - Domain: "domain1", + testTransactionID := uuid.New() + + vtx := &components.ValidatedTransaction{ + Function: &components.ResolvedFunction{ + Definition: testABI[0], + }, + Transaction: &pldapi.Transaction{ + ID: &testTransactionID, + SubmitMode: pldapi.SubmitModeAuto.Enum(), + TransactionBase: pldapi.TransactionBase{ + Domain: "domain1", + From: "alice@node1", + }, + }, Inputs: tktypes.JSONString(`{"inputs": ["0xfeedbeef"]}`), } - mocks.domain.On("InitDeploy", mock.Anything, tx).Run(func(args mock.Arguments) { + mocks.domain.On("InitDeploy", mock.Anything, mock.MatchedBy(privateDeployTransactionMatcher(testTransactionID))).Run(func(args mock.Arguments) { + tx := args.Get(1).(*components.PrivateContractDeploy) tx.RequiredVerifiers = []*prototk.ResolveVerifierRequest{ { Lookup: notary.identityLocator, @@ -1345,7 +2093,8 @@ func TestPrivateTxManagerDeployFailNoInvokeOrDeploy(t *testing.T) { } }).Return(nil) - mocks.domain.On("PrepareDeploy", mock.Anything, tx).Run(func(args mock.Arguments) { + mocks.domain.On("PrepareDeploy", mock.Anything, mock.MatchedBy(privateDeployTransactionMatcher(testTransactionID))).Run(func(args mock.Arguments) { + tx := args.Get(1).(*components.PrivateContractDeploy) tx.InvokeTransaction = nil tx.DeployTransaction = nil tx.Signer = "signer1" @@ -1368,14 +2117,14 @@ func TestPrivateTxManagerDeployFailNoInvokeOrDeploy(t *testing.T) { err := privateTxManager.Start() require.NoError(t, err) - err = privateTxManager.handleDeployTx(ctx, tx) + err = privateTxManager.HandleNewTx(ctx, privateTxManager.DB(), vtx) require.NoError(t, err) deadlineTimer := time.NewTimer(timeTillDeadline(t)) select { case receipts := <-reverted: assert.Len(t, receipts, 1) - assert.Equal(t, tx.ID, receipts[0].TransactionID) + assert.Equal(t, testTransactionID, receipts[0].TransactionID) assert.Regexp(t, regexp.MustCompile("PD011801"), receipts[0].FailureMessage) assert.Regexp(t, regexp.MustCompile("PD011820"), receipts[0].FailureMessage) @@ -1384,276 +2133,151 @@ func TestPrivateTxManagerDeployFailNoInvokeOrDeploy(t *testing.T) { } } -func TestPrivateTxManagerMiniLoad(t *testing.T) { - t.Skip("This test takes too long to be included by default. It is still useful for local testing") - //TODO this is actually quite a complex test given all the mocking. Maybe this should be converted to a wider component test - // where the real publicTxManager is used rather than a mock - r := rand.New(rand.NewSource(42)) - loadTests := []struct { - name string - latency func() time.Duration - numTransactions int - }{ - {"no-latency", func() time.Duration { return 0 }, 5}, - {"low-latency", func() time.Duration { return 10 * time.Millisecond }, 500}, - {"medium-latency", func() time.Duration { return 50 * time.Millisecond }, 500}, - {"high-latency", func() time.Duration { return 100 * time.Millisecond }, 500}, - {"random-none-to-low-latency", func() time.Duration { return time.Duration(r.Intn(10)) * time.Millisecond }, 500}, - {"random-none-to-high-latency", func() time.Duration { return time.Duration(r.Intn(100)) * time.Millisecond }, 500}, - } - //500 is the maximum we can do in this test for now until either - //a) implement config to allow us to define MaxConcurrentTransactions - //b) implement ( or mock) transaction dispatch processing all the way to confirmation +func TestCallPrivateSmartContractOk(t *testing.T) { - for _, test := range loadTests { - t.Run(test.name, func(t *testing.T) { + ctx := context.Background() + ptx, m := NewPrivateTransactionMgrForPackageTesting(t, "node1") - ctx := context.Background() + _, mPSC := mockDomainSmartContractAndCtx(t, m) - domainAddress := tktypes.MustEthAddress(tktypes.RandHex(20)) - privateTxManager, mocks := NewPrivateTransactionMgrForTestingWithFakePublicTxManager(t, newFakePublicTxManager(t), "node1") - mocks.mockDomain(domainAddress) + fnDef := &abi.Entry{Name: "getIt", Type: abi.Function, Outputs: abi.ParameterArray{ + {Name: "it", Type: "string"}, + }} + resultCV, err := fnDef.Outputs.ParseJSON([]byte(`["thing"]`)) + require.NoError(t, err) - remoteEngine, remoteEngineMocks := NewPrivateTransactionMgrForTestingWithFakePublicTxManager(t, newFakePublicTxManager(t), "node2") - remoteEngineMocks.mockDomain(domainAddress) + bobAddr := tktypes.RandAddress() + m.identityResolver.On("ResolveVerifier", mock.Anything, "bob@node1", algorithms.ECDSA_SECP256K1, verifiers.ETH_ADDRESS). + Return(bobAddr.String(), nil) + mPSC.On("InitCall", mock.Anything, mock.Anything).Return( + []*prototk.ResolveVerifierRequest{ + {Lookup: "bob@node1", Algorithm: algorithms.ECDSA_SECP256K1, VerifierType: verifiers.ETH_ADDRESS}, + }, nil, + ) + mPSC.On("ExecCall", mock.Anything, mock.Anything, mock.Anything, mock.MatchedBy(func(verifiers []*prototk.ResolvedVerifier) bool { + require.Equal(t, bobAddr.String(), verifiers[0].Verifier) + return true + })).Return( + resultCV, nil, + ) - dependenciesByTransactionID := make(map[string][]string) // populated during assembly stage - nonceByTransactionID := make(map[string]uint64) // populated when dispatch event recieved and used later to check that the nonce order matchs the dependency order + res, err := ptx.CallPrivateSmartContract(ctx, &components.TransactionInputs{ + To: mPSC.Address(), + Inputs: tktypes.RawJSON(`{}`), + Function: fnDef, + }) + require.NoError(t, err) + jsonData, err := res.JSON() + require.NoError(t, err) + require.JSONEq(t, `{"it": "thing"}`, string(jsonData)) - unclaimedPendingStatesToMintingTransaction := make(map[tktypes.Bytes32]string) +} - mocks.domainSmartContract.On("InitTransaction", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { - tx := args.Get(1).(*components.PrivateTransaction) - tx.PreAssembly = &components.TransactionPreAssembly{ - RequiredVerifiers: []*prototk.ResolveVerifierRequest{ - { - Lookup: "alice", - Algorithm: algorithms.ECDSA_SECP256K1, - VerifierType: verifiers.ETH_ADDRESS, - }, - }, - } - }).Return(nil) - mocks.identityResolver.On("ResolveVerifierAsync", mock.Anything, "alice", algorithms.ECDSA_SECP256K1, verifiers.ETH_ADDRESS, mock.Anything, mock.Anything).Run(func(args mock.Arguments) { - resovleFn := args.Get(4).(func(context.Context, string)) - resovleFn(ctx, "aliceVerifier") - }).Return(nil) - - failEarly := make(chan string, 1) - - assembleConcurrency := 0 - mocks.domainSmartContract.On("AssembleTransaction", mock.Anything, mock.Anything, mock.Anything).Run(func(args mock.Arguments) { - //assert that we are not assembling more than 1 transaction at a time - if assembleConcurrency > 0 { - failEarly <- "Assembling more than one transaction at a time" - } - require.Equal(t, assembleConcurrency, 0, "Assembling more than one transaction at a time") - - assembleConcurrency++ - defer func() { assembleConcurrency-- }() - - // chose a number of dependencies at random 0, 1, 2, 3 - // for each dependency, chose a different unclaimed pending state to spend - tx := args.Get(2).(*components.PrivateTransaction) - - var inputStates []*components.FullState - numDependencies := min(r.Intn(4), len(unclaimedPendingStatesToMintingTransaction)) - dependencies := make([]string, numDependencies) - for i := 0; i < numDependencies; i++ { - // chose a random unclaimed pending state to spend - stateIndex := r.Intn(len(unclaimedPendingStatesToMintingTransaction)) - - keys := make([]tktypes.Bytes32, len(unclaimedPendingStatesToMintingTransaction)) - keyIndex := 0 - for keyName := range unclaimedPendingStatesToMintingTransaction { - - keys[keyIndex] = keyName - keyIndex++ - } - stateID := keys[stateIndex] - inputStates = append(inputStates, &components.FullState{ - ID: stateID[:], - }) - - log.L(ctx).Infof("input state %s, numDependencies %d i %d", stateID, numDependencies, i) - dependencies[i] = unclaimedPendingStatesToMintingTransaction[stateID] - delete(unclaimedPendingStatesToMintingTransaction, stateID) - } - dependenciesByTransactionID[tx.ID.String()] = dependencies - - numOutputStates := r.Intn(4) - outputStates := make([]*components.FullState, numOutputStates) - for i := 0; i < numOutputStates; i++ { - stateID := tktypes.Bytes32(tktypes.RandBytes(32)) - outputStates[i] = &components.FullState{ - ID: stateID[:], - } - unclaimedPendingStatesToMintingTransaction[stateID] = tx.ID.String() - } +func TestCallPrivateSmartContractBadContract(t *testing.T) { - tx.PostAssembly = &components.TransactionPostAssembly{ - - AssemblyResult: prototk.AssembleTransactionResponse_OK, - OutputStates: outputStates, - InputStates: inputStates, - AttestationPlan: []*prototk.AttestationRequest{ - { - Name: "notary", - AttestationType: prototk.AttestationType_ENDORSE, - Algorithm: algorithms.ECDSA_SECP256K1, - VerifierType: verifiers.ETH_ADDRESS, - PayloadType: signpayloads.OPAQUE_TO_RSV, - Parties: []string{ - "domain1.contract1.notary@othernode", - }, - }, - }, - } - }).Return(nil) - - mocks.transportManager.On("Send", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { - go func() { - //inject random latency on the network - time.Sleep(test.latency()) - transportMessage := args.Get(1).(*components.TransportMessage) - remoteEngine.ReceiveTransportMessage(ctx, transportMessage) - }() - }).Return(nil).Maybe() - - remoteEngineMocks.transportManager.On("Send", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { - go func() { - //inject random latency on the network - time.Sleep(test.latency()) - transportMessage := args.Get(1).(*components.TransportMessage) - privateTxManager.ReceiveTransportMessage(ctx, transportMessage) - }() - }).Return(nil).Maybe() - remoteEngineMocks.domainMgr.On("GetSmartContractByAddress", mock.Anything, *domainAddress).Return(remoteEngineMocks.domainSmartContract, nil) - - remoteEngineMocks.keyManager.On("ResolveKey", mock.Anything, "domain1.contract1.notary@othernode", algorithms.ECDSA_SECP256K1, verifiers.ETH_ADDRESS).Return("domain1.contract1.notary", "notaryVerifier", nil) - - signingAddress := tktypes.RandHex(32) - - mocks.domainSmartContract.On("ResolveDispatch", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { - tx := args.Get(1).(*components.PrivateTransaction) - tx.Signer = signingAddress - }).Return(nil) - - //TODO match endorsement request and verifier args - remoteEngineMocks.domainSmartContract.On("EndorseTransaction", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&components.EndorsementResult{ - Result: prototk.EndorseTransactionResponse_SIGN, - Payload: []byte("some-endorsement-bytes"), - Endorser: &prototk.ResolvedVerifier{ - Lookup: "domain1.contract1.notary", - Verifier: "notaryVerifier", - Algorithm: algorithms.ECDSA_SECP256K1, - VerifierType: verifiers.ETH_ADDRESS, - }, - }, nil) - remoteEngineMocks.keyManager.On("Sign", mock.Anything, &signerapi.SignRequest{ - KeyHandle: "domain1.contract1.notary", - Algorithm: algorithms.ECDSA_SECP256K1, - PayloadType: signpayloads.OPAQUE_TO_RSV, - Payload: []byte("some-endorsement-bytes"), - }).Return(&signerapi.SignResponse{ - Payload: []byte("some-signature-bytes"), - }, nil) - - mocks.domainSmartContract.On("PrepareTransaction", mock.Anything, mock.Anything, mock.Anything).Return(nil) - - expectedNonce := uint64(0) - - numDispatched := 0 - allDispatched := make(chan bool, 1) - nonceWriterLock := sync.Mutex{} - privateTxManager.Subscribe(ctx, func(event components.PrivateTxEvent) { - nonceWriterLock.Lock() - defer nonceWriterLock.Unlock() - numDispatched++ - switch event := event.(type) { - case *components.TransactionDispatchedEvent: - assert.Equal(t, expectedNonce, event.Nonce) - expectedNonce++ - nonceByTransactionID[event.TransactionID] = event.Nonce - } - if numDispatched == test.numTransactions { - allDispatched <- true - } - }) + ctx := context.Background() + ptx, m := NewPrivateTransactionMgrForPackageTesting(t, "node1") - err := privateTxManager.Start() - require.NoError(t, err) + m.domainMgr.On("GetSmartContractByAddress", mock.Anything, mock.Anything, mock.Anything).Return(nil, fmt.Errorf("not found")) - for i := 0; i < test.numTransactions; i++ { - tx := &components.PrivateTransaction{ - ID: uuid.New(), - Inputs: &components.TransactionInputs{ - Domain: "domain1", - To: *domainAddress, - From: "Alice", - }, - } - err = privateTxManager.handleNewTx(ctx, tx) - require.NoError(t, err) - } + _, err := ptx.CallPrivateSmartContract(ctx, &components.TransactionInputs{ + To: *tktypes.RandAddress(), + Inputs: tktypes.RawJSON(`{}`), + }) + assert.Regexp(t, "not found", err) - haveAllDispatched := false - out: - for { - select { - case <-time.After(timeTillDeadline(t)): - log.L(ctx).Errorf("Timed out waiting for all transactions to be dispatched") - assert.Fail(t, "Timed out waiting for all transactions to be dispatched") - break out - case <-allDispatched: - haveAllDispatched = true - break out - case reason := <-failEarly: - require.Fail(t, reason) - } - } +} +func TestCallPrivateSmartContractBadDomainName(t *testing.T) { + + ctx := context.Background() + ptx, m := NewPrivateTransactionMgrForPackageTesting(t, "node1") + + _, mPSC := mockDomainSmartContractAndCtx(t, m) + + fnDef := &abi.Entry{Name: "getIt", Type: abi.Function, Outputs: abi.ParameterArray{ + {Name: "it", Type: "string"}, + }} + + _, err := ptx.CallPrivateSmartContract(ctx, &components.TransactionInputs{ + Domain: "does-not-match", + To: mPSC.Address(), + Inputs: tktypes.RawJSON(`{}`), + Function: fnDef, + }) + assert.Regexp(t, "PD011825", err) - if haveAllDispatched { - //check that they were dispatched a valid order ( i.e. no transaction was dispatched before its dependencies) - for txId, nonce := range nonceByTransactionID { - dependencies := dependenciesByTransactionID[txId] - for _, depTxID := range dependencies { - depNonce, ok := nonceByTransactionID[depTxID] - assert.True(t, ok) - assert.True(t, depNonce < nonce, "Transaction %s (nonce %d) was dispatched before its dependency %s (nonce %d)", txId, nonce, depTxID, depNonce) - } - } - } - }) - } } -func pollForStatus(ctx context.Context, t *testing.T, expectedStatus string, privateTxManager components.PrivateTxManager, domainAddressString, txID string, duration time.Duration) string { - timeout := time.After(duration) - tick := time.Tick(100 * time.Millisecond) +func TestCallPrivateSmartContractInitCallFail(t *testing.T) { + + ctx := context.Background() + ptx, m := NewPrivateTransactionMgrForPackageTesting(t, "node1") + + _, mPSC := mockDomainSmartContractAndCtx(t, m) + + mPSC.On("InitCall", mock.Anything, mock.Anything).Return( + nil, fmt.Errorf("pop"), + ) + + _, err := ptx.CallPrivateSmartContract(ctx, &components.TransactionInputs{ + To: mPSC.Address(), + Inputs: tktypes.RawJSON(`{}`), + }) + require.Regexp(t, "pop", err) + +} + +func TestCallPrivateSmartContractResolveFail(t *testing.T) { + + ctx := context.Background() + ptx, m := NewPrivateTransactionMgrForPackageTesting(t, "node1") + + _, mPSC := mockDomainSmartContractAndCtx(t, m) + + mPSC.On("InitCall", mock.Anything, mock.Anything).Return( + []*prototk.ResolveVerifierRequest{ + {Lookup: "bob@node1", Algorithm: algorithms.ECDSA_SECP256K1, VerifierType: verifiers.ETH_ADDRESS}, + }, nil, + ) + m.identityResolver.On("ResolveVerifier", mock.Anything, "bob@node1", algorithms.ECDSA_SECP256K1, verifiers.ETH_ADDRESS). + Return("", fmt.Errorf("pop")) + + _, err := ptx.CallPrivateSmartContract(ctx, &components.TransactionInputs{ + To: mPSC.Address(), + Inputs: tktypes.RawJSON(`{}`), + }) + require.Regexp(t, "pop", err) + +} + +func TestCallPrivateSmartContractExecCallFail(t *testing.T) { + + ctx := context.Background() + ptx, m := NewPrivateTransactionMgrForPackageTesting(t, "node1") + + _, mPSC := mockDomainSmartContractAndCtx(t, m) + + mPSC.On("InitCall", mock.Anything, mock.Anything).Return( + []*prototk.ResolveVerifierRequest{}, nil, + ) + mPSC.On("ExecCall", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return( + nil, fmt.Errorf("pop"), + ) + + _, err := ptx.CallPrivateSmartContract(ctx, &components.TransactionInputs{ + To: mPSC.Address(), + Inputs: tktypes.RawJSON(`{}`), + }) + require.Regexp(t, "pop", err) - for { - if t.Failed() { - panic("test failed") - } - select { - case <-timeout: - // Timeout reached, exit the loop - assert.Failf(t, "Timed out waiting for status %s", expectedStatus) - s, err := privateTxManager.GetTxStatus(ctx, domainAddressString, txID) - require.NoError(t, err) - return s.Status - case <-tick: - s, err := privateTxManager.GetTxStatus(ctx, domainAddressString, txID) - if s.Status == expectedStatus { - return s.Status - } - require.NoError(t, err) - } - } } +/* Unit tests */ + +/* Utils */ + type dependencyMocks struct { + preInitComponents *componentmocks.PreInitComponents allComponents *componentmocks.AllComponents db *mockpersistence.SQLMockProvider domain *componentmocks.Domain @@ -1664,164 +2288,194 @@ type dependencyMocks struct { stateStore *componentmocks.StateManager keyManager *componentmocks.KeyManager keyResolver *componentmocks.KeyResolver - publicTxManager components.PublicTxManager /* could be fake or mock */ + publicTxManager *componentmocks.PublicTxManager identityResolver *componentmocks.IdentityResolver txManager *componentmocks.TXManager } -// For Black box testing we return components.PrivateTxManager -func NewPrivateTransactionMgrForTesting(t *testing.T, nodeName string) (*privateTxManager, *dependencyMocks) { - // by default create a mock publicTxManager if no fake was provided - fakePublicTxManager := componentmocks.NewPublicTxManager(t) - return NewPrivateTransactionMgrForTestingWithFakePublicTxManager(t, fakePublicTxManager, nodeName) -} - -type fakePublicTxManager struct { - t *testing.T - rejectErr error - prepareErr error -} - -// GetPublicTransactionForHash implements components.PublicTxManager. -func (f *fakePublicTxManager) GetPublicTransactionForHash(ctx context.Context, dbTX *gorm.DB, hash tktypes.Bytes32) (*pldapi.PublicTxWithBinding, error) { - panic("unimplemented") +func (m *dependencyMocks) mockDomain(domainAddress *tktypes.EthAddress) { + m.stateStore.On("NewDomainContext", mock.Anything, m.domain, *domainAddress, mock.Anything).Return(m.domainContext).Maybe() + m.domainMgr.On("GetSmartContractByAddress", mock.Anything, mock.Anything, *domainAddress).Maybe().Return(m.domainSmartContract, nil) + m.domain.On("Configuration").Return(&prototk.DomainConfig{}).Maybe() } -// QueryPublicTxForTransactions implements components.PublicTxManager. -func (f *fakePublicTxManager) QueryPublicTxForTransactions(ctx context.Context, dbTX *gorm.DB, boundToTxns []uuid.UUID, jq *query.QueryJSON) (map[uuid.UUID][]*pldapi.PublicTx, error) { - panic("unimplemented") -} +// Some of the tests were getting quite verbose and was difficult to see the wood for the trees so moved a lot of the boilerplate into these utility functions +// - some of these utility functions have potential to become complex so may need to think about tests for the utility functions themselves but for now, the complexity is low enough to not warrant it -// QueryPublicTxWithBindings implements components.PublicTxManager. -func (f *fakePublicTxManager) QueryPublicTxWithBindings(ctx context.Context, dbTX *gorm.DB, jq *query.QueryJSON) ([]*pldapi.PublicTxWithBinding, error) { - panic("unimplemented") -} +func (m *dependencyMocks) mockForSubmitter(t *testing.T, transactionID *uuid.UUID, domainAddress *tktypes.EthAddress, expectedEndorsements map[string][]byte /*map of verifier to endorsement signature*/, dcFlushed chan error) { + signerName := "signer1" + signingAddr := tktypes.RandAddress() + m.domainSmartContract.On("PrepareTransaction", mock.Anything, mock.Anything, mock.MatchedBy(privateTransactionMatcher(*transactionID))).Return(nil).Run( + func(args mock.Arguments) { + cv, err := testABI[0].Inputs.ParseExternalData(map[string]any{ + "inputs": []any{tktypes.Bytes32(tktypes.RandBytes(32))}, + "outputs": []any{tktypes.Bytes32(tktypes.RandBytes(32))}, + "data": "0xfeedbeef", + }) + require.NoError(t, err) + tx := args[2].(*components.PrivateTransaction) + tx.Signer = signerName + jsonData, _ := cv.JSON() + tx.PreparedPublicTransaction = &pldapi.TransactionInput{ + ABI: abi.ABI{testABI[0]}, + TransactionBase: pldapi.TransactionBase{ + To: domainAddress, + Data: tktypes.RawJSON(jsonData), + PublicTxOptions: pldapi.PublicTxOptions{Gas: confutil.P(tktypes.HexUint64(100000))}, + }, + } + endorsed := make(map[string]bool) + for _, endorsement := range tx.PostAssembly.Endorsements { + if expectedEndorsement, ok := expectedEndorsements[endorsement.Verifier.Verifier]; ok { + assert.Equal(t, expectedEndorsement, endorsement.Payload) + endorsed[endorsement.Verifier.Verifier] = true -// MatchUpdateConfirmedTransactions implements components.PublicTxManager. -func (f *fakePublicTxManager) MatchUpdateConfirmedTransactions(ctx context.Context, dbTX *gorm.DB, itxs []*blockindexer.IndexedTransactionNotify) ([]*components.PublicTxMatch, error) { - panic("unimplemented") -} + } else { + assert.Failf(t, "unexpected endorsement from %s ", endorsement.Verifier.Verifier) + } + } + assert.Len(t, endorsed, len(expectedEndorsements)) + }, + ) -// NotifyConfirmPersisted implements components.PublicTxManager. -func (f *fakePublicTxManager) NotifyConfirmPersisted(ctx context.Context, confirms []*components.PublicTxMatch) { - panic("unimplemented") -} + m.keyManager.On("ResolveEthAddressBatchNewDatabaseTX", mock.Anything, []string{"signer1"}). + Return([]*tktypes.EthAddress{signingAddr}, nil) -// PostInit implements components.PublicTxManager. -func (f *fakePublicTxManager) PostInit(components.AllComponents) error { - panic("unimplemented") -} + _ = mockWritePublicTxsOk(m) -// PreInit implements components.PublicTxManager. -func (f *fakePublicTxManager) PreInit(components.PreInitComponents) (*components.ManagerInitResult, error) { - panic("unimplemented") + m.domainContext.On("Flush", mock.Anything).Return(func(err error) { + dcFlushed <- err + }, nil) } -// Start implements components.PublicTxManager. -func (f *fakePublicTxManager) Start() error { - panic("unimplemented") -} +func mockNetwork(t *testing.T, transactionManagers []privateTransactionMgrForPackageTesting) { -// Stop implements components.PublicTxManager. -func (f *fakePublicTxManager) Stop() { - panic("unimplemented") -} + routeToNode := func(args mock.Arguments) { + go func() { + transportMessage := args.Get(1).(*components.TransportMessage) + for _, tm := range transactionManagers { + if tm.NodeName() == transportMessage.Node { + tm.ReceiveTransportMessage(context.Background(), transportMessage) + return + } + } + assert.Failf(t, "no transaction manager found for node %s", transportMessage.Node) + }() + } + for _, tm := range transactionManagers { + tm.DependencyMocks().transportManager.On("Send", mock.Anything, mock.Anything).Run(routeToNode).Return(nil).Maybe() + } -type fakePublicTxBatch struct { - t *testing.T - transactions []*components.PublicTxSubmission - accepted []components.PublicTxAccepted - rejected []components.PublicTxRejected - completeCalled bool - committed bool - submitErr error } -func (f *fakePublicTxBatch) Accepted() []components.PublicTxAccepted { - return f.accepted -} +func (m *dependencyMocks) mockForEndorsement(_ *testing.T, txID uuid.UUID, endorser *identityForTesting, endorsementPayload []byte, endorsementSignature []byte) { + endorsementRequestMatcher := func(req *components.PrivateTransactionEndorseRequest) bool { + return req.TransactionSpecification.TransactionId == txID.String() + } + m.domainSmartContract.On("EndorseTransaction", mock.Anything, mock.Anything, mock.MatchedBy(endorsementRequestMatcher)).Return(&components.EndorsementResult{ + Result: prototk.EndorseTransactionResponse_SIGN, + Payload: endorsementPayload, + Endorser: &prototk.ResolvedVerifier{ + Lookup: endorser.identityLocator, + Verifier: endorser.verifier, + Algorithm: algorithms.ECDSA_SECP256K1, + VerifierType: verifiers.ETH_ADDRESS, + }, + }, nil) -func (f *fakePublicTxBatch) Completed(ctx context.Context, committed bool) { - f.completeCalled = true - f.committed = committed + endorser.mockSign(endorsementPayload, endorsementSignature) } -func (f *fakePublicTxBatch) Rejected() []components.PublicTxRejected { - return f.rejected -} +func (m *dependencyMocks) mockForEndorsementOnTrigger(_ *testing.T, txID uuid.UUID, endorser *identityForTesting, endorsementPayload []byte, endorsementSignature []byte) func() { + triggered := false + endorsementRequestMatcher := func(req *components.PrivateTransactionEndorseRequest) bool { + return req.TransactionSpecification.TransactionId == txID.String() + } + trigger := make(chan struct{}) + m.domainSmartContract.On("EndorseTransaction", mock.Anything, mock.Anything, mock.MatchedBy(endorsementRequestMatcher)).Run(func(_ mock.Arguments) { + if !triggered { + <-trigger + } + }).Return(&components.EndorsementResult{ + Result: prototk.EndorseTransactionResponse_SIGN, + Payload: endorsementPayload, + Endorser: &prototk.ResolvedVerifier{ + Lookup: endorser.identityLocator, + Verifier: endorser.verifier, + Algorithm: algorithms.ECDSA_SECP256K1, + VerifierType: verifiers.ETH_ADDRESS, + }, + }, nil) -type fakePublicTx struct { - t *components.PublicTxSubmission - rejectErr error - pubTx *pldapi.PublicTx -} + endorser.mockSign(endorsementPayload, endorsementSignature) -func newFakePublicTx(t *components.PublicTxSubmission, rejectErr error) *fakePublicTx { - return &fakePublicTx{ - t: t, - rejectErr: rejectErr, - pubTx: &pldapi.PublicTx{ - To: t.To, - Data: t.Data, - From: *t.From, - Created: tktypes.TimestampNow(), - PublicTxOptions: t.PublicTxOptions, - }, + return func() { + //remember that the trigger has been called so that any future calls to the mock will not block + triggered = true + close(trigger) } } -func (f *fakePublicTx) RejectedError() error { - return f.rejectErr +type privateTransactionMgrForPackageTesting interface { + components.PrivateTxManager + PreCommitHandler(ctx context.Context, dbTX *gorm.DB, blocks []*pldapi.IndexedBlock, transactions []*blockindexer.IndexedTransactionNotify) (blockindexer.PostCommit, error) + DependencyMocks() *dependencyMocks + NodeName() string + //Wrapper around a call to PreCommitHandler to notify of a new block with given height + SetBlockHeight(ctx context.Context, height int64) + DB() *gorm.DB +} +type privateTransactionMgrForPackageTestingStruct struct { + *privateTxManager + preCommitHandler blockindexer.PreCommitHandler + dependencyMocks *dependencyMocks + nodeName string + t *testing.T } -func (f *fakePublicTx) RevertData() tktypes.HexBytes { - return []byte("some data") +func (p *privateTransactionMgrForPackageTestingStruct) PreCommitHandler(ctx context.Context, dbTX *gorm.DB, blocks []*pldapi.IndexedBlock, transactions []*blockindexer.IndexedTransactionNotify) (blockindexer.PostCommit, error) { + return p.preCommitHandler(ctx, dbTX, blocks, transactions) } -func (f *fakePublicTx) Bindings() []*components.PaladinTXReference { - return f.t.Bindings +func (p *privateTransactionMgrForPackageTestingStruct) DependencyMocks() *dependencyMocks { + return p.dependencyMocks } -func (f *fakePublicTx) PublicTx() *pldapi.PublicTx { - return f.pubTx +func (p *privateTransactionMgrForPackageTestingStruct) NodeName() string { + return p.nodeName } -//for this test, we need a hand written fake rather than a simple mock for publicTxManager +func (p *privateTransactionMgrForPackageTestingStruct) SetBlockHeight(ctx context.Context, height int64) { + postCommitHandler, err := p.PreCommitHandler( + ctx, + nil, + []*pldapi.IndexedBlock{ + { + Number: height, + }, + }, + nil, + ) + assert.NoError(p.t, err) + postCommitHandler() -// PrepareSubmissionBatch implements components.PublicTxManager. -func (f *fakePublicTxManager) PrepareSubmissionBatch(ctx context.Context, transactions []*components.PublicTxSubmission) (batch components.PublicTxBatch, err error) { - b := &fakePublicTxBatch{t: f.t, transactions: transactions} - if f.rejectErr != nil { - for _, t := range transactions { - b.rejected = append(b.rejected, newFakePublicTx(t, f.rejectErr)) - } - } else { - for _, t := range transactions { - b.accepted = append(b.accepted, newFakePublicTx(t, nil)) - } - } - return b, f.prepareErr } -// SubmitBatch implements components.PublicTxManager. -func (f *fakePublicTxBatch) Submit(ctx context.Context, dbTX *gorm.DB) error { - nonceBase := 1000 - for i, tx := range f.accepted { - tx.(*fakePublicTx).pubTx.Nonce = tktypes.HexUint64(nonceBase + i) - } - return f.submitErr +func (p *privateTransactionMgrForPackageTestingStruct) DB() *gorm.DB { + return p.privateTxManager.components.Persistence().DB() } -func newFakePublicTxManager(t *testing.T) *fakePublicTxManager { - return &fakePublicTxManager{ - t: t, - } -} +func NewPrivateTransactionMgrForPackageTesting(t *testing.T, nodeName string) (privateTransactionMgrForPackageTesting, *dependencyMocks) { -func NewPrivateTransactionMgrForTestingWithFakePublicTxManager(t *testing.T, publicTxMgr components.PublicTxManager, nodeName string) (*privateTxManager, *dependencyMocks) { + defaultCoordinatorSelectionMode := EndorsementCoordinatorSelectionMode + EndorsementCoordinatorSelectionMode = BlockHeightRoundRobin // unit tests all coded to this mode (work to do as production mode for leader election becomes established) + t.Cleanup(func() { + EndorsementCoordinatorSelectionMode = defaultCoordinatorSelectionMode + }) ctx := context.Background() mocks := &dependencyMocks{ + preInitComponents: componentmocks.NewPreInitComponents(t), allComponents: componentmocks.NewAllComponents(t), domain: componentmocks.NewDomain(t), domainSmartContract: componentmocks.NewDomainSmartContract(t), @@ -1831,9 +2485,9 @@ func NewPrivateTransactionMgrForTestingWithFakePublicTxManager(t *testing.T, pub stateStore: componentmocks.NewStateManager(t), keyManager: componentmocks.NewKeyManager(t), keyResolver: componentmocks.NewKeyResolver(t), - publicTxManager: publicTxMgr, identityResolver: componentmocks.NewIdentityResolver(t), txManager: componentmocks.NewTXManager(t), + publicTxManager: componentmocks.NewPublicTxManager(t), } mocks.allComponents.On("StateManager").Return(mocks.stateStore).Maybe() mocks.allComponents.On("DomainManager").Return(mocks.domainMgr).Maybe() @@ -1841,9 +2495,10 @@ func NewPrivateTransactionMgrForTestingWithFakePublicTxManager(t *testing.T, pub mocks.transportManager.On("LocalNodeName").Return(nodeName) mocks.allComponents.On("KeyManager").Return(mocks.keyManager).Maybe() mocks.allComponents.On("TxManager").Return(mocks.txManager).Maybe() - mocks.allComponents.On("PublicTxManager").Return(publicTxMgr).Maybe() + mocks.allComponents.On("PublicTxManager").Return(mocks.publicTxManager).Maybe() mocks.allComponents.On("Persistence").Return(persistence.NewUnitTestPersistence(ctx, "privatetxmgr")).Maybe() mocks.domainSmartContract.On("Domain").Return(mocks.domain).Maybe() + mocks.domainSmartContract.On("LockStates", mock.Anything, mock.Anything, mock.Anything).Return(nil).Maybe() mocks.domainMgr.On("GetDomainByName", mock.Anything, "domain1").Return(mocks.domain, nil).Maybe() mocks.domain.On("Name").Return("domain1").Maybe() mkrc := componentmocks.NewKeyResolutionContextLazyDB(t) @@ -1854,6 +2509,8 @@ func NewPrivateTransactionMgrForTestingWithFakePublicTxManager(t *testing.T, pub mocks.domainContext.On("Ctx").Return(ctx).Maybe() mocks.domainContext.On("Info").Return(components.DomainContextInfo{ID: uuid.New()}).Maybe() + mocks.domainContext.On("ExportStateLocks").Return([]byte("[]"), nil).Maybe() + mocks.domainContext.On("ImportStateLocks", mock.Anything).Return(nil).Maybe() e := NewPrivateTransactionMgr(ctx, &pldconf.PrivateTxManagerConfig{ Writer: pldconf.FlushWriterConfig{ @@ -1869,16 +2526,135 @@ func NewPrivateTransactionMgrForTestingWithFakePublicTxManager(t *testing.T, pub mocks.transportManager.On("RegisterClient", mock.Anything, mock.Anything).Return(nil).Maybe() //It is not valid to reference LateBound components before PostInit mocks.allComponents.On("IdentityResolver").Return(mocks.identityResolver).Maybe() - err := e.PostInit(mocks.allComponents) + preInitResult, err := e.PreInit(mocks.preInitComponents) assert.NoError(t, err) - return e.(*privateTxManager), mocks + postCommitHandler, err := preInitResult.PreCommitHandler( + ctx, + nil, + []*pldapi.IndexedBlock{ + { + Number: 42, + }, + }, + nil, + ) + assert.NoError(t, err) + postCommitHandler() + + err = e.PostInit(mocks.allComponents) + assert.NoError(t, err) + + return &privateTransactionMgrForPackageTestingStruct{ + privateTxManager: e.(*privateTxManager), + preCommitHandler: preInitResult.PreCommitHandler, + dependencyMocks: mocks, + nodeName: nodeName, + t: t, + }, mocks } -func (m *dependencyMocks) mockDomain(domainAddress *tktypes.EthAddress) { - m.stateStore.On("NewDomainContext", mock.Anything, m.domain, *domainAddress, mock.Anything).Return(m.domainContext).Maybe() - m.domainMgr.On("GetSmartContractByAddress", mock.Anything, *domainAddress).Maybe().Return(m.domainSmartContract, nil) - m.domain.On("Configuration").Return(&prototk.DomainConfig{}).Maybe() +type identityForTesting struct { + identity string + identityLocator string + verifier string + keyHandle string + mocks *dependencyMocks + mockSign func(payload []byte, signature []byte) +} + +func (i *identityForTesting) mockResolve(ctx context.Context, other identityForTesting) { + // in addition to the default mocks set up in newPartyForTesting, we can set up mocks to resolve remote identities + // we could have used a real IdentityResolver here but we are testing the private transaction manager in isolation and so we mock the IdentityResolver as we do with all other tests in this file + i.mocks.identityResolver.On( + "ResolveVerifierAsync", + mock.Anything, + other.identityLocator, + algorithms.ECDSA_SECP256K1, + verifiers.ETH_ADDRESS, + mock.Anything, + mock.Anything). + Run(func(args mock.Arguments) { + resolveFn := args.Get(4).(func(context.Context, string)) + resolveFn(ctx, other.verifier) + }).Return(nil).Maybe() +} + +var testABI = abi.ABI{ + { + Name: "execute", + Type: abi.Function, + Inputs: abi.ParameterArray{ + { + Name: "inputs", + Type: "bytes32[]", + }, + { + Name: "outputs", + Type: "bytes32[]", + }, + { + Name: "data", + Type: "bytes", + }, + }, + }, +} + +func privateDeployTransactionMatcher(txID uuid.UUID) func(*components.PrivateContractDeploy) bool { + return func(tx *components.PrivateContractDeploy) bool { + return tx.ID == txID + } +} + +func privateTransactionMatcher(txID ...uuid.UUID) func(*components.PrivateTransaction) bool { + return func(tx *components.PrivateTransaction) bool { + for _, id := range txID { + if tx.ID == id { + return true + } + } + return false + } +} + +func newPartyForTesting(ctx context.Context, name, node string, mocks *dependencyMocks) identityForTesting { + party := identityForTesting{ + identity: name, + identityLocator: name + "@" + node, + verifier: tktypes.RandAddress().String(), + keyHandle: name + "KeyHandle", + mocks: mocks, + } + + mocks.identityResolver.On("ResolveVerifierAsync", mock.Anything, party.identity, algorithms.ECDSA_SECP256K1, verifiers.ETH_ADDRESS, mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + resolveFn := args.Get(4).(func(context.Context, string)) + resolveFn(ctx, party.verifier) + }).Return(nil).Maybe() + + mocks.identityResolver.On("ResolveVerifierAsync", mock.Anything, party.identityLocator, algorithms.ECDSA_SECP256K1, verifiers.ETH_ADDRESS, mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + resolveFn := args.Get(4).(func(context.Context, string)) + resolveFn(ctx, party.verifier) + }).Return(nil).Maybe() + + mocks.identityResolver.On("ResolveVerifier", mock.Anything, party.identityLocator, algorithms.ECDSA_SECP256K1, verifiers.ETH_ADDRESS). + Return(party.verifier, nil).Maybe() + + keyMapping := &pldapi.KeyMappingAndVerifier{ + KeyMappingWithPath: &pldapi.KeyMappingWithPath{KeyMapping: &pldapi.KeyMapping{ + Identifier: party.identity, + KeyHandle: party.keyHandle, + }}, + Verifier: &pldapi.KeyVerifier{Verifier: party.verifier}, + } + mocks.keyManager.On("ResolveKeyNewDatabaseTX", mock.Anything, party.identity, algorithms.ECDSA_SECP256K1, verifiers.ETH_ADDRESS).Return(keyMapping, nil).Maybe() + + party.mockSign = func(payload []byte, signature []byte) { + mocks.keyManager.On("Sign", mock.Anything, keyMapping, signpayloads.OPAQUE_TO_RSV, payload). + Return(signature, nil) + } + + return party } func timeTillDeadline(t *testing.T) time.Duration { @@ -1906,7 +2682,7 @@ func mockDomainSmartContractAndCtx(t *testing.T, m *dependencyMocks) (*component mPSC.On("Address").Return(contractAddr).Maybe() mPSC.On("Domain").Return(mDomain).Maybe() - m.domainMgr.On("GetSmartContractByAddress", mock.Anything, contractAddr).Return(mPSC, nil) + m.domainMgr.On("GetSmartContractByAddress", mock.Anything, mock.Anything, contractAddr).Return(mPSC, nil) mDC := componentmocks.NewDomainContext(t) m.stateStore.On("NewDomainContext", mock.Anything, mDomain, contractAddr).Return(mDC).Maybe() @@ -1915,141 +2691,27 @@ func mockDomainSmartContractAndCtx(t *testing.T, m *dependencyMocks) (*component return mDomain, mPSC } -func TestCallPrivateSmartContractOk(t *testing.T) { - - ctx := context.Background() - ptx, m := NewPrivateTransactionMgrForTesting(t, "node1") - - _, mPSC := mockDomainSmartContractAndCtx(t, m) - - fnDef := &abi.Entry{Name: "getIt", Type: abi.Function, Outputs: abi.ParameterArray{ - {Name: "it", Type: "string"}, - }} - resultCV, err := fnDef.Outputs.ParseJSON([]byte(`["thing"]`)) - require.NoError(t, err) - - bobAddr := tktypes.RandAddress() - m.identityResolver.On("ResolveVerifier", mock.Anything, "bob@node1", algorithms.ECDSA_SECP256K1, verifiers.ETH_ADDRESS). - Return(bobAddr.String(), nil) - mPSC.On("InitCall", mock.Anything, mock.Anything).Return( - []*prototk.ResolveVerifierRequest{ - {Lookup: "bob@node1", Algorithm: algorithms.ECDSA_SECP256K1, VerifierType: verifiers.ETH_ADDRESS}, - }, nil, - ) - mPSC.On("ExecCall", mock.Anything, mock.Anything, mock.Anything, mock.MatchedBy(func(verifiers []*prototk.ResolvedVerifier) bool { - require.Equal(t, bobAddr.String(), verifiers[0].Verifier) - return true - })).Return( - resultCV, nil, - ) - - res, err := ptx.CallPrivateSmartContract(ctx, &components.TransactionInputs{ - To: mPSC.Address(), - Inputs: tktypes.RawJSON(`{}`), - Function: fnDef, - }) - require.NoError(t, err) - jsonData, err := res.JSON() - require.NoError(t, err) - require.JSONEq(t, `{"it": "thing"}`, string(jsonData)) - -} - -func TestCallPrivateSmartContractBadContract(t *testing.T) { - - ctx := context.Background() - ptx, m := NewPrivateTransactionMgrForTesting(t, "node1") - - m.domainMgr.On("GetSmartContractByAddress", mock.Anything, mock.Anything).Return(nil, fmt.Errorf("not found")) - - _, err := ptx.CallPrivateSmartContract(ctx, &components.TransactionInputs{ - To: *tktypes.RandAddress(), - Inputs: tktypes.RawJSON(`{}`), - }) - assert.Regexp(t, "not found", err) - -} -func TestCallPrivateSmartContractBadDomainName(t *testing.T) { - - ctx := context.Background() - ptx, m := NewPrivateTransactionMgrForTesting(t, "node1") - - _, mPSC := mockDomainSmartContractAndCtx(t, m) - - fnDef := &abi.Entry{Name: "getIt", Type: abi.Function, Outputs: abi.ParameterArray{ - {Name: "it", Type: "string"}, - }} - - _, err := ptx.CallPrivateSmartContract(ctx, &components.TransactionInputs{ - Domain: "does-not-match", - To: mPSC.Address(), - Inputs: tktypes.RawJSON(`{}`), - Function: fnDef, - }) - assert.Regexp(t, "PD011825", err) - -} - -func TestCallPrivateSmartContractInitCallFail(t *testing.T) { - - ctx := context.Background() - ptx, m := NewPrivateTransactionMgrForTesting(t, "node1") - - _, mPSC := mockDomainSmartContractAndCtx(t, m) - - mPSC.On("InitCall", mock.Anything, mock.Anything).Return( - nil, fmt.Errorf("pop"), - ) - - _, err := ptx.CallPrivateSmartContract(ctx, &components.TransactionInputs{ - To: mPSC.Address(), - Inputs: tktypes.RawJSON(`{}`), - }) - require.Regexp(t, "pop", err) - -} - -func TestCallPrivateSmartContractResolveFail(t *testing.T) { - - ctx := context.Background() - ptx, m := NewPrivateTransactionMgrForTesting(t, "node1") - - _, mPSC := mockDomainSmartContractAndCtx(t, m) - - mPSC.On("InitCall", mock.Anything, mock.Anything).Return( - []*prototk.ResolveVerifierRequest{ - {Lookup: "bob@node1", Algorithm: algorithms.ECDSA_SECP256K1, VerifierType: verifiers.ETH_ADDRESS}, - }, nil, - ) - m.identityResolver.On("ResolveVerifier", mock.Anything, "bob@node1", algorithms.ECDSA_SECP256K1, verifiers.ETH_ADDRESS). - Return("", fmt.Errorf("pop")) - - _, err := ptx.CallPrivateSmartContract(ctx, &components.TransactionInputs{ - To: mPSC.Address(), - Inputs: tktypes.RawJSON(`{}`), - }) - require.Regexp(t, "pop", err) - -} - -func TestCallPrivateSmartContractExecCallFail(t *testing.T) { - - ctx := context.Background() - ptx, m := NewPrivateTransactionMgrForTesting(t, "node1") - - _, mPSC := mockDomainSmartContractAndCtx(t, m) - - mPSC.On("InitCall", mock.Anything, mock.Anything).Return( - []*prototk.ResolveVerifierRequest{}, nil, - ) - mPSC.On("ExecCall", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return( - nil, fmt.Errorf("pop"), - ) - - _, err := ptx.CallPrivateSmartContract(ctx, &components.TransactionInputs{ - To: mPSC.Address(), - Inputs: tktypes.RawJSON(`{}`), - }) - require.Regexp(t, "pop", err) +func pollForStatus(ctx context.Context, t *testing.T, expectedStatus string, privateTxManager components.PrivateTxManager, domainAddressString, txID string, duration time.Duration) string { + timeout := time.After(duration) + tick := time.Tick(100 * time.Millisecond) + for { + if t.Failed() { + panic("test failed") + } + select { + case <-timeout: + // Timeout reached, exit the loop + assert.Failf(t, "Timed out waiting for status %s", expectedStatus) + s, err := privateTxManager.GetTxStatus(ctx, domainAddressString, uuid.MustParse(txID)) + require.NoError(t, err) + return s.Status + case <-tick: + s, err := privateTxManager.GetTxStatus(ctx, domainAddressString, uuid.MustParse(txID)) + if s.Status == expectedStatus { + return s.Status + } + require.NoError(t, err) + } + } } diff --git a/core/go/internal/privatetxnmgr/ptmgrtypes/.gitignore b/core/go/internal/privatetxnmgr/ptmgrtypes/.gitignore new file mode 100644 index 000000000..32fd036b5 --- /dev/null +++ b/core/go/internal/privatetxnmgr/ptmgrtypes/.gitignore @@ -0,0 +1 @@ +mock_* \ No newline at end of file diff --git a/core/go/internal/privatetxnmgr/ptmgrtypes/event_types.go b/core/go/internal/privatetxnmgr/ptmgrtypes/event_types.go index 544da5fea..6f21dd799 100644 --- a/core/go/internal/privatetxnmgr/ptmgrtypes/event_types.go +++ b/core/go/internal/privatetxnmgr/ptmgrtypes/event_types.go @@ -19,6 +19,7 @@ import ( "context" "github.com/hyperledger/firefly-common/pkg/i18n" + "github.com/kaleido-io/paladin/core/internal/components" "github.com/kaleido-io/paladin/core/internal/msgs" "github.com/kaleido-io/paladin/toolkit/pkg/log" "github.com/kaleido-io/paladin/toolkit/pkg/prototk" @@ -63,13 +64,21 @@ type TransactionSwappedInEvent struct { PrivateTransactionEventBase } +type DelegationForInFlightEvent struct { + PrivateTransactionEventBase + BlockHeight int64 +} + type TransactionAssembledEvent struct { PrivateTransactionEventBase + PostAssembly *components.TransactionPostAssembly + AssembleRequestID string } type TransactionAssembleFailedEvent struct { PrivateTransactionEventBase - Error string + AssembleRequestID string + Error string } type TransactionSignedEvent struct { @@ -79,8 +88,11 @@ type TransactionSignedEvent struct { type TransactionEndorsedEvent struct { PrivateTransactionEventBase - RevertReason *string - Endorsement *prototk.AttestationResult + RevertReason *string + Endorsement *prototk.AttestationResult + Party string // In case Endorsement is nil, this is need to correlate with the attestation request + AttestationRequestName string // In case Endorsement is nil, this is need to correlate with the attestation request + IdempotencyKey string } type TransactionDispatchedEvent struct { @@ -89,6 +101,10 @@ type TransactionDispatchedEvent struct { SigningAddress string } +type TransactionPreparedEvent struct { + PrivateTransactionEventBase +} + type TransactionConfirmedEvent struct { PrivateTransactionEventBase } @@ -97,8 +113,9 @@ type TransactionRevertedEvent struct { PrivateTransactionEventBase } -type TransactionDelegatedEvent struct { +type TransactionDelegationAcknowledgedEvent struct { PrivateTransactionEventBase + DelegationRequestID string } type TransactionBlockedEvent struct { @@ -145,6 +162,12 @@ type TransactionFinalizedEvent struct { PrivateTransactionEventBase } +type TransactionNudgeEvent struct { + //used to trigger the sequence to re-evaluate a transaction's state and next action + //in lieu of a real event + PrivateTransactionEventBase +} + type TransactionFinalizeError struct { PrivateTransactionEventBase RevertReason string // reason we were trying to finalize the transaction diff --git a/core/go/internal/privatetxnmgr/ptmgrtypes/types.go b/core/go/internal/privatetxnmgr/ptmgrtypes/types.go index 5d9fb1670..ca466cd07 100644 --- a/core/go/internal/privatetxnmgr/ptmgrtypes/types.go +++ b/core/go/internal/privatetxnmgr/ptmgrtypes/types.go @@ -44,21 +44,33 @@ type Transaction struct { type Publisher interface { //Service for sending messages and events within the local node - PublishTransactionBlockedEvent(ctx context.Context, transactionId string) PublishTransactionDispatchedEvent(ctx context.Context, transactionId string, nonce uint64, signingAddress string) - PublishTransactionAssembledEvent(ctx context.Context, transactionId string) - PublishTransactionAssembleFailedEvent(ctx context.Context, transactionId string, errorMessage string) + PublishTransactionPreparedEvent(ctx context.Context, transactionId string) + PublishTransactionAssembledEvent(ctx context.Context, transactionId string, postAssembly *components.TransactionPostAssembly, requestID string) + PublishTransactionAssembleFailedEvent(ctx context.Context, transactionId string, errorMessage string, requestID string) PublishTransactionSignedEvent(ctx context.Context, transactionId string, attestationResult *prototk.AttestationResult) - PublishTransactionEndorsedEvent(ctx context.Context, transactionId string, attestationResult *prototk.AttestationResult, revertReason *string) + PublishTransactionEndorsedEvent(ctx context.Context, transactionId string, idempotencyKey string, party string, attestationRequestName string, attestationResult *prototk.AttestationResult, revertReason *string) PublishResolveVerifierResponseEvent(ctx context.Context, transactionId string, lookup, algorithm, verifier, verifierType string) PublishResolveVerifierErrorEvent(ctx context.Context, transactionId string, lookup, algorithm, errorMessage string) PublishTransactionFinalizedEvent(ctx context.Context, transactionId string) PublishTransactionFinalizeError(ctx context.Context, transactionId string, revertReason string, err error) PublishTransactionConfirmedEvent(ctx context.Context, transactionId string) + PublishNudgeEvent(ctx context.Context, transactionId string) +} + +// Map of signing address to an ordered list of transaction flows that are ready to be dispatched by that signing address +type DispatchableTransactions map[string][]TransactionFlow + +func (dtxs *DispatchableTransactions) IDs(ctx context.Context) []string { + var ids []string + for _, txs := range *dtxs { + for _, tx := range txs { + ids = append(ids, tx.ID(ctx).String()) + } + } + return ids } -// Map of signing address to an ordered list of transaction IDs that are ready to be dispatched by that signing address -type DispatchableTransactions map[string][]string type Dispatcher interface { // Dispatcher is the component that takes responsibility for submitting the transactions in the sequence to the base ledger in the correct order DispatchTransactions(context.Context, DispatchableTransactions) error @@ -88,8 +100,10 @@ type ContentionResolver interface { } type TransportWriter interface { - SendDelegationRequest(ctx context.Context, delegationId string, delegateNodeId string, transaction *components.PrivateTransaction) error - SendEndorsementRequest(ctx context.Context, party string, targetNode string, contractAddress string, transactionID string, attRequest *prototk.AttestationRequest, transactionSpecification *prototk.TransactionSpecification, verifiers []*prototk.ResolvedVerifier, signatures []*prototk.AttestationResult, inputStates []*components.FullState, outputStates []*components.FullState, infoStates []*components.FullState) error + SendDelegationRequest(ctx context.Context, delegationId string, delegateNodeName string, transaction *components.PrivateTransaction, blockHeight int64) error + SendDelegationRequestAcknowledgment(ctx context.Context, delegatingNodeName string, delegationId string, delegateNodeName string, transactionID string) error + SendEndorsementRequest(ctx context.Context, idempotencyKey string, party string, targetNode string, contractAddress string, transactionID string, attRequest *prototk.AttestationRequest, transactionSpecification *prototk.TransactionSpecification, verifiers []*prototk.ResolvedVerifier, signatures []*prototk.AttestationResult, inputStates []*components.FullState, outputStates []*components.FullState, infoStates []*components.FullState) error + SendAssembleRequest(ctx context.Context, assemblingNode string, assembleRequestID string, txID uuid.UUID, contractAddress string, transactionInputs *components.TransactionInputs, preAssembly *components.TransactionPreAssembly, stateLocksJSON []byte, blockHeight int64) error } type TransactionFlowStatus int @@ -109,15 +123,15 @@ type TransactionFlow interface { PrepareTransaction(ctx context.Context, defaultSigner string) (*components.PrivateTransaction, error) GetStateDistributions(ctx context.Context) (*components.StateDistributionSet, error) - CoordinatingLocally() bool - IsComplete() bool - ReadyForSequencing() bool - Dispatched() bool - ID() uuid.UUID + CoordinatingLocally(ctx context.Context) bool + IsComplete(ctx context.Context) bool + ReadyForSequencing(ctx context.Context) bool + Dispatched(ctx context.Context) bool + ID(ctx context.Context) uuid.UUID IsEndorsed(ctx context.Context) bool - InputStateIDs() []string - OutputStateIDs() []string - Signer() string + InputStateIDs(ctx context.Context) []string + OutputStateIDs(ctx context.Context) []string + Signer(ctx context.Context) string } type Clock interface { @@ -133,3 +147,30 @@ func (c *realClock) Now() time.Time { func RealClock() Clock { return &realClock{} } + +type CoordinatorSelector interface { + SelectCoordinatorNode(ctx context.Context, transaction *components.PrivateTransaction, environment SequencerEnvironment) (int64, string, error) +} + +type SequencerEnvironment interface { + GetBlockHeight() int64 +} + +// AssembleCoordinator is a component that is responsible for coordinating the assembly of all transactions for a given domain contract instance +// requests to assemble transactions are queued and the queue is processed on a single thread that blocks until one assemble completes before starting the next +type AssembleCoordinator interface { + Start() + Stop() + QueueAssemble(ctx context.Context, assemblingNode string, transactionID uuid.UUID, transactionInputs *components.TransactionInputs, transactionPreAssembly *components.TransactionPreAssembly) + Complete(requestID string, stateDistributions []*components.StateDistribution) +} + +type LocalAssembler interface { + AssembleLocal( + ctx context.Context, + requestID string, + transactionID uuid.UUID, + transactionInputs *components.TransactionInputs, + preAssembly *components.TransactionPreAssembly, + ) +} diff --git a/core/go/internal/privatetxnmgr/ptmgrtypes/types_test.go b/core/go/internal/privatetxnmgr/ptmgrtypes/types_test.go new file mode 100644 index 000000000..f4e5456e7 --- /dev/null +++ b/core/go/internal/privatetxnmgr/ptmgrtypes/types_test.go @@ -0,0 +1,63 @@ +// Copyright © 2024 Kaleido, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ptmgrtypes + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + + "github.com/google/uuid" +) + +func TestDispatchableTransactionsIDs(t *testing.T) { + ctx := context.Background() + id1 := uuid.New() + id2 := uuid.New() + id3 := uuid.New() + id4 := uuid.New() + mockTransactionFlow1 := NewMockTransactionFlow(t) + mockTransactionFlow1.On("ID", mock.Anything).Return(id1) + + mockTransactionFlow2 := NewMockTransactionFlow(t) + mockTransactionFlow2.On("ID", mock.Anything).Return(id2) + mockTransactionFlow3 := NewMockTransactionFlow(t) + mockTransactionFlow3.On("ID", mock.Anything).Return(id3) + mockTransactionFlow4 := NewMockTransactionFlow(t) + mockTransactionFlow4.On("ID", mock.Anything).Return(id4) + + dispatchableTransactions := DispatchableTransactions{ + "A": { + mockTransactionFlow1, + mockTransactionFlow2, + }, + "B": { + mockTransactionFlow3, + mockTransactionFlow4, + }, + } + + ids := dispatchableTransactions.IDs(ctx) + assert.Len(t, ids, 4) + assert.Contains(t, ids, id1.String()) + assert.Contains(t, ids, id2.String()) + assert.Contains(t, ids, id3.String()) + assert.Contains(t, ids, id4.String()) + +} diff --git a/core/go/internal/privatetxnmgr/publisher.go b/core/go/internal/privatetxnmgr/publisher.go index 76ff1ede4..96b5df08f 100644 --- a/core/go/internal/privatetxnmgr/publisher.go +++ b/core/go/internal/privatetxnmgr/publisher.go @@ -35,17 +35,6 @@ type publisher struct { contractAddress string } -func (p *publisher) PublishTransactionBlockedEvent(ctx context.Context, transactionId string) { - - p.privateTxManager.HandleNewEvent(ctx, &ptmgrtypes.TransactionBlockedEvent{ - PrivateTransactionEventBase: ptmgrtypes.PrivateTransactionEventBase{ - ContractAddress: p.contractAddress, - TransactionID: transactionId, - }, - }) - -} - func (p *publisher) PublishTransactionDispatchedEvent(ctx context.Context, transactionId string, nonce uint64, signingAddress string) { p.privateTxManager.HandleNewEvent(ctx, &ptmgrtypes.TransactionDispatchedEvent{ @@ -61,26 +50,37 @@ func (p *publisher) PublishTransactionDispatchedEvent(ctx context.Context, trans Nonce: nonce, SigningAddress: signingAddress, }) +} +func (p *publisher) PublishTransactionPreparedEvent(ctx context.Context, transactionId string) { + p.privateTxManager.HandleNewEvent(ctx, &ptmgrtypes.TransactionPreparedEvent{ + PrivateTransactionEventBase: ptmgrtypes.PrivateTransactionEventBase{ + ContractAddress: p.contractAddress, + TransactionID: transactionId, + }, + }) } -func (p *publisher) PublishTransactionAssembledEvent(ctx context.Context, transactionId string) { +func (p *publisher) PublishTransactionAssembledEvent(ctx context.Context, transactionId string, postAssembly *components.TransactionPostAssembly, requestID string) { event := &ptmgrtypes.TransactionAssembledEvent{ PrivateTransactionEventBase: ptmgrtypes.PrivateTransactionEventBase{ ContractAddress: p.contractAddress, TransactionID: transactionId, }, + PostAssembly: postAssembly, + AssembleRequestID: requestID, } p.privateTxManager.HandleNewEvent(ctx, event) } -func (p *publisher) PublishTransactionAssembleFailedEvent(ctx context.Context, transactionId string, errorMessage string) { +func (p *publisher) PublishTransactionAssembleFailedEvent(ctx context.Context, transactionId string, errorMessage string, requestID string) { event := &ptmgrtypes.TransactionAssembleFailedEvent{ PrivateTransactionEventBase: ptmgrtypes.PrivateTransactionEventBase{ ContractAddress: p.contractAddress, TransactionID: transactionId, }, - Error: errorMessage, + Error: errorMessage, + AssembleRequestID: requestID, } p.privateTxManager.HandleNewEvent(ctx, event) } @@ -96,14 +96,17 @@ func (p *publisher) PublishTransactionSignedEvent(ctx context.Context, transacti p.privateTxManager.HandleNewEvent(ctx, event) } -func (p *publisher) PublishTransactionEndorsedEvent(ctx context.Context, transactionId string, endorsement *prototk.AttestationResult, revertReason *string) { +func (p *publisher) PublishTransactionEndorsedEvent(ctx context.Context, transactionId string, idempotencyKey string, party string, attestationRequestName string, endorsement *prototk.AttestationResult, revertReason *string) { event := &ptmgrtypes.TransactionEndorsedEvent{ PrivateTransactionEventBase: ptmgrtypes.PrivateTransactionEventBase{ ContractAddress: p.contractAddress, TransactionID: transactionId, }, - Endorsement: endorsement, - RevertReason: revertReason, + Endorsement: endorsement, + RevertReason: revertReason, + Party: party, + AttestationRequestName: attestationRequestName, + IdempotencyKey: idempotencyKey, } p.privateTxManager.HandleNewEvent(ctx, event) } @@ -168,3 +171,13 @@ func (p *publisher) PublishTransactionConfirmedEvent(ctx context.Context, transa } p.privateTxManager.HandleNewEvent(ctx, event) } + +func (p *publisher) PublishNudgeEvent(ctx context.Context, transactionId string) { + event := &ptmgrtypes.TransactionNudgeEvent{ + PrivateTransactionEventBase: ptmgrtypes.PrivateTransactionEventBase{ + ContractAddress: p.contractAddress, + TransactionID: transactionId, + }, + } + p.privateTxManager.HandleNewEvent(ctx, event) +} diff --git a/core/go/internal/privatetxnmgr/sequencer.go b/core/go/internal/privatetxnmgr/sequencer.go index 90c0ba56a..95d1e92c9 100644 --- a/core/go/internal/privatetxnmgr/sequencer.go +++ b/core/go/internal/privatetxnmgr/sequencer.go @@ -31,6 +31,7 @@ import ( "github.com/kaleido-io/paladin/core/internal/privatetxnmgr/ptmgrtypes" "github.com/kaleido-io/paladin/core/internal/privatetxnmgr/syncpoints" "github.com/kaleido-io/paladin/core/internal/statedistribution" + pbEngine "github.com/kaleido-io/paladin/core/pkg/proto/engine" "github.com/kaleido-io/paladin/toolkit/pkg/log" "github.com/kaleido-io/paladin/toolkit/pkg/tktypes" @@ -70,6 +71,14 @@ var AllSequencerStates = []string{ string(SequencerStateStopped), } +type sequencerEnvironment struct { + blockHeight int64 +} + +func (e *sequencerEnvironment) GetBlockHeight() int64 { + return e.blockHeight +} + type Sequencer struct { ctx context.Context privateTxManager components.PrivateTxManager @@ -98,12 +107,14 @@ type Sequencer struct { staleTimeout time.Duration - pendingEvents chan ptmgrtypes.PrivateTransactionEvent + pendingTransactionEvents chan ptmgrtypes.PrivateTransactionEvent contractAddress tktypes.EthAddress // the contract address managed by the current sequencer defaultSigner string - nodeID string + nodeName string domainAPI components.DomainSmartContract + coordinatorDomainContext components.DomainContext + delegateDomainContext components.DomainContext components components.AllComponents endorsementGatherer ptmgrtypes.EndorsementGatherer publisher ptmgrtypes.Publisher @@ -114,12 +125,16 @@ type Sequencer struct { transportWriter ptmgrtypes.TransportWriter graph Graph requestTimeout time.Duration + coordinatorSelector ptmgrtypes.CoordinatorSelector + newBlockEvents chan int64 + assembleCoordinator ptmgrtypes.AssembleCoordinator + environment *sequencerEnvironment } func NewSequencer( ctx context.Context, privateTxManager components.PrivateTxManager, - nodeID string, + nodeName string, contractAddress tktypes.EthAddress, sequencerConfig *pldconf.PrivateTxManagerSequencerConfig, allComponents components.AllComponents, @@ -132,7 +147,9 @@ func NewSequencer( preparedTransactionDistributer preparedtxdistribution.PreparedTransactionDistributer, transportWriter ptmgrtypes.TransportWriter, requestTimeout time.Duration, -) *Sequencer { + blockHeight int64, + +) (*Sequencer, error) { newSequencer := &Sequencer{ ctx: log.WithLogField(ctx, "role", fmt.Sprintf("sequencer-%s", contractAddress)), @@ -151,8 +168,8 @@ func NewSequencer( processedTxIDs: make(map[string]bool), orchestrationEvalRequestChan: make(chan bool, 1), stopProcess: make(chan bool, 1), - pendingEvents: make(chan ptmgrtypes.PrivateTransactionEvent, *pldconf.PrivateTxManagerDefaults.Sequencer.MaxPendingEvents), - nodeID: nodeID, + pendingTransactionEvents: make(chan ptmgrtypes.PrivateTransactionEvent, *pldconf.PrivateTxManagerDefaults.Sequencer.MaxPendingEvents), + nodeName: nodeName, domainAPI: domainAPI, components: allComponents, endorsementGatherer: endorsementGatherer, @@ -164,15 +181,55 @@ func NewSequencer( transportWriter: transportWriter, graph: NewGraph(), requestTimeout: requestTimeout, + environment: &sequencerEnvironment{ + blockHeight: blockHeight, + }, // Randomly allocate a signer. // TODO: rotation - defaultSigner: fmt.Sprintf("domains.%s.submit.%s", contractAddress, uuid.New()), + defaultSigner: fmt.Sprintf("domains.%s.submit.%s", contractAddress, uuid.New()), + newBlockEvents: make(chan int64, 10), //TODO do we want to make the buffer size configurable? Or should we put in non blocking mode? Does it matter if we miss a block? + } log.L(ctx).Debugf("NewSequencer for contract address %s created: %+v", newSequencer.contractAddress, newSequencer) - return newSequencer + coordinatorSelector, err := NewCoordinatorSelector(ctx, nodeName, domainAPI.ContractConfig(), *sequencerConfig) + if err != nil { + log.L(ctx).Errorf("Failed to create coordinator selector: %s", err) + return nil, i18n.WrapError(ctx, err, msgs.MsgPrivateTxManagerNewSequencerError, domainAPI.ContractConfig().GetCoordinatorSelection()) + } + newSequencer.coordinatorSelector = coordinatorSelector + + //TODO consolidate the initialization of the endorsement gatherer and the assemble coordinator. Both need the same domain context - but maybe the assemble coordinator should provide the domain context to the endorsement gatherer on a per request basis + // + domainSmartContract, err := allComponents.DomainManager().GetSmartContractByAddress(ctx, allComponents.Persistence().DB(), contractAddress) + if err != nil { + log.L(ctx).Errorf("Failed to get domain smart contract for contract address %s: %s", contractAddress, err) + + return nil, i18n.WrapError(ctx, err, msgs.MsgPrivateTxManagerNewSequencerError, contractAddress) + } + + // create 2 domain contexts. One to keep track of all transactions that we are coordinating and one for assembling transactions on behalf of a remote coordinator + newSequencer.coordinatorDomainContext = allComponents.StateManager().NewDomainContext(newSequencer.ctx /* background context */, domainSmartContract.Domain(), contractAddress) + newSequencer.delegateDomainContext = allComponents.StateManager().NewDomainContext(newSequencer.ctx /* background context */, domainSmartContract.Domain(), contractAddress) + + newSequencer.assembleCoordinator = NewAssembleCoordinator( + ctx, + nodeName, + confutil.Int(sequencerConfig.MaxInflightTransactions, *pldconf.PrivateTxManagerDefaults.Sequencer.MaxInflightTransactions)+1, // make sure the coordinator has more slots that we do so that we don't get stuck waiting for it + allComponents, + domainAPI, + newSequencer.coordinatorDomainContext, + transportWriter, + contractAddress, + newSequencer.environment, + confutil.DurationMin(sequencerConfig.AssembleRequestTimeout, 1*time.Millisecond, *pldconf.PrivateTxManagerDefaults.Sequencer.AssembleRequestTimeout), + stateDistributer, + newSequencer, + ) + + return newSequencer, nil } func (s *Sequencer) abort(err error) { @@ -185,7 +242,7 @@ func (s *Sequencer) getTransactionProcessor(txID string) ptmgrtypes.TransactionF defer s.incompleteTxProcessMapMutex.Unlock() transactionProcessor, ok := s.incompleteTxSProcessMap[txID] if !ok { - log.L(s.ctx).Errorf("Transaction processor not found for transaction ID %s", txID) + log.L(s.ctx).Debugf("Transaction processor not found for transaction ID %s", txID) return nil } return transactionProcessor @@ -197,6 +254,12 @@ func (s *Sequencer) removeTransactionProcessor(txID string) { delete(s.incompleteTxSProcessMap, txID) } +func (s *Sequencer) OnNewBlockHeight(ctx context.Context, blockHeight int64) { + log.L(ctx).Debugf("Sequencer OnNewBlockHeight %d", blockHeight) + s.environment.blockHeight = blockHeight + +} + func (s *Sequencer) ProcessNewTransaction(ctx context.Context, tx *components.PrivateTransaction) (queued bool) { s.incompleteTxProcessMapMutex.Lock() defer s.incompleteTxProcessMapMutex.Unlock() @@ -206,16 +269,16 @@ func (s *Sequencer) ProcessNewTransaction(ctx context.Context, tx *components.Pr // tx processing pool is full, queue the item return true } else { - s.incompleteTxSProcessMap[tx.ID.String()] = NewTransactionFlow(ctx, tx, s.nodeID, s.components, s.domainAPI, s.publisher, s.endorsementGatherer, s.identityResolver, s.syncPoints, s.transportWriter, s.requestTimeout) + s.incompleteTxSProcessMap[tx.ID.String()] = NewTransactionFlow(ctx, tx, s.nodeName, s.components, s.domainAPI, s.coordinatorDomainContext, s.publisher, s.endorsementGatherer, s.identityResolver, s.syncPoints, s.transportWriter, s.requestTimeout, s.coordinatorSelector, s.assembleCoordinator, s.environment) } - s.pendingEvents <- &ptmgrtypes.TransactionSubmittedEvent{ + s.pendingTransactionEvents <- &ptmgrtypes.TransactionSubmittedEvent{ PrivateTransactionEventBase: ptmgrtypes.PrivateTransactionEventBase{TransactionID: tx.ID.String()}, } } return false } -func (s *Sequencer) ProcessInFlightTransaction(ctx context.Context, tx *components.PrivateTransaction) (queued bool) { +func (s *Sequencer) ProcessInFlightTransaction(ctx context.Context, tx *components.PrivateTransaction, delegationBlockHeight *int64) (queued bool) { log.L(ctx).Infof("Processing in flight transaction %s", tx.ID) //a transaction that already has had some processing done on it // currently the only case this can happen is a transaction delegated from another node @@ -224,18 +287,26 @@ func (s *Sequencer) ProcessInFlightTransaction(ctx context.Context, tx *componen defer s.incompleteTxProcessMapMutex.Unlock() _, alreadyInMemory := s.incompleteTxSProcessMap[tx.ID.String()] if alreadyInMemory { - log.L(ctx).Warnf("Transaction %s already in memory. Ignoring", tx.ID) + if delegationBlockHeight != nil { + // We need to inform the in-flight processor that the delegation is received, because it might override a + // delegating status that we have locally (depending on the block height) + s.pendingTransactionEvents <- &ptmgrtypes.DelegationForInFlightEvent{ + PrivateTransactionEventBase: ptmgrtypes.PrivateTransactionEventBase{TransactionID: tx.ID.String()}, + BlockHeight: *delegationBlockHeight, + } + } else { + log.L(ctx).Warnf("Transaction %s already in memory. Ignoring", tx.ID) + } return false - } - if s.incompleteTxSProcessMap[tx.ID.String()] == nil { + } else { if len(s.incompleteTxSProcessMap) >= s.maxConcurrentProcess { // TODO: decide how this map is managed, it shouldn't track the entire lifecycle // tx processing pool is full, queue the item return true } else { - s.incompleteTxSProcessMap[tx.ID.String()] = NewTransactionFlow(ctx, tx, s.nodeID, s.components, s.domainAPI, s.publisher, s.endorsementGatherer, s.identityResolver, s.syncPoints, s.transportWriter, s.requestTimeout) + s.incompleteTxSProcessMap[tx.ID.String()] = NewTransactionFlow(ctx, tx, s.nodeName, s.components, s.domainAPI, s.coordinatorDomainContext, s.publisher, s.endorsementGatherer, s.identityResolver, s.syncPoints, s.transportWriter, s.requestTimeout, s.coordinatorSelector, s.assembleCoordinator, s.environment) } - s.pendingEvents <- &ptmgrtypes.TransactionSwappedInEvent{ + s.pendingTransactionEvents <- &ptmgrtypes.TransactionSwappedInEvent{ PrivateTransactionEventBase: ptmgrtypes.PrivateTransactionEventBase{TransactionID: tx.ID.String()}, } } @@ -243,12 +314,14 @@ func (s *Sequencer) ProcessInFlightTransaction(ctx context.Context, tx *componen } func (s *Sequencer) HandleEvent(ctx context.Context, event ptmgrtypes.PrivateTransactionEvent) { - s.pendingEvents <- event + s.pendingTransactionEvents <- event } -func (s *Sequencer) Start(c context.Context) (done <-chan struct{}, err error) { +func (s *Sequencer) Start(ctx context.Context) (done <-chan struct{}, err error) { + log.L(ctx).Info("Starting Sequencer") s.syncPoints.Start() s.sequencerLoopDone = make(chan struct{}) + s.assembleCoordinator.Start() go s.evaluationLoop() s.TriggerSequencerEvaluation() return s.sequencerLoopDone, nil @@ -258,6 +331,7 @@ func (s *Sequencer) Start(c context.Context) (done <-chan struct{}, err error) { func (s *Sequencer) Stop() { // try to send an item in `stopProcess` channel, which has a buffer of 1 // if it already has an item in the channel, this function does nothing + s.assembleCoordinator.Stop() select { case s.stopProcess <- true: default: @@ -274,14 +348,49 @@ func (s *Sequencer) TriggerSequencerEvaluation() { } } -func (s *Sequencer) GetTxStatus(ctx context.Context, txID string) (status components.PrivateTxStatus, err error) { - //TODO This is primarily here to help with testing for now - // this needs to be revisited ASAP as part of a holisitic review of the persistence model +func (s *Sequencer) GetTxStatus(ctx context.Context, txID uuid.UUID) (components.PrivateTxStatus, error) { + s.incompleteTxProcessMapMutex.Lock() defer s.incompleteTxProcessMapMutex.Unlock() - if txProc, ok := s.incompleteTxSProcessMap[txID]; ok { + if txProc, ok := s.incompleteTxSProcessMap[txID.String()]; ok { return txProc.GetTxStatus(ctx) } - //TODO should be possible to query the status of a transaction that is not inflight - return components.PrivateTxStatus{}, i18n.NewError(ctx, msgs.MsgPrivateTxManagerInternalError, "Transaction not found") + persistedTxn, err := s.components.TxManager().GetTransactionByIDFull(ctx, txID) + if err != nil { + log.L(ctx).Errorf("Error getting persisted transaction by ID: %s", err) + return components.PrivateTxStatus{}, err + } + + status := "unknown" + failureMessage := "" + if persistedTxn.Receipt != nil { + if persistedTxn.Receipt.Success { + status = "confirmed" + } else { + failureMessage = persistedTxn.Receipt.FailureMessage + status = "failed" + } + + } + + return components.PrivateTxStatus{ + TxID: txID.String(), + Status: status, + FailureMessage: failureMessage, + }, nil +} + +func (s *Sequencer) HandleStateProducedEvent(ctx context.Context, stateProducedEvent *pbEngine.StateProducedEvent) { + readTX := s.components.Persistence().DB() // no DB transaction required here for the reads from the DB + log.L(ctx).Debug("Sequencer:HandleStateProducedEvent Upserting state to delegateDomainContext") + + states, err := s.delegateDomainContext.UpsertStates(readTX, &components.StateUpsert{ + SchemaID: tktypes.MustParseBytes32(stateProducedEvent.SchemaId), + Data: tktypes.RawJSON(stateProducedEvent.StateDataJson), + }) + if err != nil { + log.L(ctx).Errorf("Error upserting states: %s", err) + return + } + log.L(ctx).Debugf("Upserted states: %v", states) } diff --git a/core/go/internal/privatetxnmgr/sequencer_dispatch.go b/core/go/internal/privatetxnmgr/sequencer_dispatch.go index 8a7945a35..d67c92c74 100644 --- a/core/go/internal/privatetxnmgr/sequencer_dispatch.go +++ b/core/go/internal/privatetxnmgr/sequencer_dispatch.go @@ -49,27 +49,19 @@ func (s *Sequencer) DispatchTransactions(ctx context.Context, dispatchableTransa localStateDistributions := make([]*components.StateDistribution, 0) preparedTxnDistributions := make([]*preparedtxdistribution.PreparedTxnDistribution, 0) - completed := false // and include whether we committed the DB transaction or not - for signingAddress, transactionIDs := range dispatchableTransactions { - log.L(ctx).Debugf("DispatchTransactions: %d transactions for signingAddress %s", len(transactionIDs), signingAddress) + for signingAddress, transactionFlows := range dispatchableTransactions { + log.L(ctx).Debugf("DispatchTransactions: %d transactions for signingAddress %s", len(transactionFlows), signingAddress) - publicTransactionsToSend := make([]*components.PrivateTransaction, 0, len(transactionIDs)) + publicTransactionsToSend := make([]*components.PrivateTransaction, 0, len(transactionFlows)) sequence := &syncpoints.PublicDispatch{} - for _, transactionID := range transactionIDs { - // prepare all transactions for the given transaction IDs - - txProcessor := s.getTransactionProcessor(transactionID) - if txProcessor == nil { - //TODO currently assume that all the transactions are in flight and in memory - // need to reload from database if not in memory - panic("Transaction not found") - } + for _, transactionFlow := range transactionFlows { + // prepare all transactions // If we don't have a signing key for the TX at this point, we use our randomly assigned one // TODO: Rotation - preparedTransaction, err := txProcessor.PrepareTransaction(ctx, s.defaultSigner) + preparedTransaction, err := transactionFlow.PrepareTransaction(ctx, s.defaultSigner) if err != nil { log.L(ctx).Errorf("Error preparing transaction: %s", err) //TODO this is a really bad time to be getting an error. need to think carefully about how to handle this @@ -79,10 +71,10 @@ func (s *Sequencer) DispatchTransactions(ctx context.Context, dispatchableTransa hasPrivateTransaction := preparedTransaction.PreparedPrivateTransaction != nil switch { case preparedTransaction.Inputs.Intent == prototk.TransactionSpecification_SEND_TRANSACTION && hasPublicTransaction && !hasPrivateTransaction: - log.L(ctx).Infof("Result of transaction %s is a prepared public transaction", preparedTransaction.ID) + log.L(ctx).Infof("Result of transaction %s is a public transaction (gas=%d)", preparedTransaction.ID, *preparedTransaction.PreparedPublicTransaction.PublicTxOptions.Gas) publicTransactionsToSend = append(publicTransactionsToSend, preparedTransaction) sequence.PrivateTransactionDispatches = append(sequence.PrivateTransactionDispatches, &syncpoints.DispatchPersisted{ - PrivateTransactionID: transactionID, + PrivateTransactionID: transactionFlow.ID(ctx).String(), }) case preparedTransaction.Inputs.Intent == prototk.TransactionSpecification_SEND_TRANSACTION && hasPrivateTransaction && !hasPublicTransaction: log.L(ctx).Infof("Result of transaction %s is a chained private transaction", preparedTransaction.ID) @@ -119,7 +111,7 @@ func (s *Sequencer) DispatchTransactions(ctx context.Context, dispatchableTransa return err } - sds, err := txProcessor.GetStateDistributions(ctx) + sds, err := transactionFlow.GetStateDistributions(ctx) if err != nil { return err } @@ -127,70 +119,62 @@ func (s *Sequencer) DispatchTransactions(ctx context.Context, dispatchableTransa localStateDistributions = append(localStateDistributions, sds.Local...) } - preparedTransactionPayloads := make([]*pldapi.TransactionInput, len(publicTransactionsToSend)) - - for j, preparedTransaction := range publicTransactionsToSend { - preparedTransactionPayloads[j] = preparedTransaction.PreparedPublicTransaction - } - //Now we have the payloads, we can prepare the submission publicTransactionEngine := s.components.PublicTxManager() - signers := make([]string, len(publicTransactionsToSend)) - for i, pt := range publicTransactionsToSend { - unqualifiedSigner, err := tktypes.PrivateIdentityLocator(pt.Signer).Identity(ctx) - if err != nil { - errorMessage := fmt.Sprintf("failed to parse lookup key for signer %s : %s", pt.Signer, err) - log.L(ctx).Error(errorMessage) - return i18n.WrapError(ctx, err, msgs.MsgPrivateTxManagerInternalError, errorMessage) - } + // we may or may not have any transactions to send depending on the submit mode + if len(publicTransactionsToSend) == 0 { + log.L(ctx).Debugf("No public transactions to send for signing address %s", signingAddress) + } else { - signers[i] = unqualifiedSigner - } - keyMgr := s.components.KeyManager() - resolvedAddrs, err := keyMgr.ResolveEthAddressBatchNewDatabaseTX(ctx, signers) - if err != nil { - return err - } + signers := make([]string, len(publicTransactionsToSend)) + for i, pt := range publicTransactionsToSend { + unqualifiedSigner, err := tktypes.PrivateIdentityLocator(pt.Signer).Identity(ctx) + if err != nil { + errorMessage := fmt.Sprintf("failed to parse lookup key for signer %s : %s", pt.Signer, err) + log.L(ctx).Error(errorMessage) + return i18n.WrapError(ctx, err, msgs.MsgPrivateTxManagerInternalError, errorMessage) + } - publicTXs := make([]*components.PublicTxSubmission, len(publicTransactionsToSend)) - for i, pt := range publicTransactionsToSend { - log.L(ctx).Debugf("DispatchTransactions: creating PublicTxSubmission from %s", pt.Signer) - publicTXs[i] = &components.PublicTxSubmission{ - Bindings: []*components.PaladinTXReference{{TransactionID: pt.ID, TransactionType: pldapi.TransactionTypePrivate.Enum()}}, - PublicTxInput: pldapi.PublicTxInput{ - From: resolvedAddrs[i], - To: &s.contractAddress, - PublicTxOptions: pt.PublicTxOptions, - }, + signers[i] = unqualifiedSigner } - - // TODO: This aligning with submission in public Tx manage - data, err := pt.PreparedPublicTransaction.ABI[0].EncodeCallDataJSONCtx(ctx, pt.PreparedPublicTransaction.Data) + keyMgr := s.components.KeyManager() + resolvedAddrs, err := keyMgr.ResolveEthAddressBatchNewDatabaseTX(ctx, signers) if err != nil { return err } - publicTXs[i].Data = tktypes.HexBytes(data) - } - pubBatch, err := publicTransactionEngine.PrepareSubmissionBatch(ctx, publicTXs) - if err != nil { - return i18n.WrapError(ctx, err, msgs.MsgPrivTxMgrPublicTxFail) - } - // Must make sure from this point we return the nonces - sequence.PublicTxBatch = pubBatch - defer func() { - pubBatch.Completed(ctx, completed) - }() - if len(pubBatch.Rejected()) > 0 { - // We do not handle partial success - roll everything back - return i18n.WrapError(ctx, pubBatch.Rejected()[0].RejectedError(), msgs.MsgPrivTxMgrPublicTxFail) - } - dispatchBatch.PublicDispatches = append(dispatchBatch.PublicDispatches, sequence) + publicTXs := make([]*components.PublicTxSubmission, len(publicTransactionsToSend)) + for i, pt := range publicTransactionsToSend { + log.L(ctx).Debugf("DispatchTransactions: creating PublicTxSubmission from %s", pt.Signer) + publicTXs[i] = &components.PublicTxSubmission{ + Bindings: []*components.PaladinTXReference{{TransactionID: pt.ID, TransactionType: pldapi.TransactionTypePrivate.Enum()}}, + PublicTxInput: pldapi.PublicTxInput{ + From: resolvedAddrs[i], + To: &s.contractAddress, + PublicTxOptions: pt.PreparedPublicTransaction.PublicTxOptions, + }, + } + + // TODO: This aligning with submission in public Tx manage + data, err := pt.PreparedPublicTransaction.ABI[0].EncodeCallDataJSONCtx(ctx, pt.PreparedPublicTransaction.Data) + if err != nil { + return err + } + publicTXs[i].Data = tktypes.HexBytes(data) + + err = publicTransactionEngine.ValidateTransaction(ctx, s.components.Persistence().DB(), publicTXs[i]) + if err != nil { + return i18n.WrapError(ctx, err, msgs.MsgPrivTxMgrPublicTxFail) + } + } + sequence.PublicTxs = publicTXs + dispatchBatch.PublicDispatches = append(dispatchBatch.PublicDispatches, sequence) + + } } - // TODO: per notes in endorsementGatherer determine if that's the right place to hold the domain context - dCtx := s.endorsementGatherer.DomainContext() + dCtx := s.coordinatorDomainContext // Determine if there are any local nullifiers that need to be built and put into the domain context // before we persist the dispatch batch @@ -202,17 +186,19 @@ func (s *Sequencer) DispatchTransactions(ctx context.Context, dispatchableTransa return err } - err = s.syncPoints.PersistDispatchBatch(s.endorsementGatherer.DomainContext(), s.contractAddress, dispatchBatch, stateDistributions, preparedTxnDistributions) + err = s.syncPoints.PersistDispatchBatch(dCtx, s.contractAddress, dispatchBatch, stateDistributions, preparedTxnDistributions) if err != nil { log.L(ctx).Errorf("Error persisting batch: %s", err) return err } - completed = true for signingAddress, sequence := range dispatchableTransactions { - for _, privateTransactionID := range sequence { - s.publisher.PublishTransactionDispatchedEvent(ctx, privateTransactionID, uint64(0) /*TODO*/, signingAddress) + for _, transactionFlow := range sequence { + s.publisher.PublishTransactionDispatchedEvent(ctx, transactionFlow.ID(ctx).String(), uint64(0) /*TODO*/, signingAddress) } } + for _, preparedTransaction := range dispatchBatch.PreparedTransactions { + s.publisher.PublishTransactionPreparedEvent(ctx, preparedTransaction.ID.String()) + } //now that the DB write has been persisted, we can trigger the in-memory distribution of the prepared transactions and states s.stateDistributer.DistributeStates(ctx, stateDistributions) @@ -220,7 +206,7 @@ func (s *Sequencer) DispatchTransactions(ctx context.Context, dispatchableTransa // We also need to trigger ourselves for any private TX we chained for _, tx := range dispatchBatch.PrivateDispatches { - if err := s.privateTxManager.HandleNewTx(ctx, tx); err != nil { + if err := s.privateTxManager.HandleNewTx(ctx, s.components.Persistence().DB(), tx); err != nil { log.L(ctx).Errorf("Sequencer failed to notify private TX manager for chained transaction") } } diff --git a/core/go/internal/privatetxnmgr/sequencer_event_loop.go b/core/go/internal/privatetxnmgr/sequencer_event_loop.go index d0a1aef36..326f94e1f 100644 --- a/core/go/internal/privatetxnmgr/sequencer_event_loop.go +++ b/core/go/internal/privatetxnmgr/sequencer_event_loop.go @@ -41,8 +41,11 @@ func (s *Sequencer) evaluationLoop() { for { // an InFlight select { - case pendingEvent := <-s.pendingEvents: - s.handleEvent(ctx, pendingEvent) + case blockHeight := <-s.newBlockEvents: + //TODO should we use this is as the metronome to periodically trigger any inflight transactions to re-evaluate their state? + s.environment.blockHeight = blockHeight + case pendingEvent := <-s.pendingTransactionEvents: + s.handleTransactionEvent(ctx, pendingEvent) case <-s.orchestrationEvalRequestChan: case <-ticker.C: case <-ctx.Done(): @@ -59,7 +62,7 @@ func (s *Sequencer) evaluationLoop() { } } -func (s *Sequencer) handleEvent(ctx context.Context, event ptmgrtypes.PrivateTransactionEvent) { +func (s *Sequencer) handleTransactionEvent(ctx context.Context, event ptmgrtypes.PrivateTransactionEvent) { //For any event that is specific to a single transaction, // find (or create) the transaction processor for that transaction // and pass the event to it @@ -94,7 +97,6 @@ func (s *Sequencer) handleEvent(ctx context.Context, event ptmgrtypes.PrivateTra it must either - decide to ignore the event altogether - completely apply the the event - - panic if the event data is partially applied and an unexpected error occurs before it can be completely applied */ transactionProcessor.ApplyEvent(ctx, event) @@ -102,7 +104,7 @@ func (s *Sequencer) handleEvent(ctx context.Context, event ptmgrtypes.PrivateTra After applying the event to the transaction, we can either a) clean up that transaction ( if we have just learned, from the event that the transaction is complete and needs no further actions) or b) perform any necessary actions (e.g. sending requests for signatures, endorsements etc.) */ - if transactionProcessor.IsComplete() { + if transactionProcessor.IsComplete(ctx) { s.graph.RemoveTransaction(ctx, transactionID) s.removeTransactionProcessor(transactionID) @@ -116,10 +118,18 @@ func (s *Sequencer) handleEvent(ctx context.Context, event ptmgrtypes.PrivateTra transactionProcessor.Action(ctx) } - if transactionProcessor.CoordinatingLocally() && transactionProcessor.ReadyForSequencing() && !transactionProcessor.Dispatched() { + if transactionProcessor.CoordinatingLocally(ctx) && transactionProcessor.ReadyForSequencing(ctx) && !transactionProcessor.Dispatched(ctx) { // we are responsible for coordinating the endorsement flow for this transaction, ensure that it has been added it to the graph // NOTE: AddTransaction is idempotent so we don't need to check whether we have already added it s.graph.AddTransaction(ctx, transactionProcessor) + } else { + // incase the transaction was previously added to the graph but is no longer coordinating locally or is no longer ready for sequencing + // then we need to remove it from the graph + // this is a no-op if the transaction was not previously added to the graph + + //TODO - this should really be a method on the graph itself ( similar to GetDispatchableTransactions) to find all transactions ( and there dependents) + // that are no longer ready for sequencing and remove them from the graph + s.graph.RemoveTransaction(ctx, transactionID) } //analyze the graph to see if we can dispatch any transactions @@ -144,6 +154,6 @@ func (s *Sequencer) handleEvent(ctx context.Context, event ptmgrtypes.PrivateTra } //DispatchTransactions is a persistence point so we can remove the transactions from our graph now that they are dispatched - s.graph.RemoveTransactions(ctx, dispatchableTransactions) + s.graph.RemoveTransactions(ctx, dispatchableTransactions.IDs(ctx)) } diff --git a/core/go/internal/privatetxnmgr/sequencer_test.go b/core/go/internal/privatetxnmgr/sequencer_test.go index 2739b038b..a64a92d6d 100644 --- a/core/go/internal/privatetxnmgr/sequencer_test.go +++ b/core/go/internal/privatetxnmgr/sequencer_test.go @@ -31,6 +31,7 @@ import ( "github.com/kaleido-io/paladin/core/mocks/statedistributionmocks" "github.com/kaleido-io/paladin/core/pkg/persistence" + "github.com/kaleido-io/paladin/toolkit/pkg/prototk" "github.com/kaleido-io/paladin/toolkit/pkg/tktypes" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -43,6 +44,7 @@ type sequencerDepencyMocks struct { domainSmartContract *componentmocks.DomainSmartContract domainContext *componentmocks.DomainContext domainMgr *componentmocks.DomainManager + domain *componentmocks.Domain transportManager *componentmocks.TransportManager stateStore *componentmocks.StateManager keyManager *componentmocks.KeyManager @@ -52,6 +54,7 @@ type sequencerDepencyMocks struct { stateDistributer *statedistributionmocks.StateDistributer preparedTransactionDistributer *preparedtxdistributionmocks.PreparedTransactionDistributer txManager *componentmocks.TXManager + pubTxManager *componentmocks.PublicTxManager transportWriter *privatetxnmgrmocks.TransportWriter } @@ -66,6 +69,7 @@ func newSequencerForTesting(t *testing.T, ctx context.Context, domainAddress *tk domainSmartContract: componentmocks.NewDomainSmartContract(t), domainContext: componentmocks.NewDomainContext(t), domainMgr: componentmocks.NewDomainManager(t), + domain: componentmocks.NewDomain(t), transportManager: componentmocks.NewTransportManager(t), stateStore: componentmocks.NewStateManager(t), keyManager: componentmocks.NewKeyManager(t), @@ -75,6 +79,7 @@ func newSequencerForTesting(t *testing.T, ctx context.Context, domainAddress *tk stateDistributer: statedistributionmocks.NewStateDistributer(t), preparedTransactionDistributer: preparedtxdistributionmocks.NewPreparedTransactionDistributer(t), txManager: componentmocks.NewTXManager(t), + pubTxManager: componentmocks.NewPublicTxManager(t), transportWriter: privatetxnmgrmocks.NewTransportWriter(t), } mocks.allComponents.On("StateManager").Return(mocks.stateStore).Maybe() @@ -82,15 +87,24 @@ func newSequencerForTesting(t *testing.T, ctx context.Context, domainAddress *tk mocks.allComponents.On("TransportManager").Return(mocks.transportManager).Maybe() mocks.allComponents.On("KeyManager").Return(mocks.keyManager).Maybe() mocks.allComponents.On("TxManager").Return(mocks.txManager).Maybe() - mocks.domainMgr.On("GetSmartContractByAddress", mock.Anything, *domainAddress).Maybe().Return(mocks.domainSmartContract, nil) + mocks.allComponents.On("PublicTxManager").Return(mocks.pubTxManager).Maybe() + mocks.domainMgr.On("GetSmartContractByAddress", mock.Anything, mock.Anything, *domainAddress).Maybe().Return(mocks.domainSmartContract, nil) p, persistenceDone, err := persistence.NewUnitTestPersistence(ctx, "privatetxmgr") require.NoError(t, err) mocks.allComponents.On("Persistence").Return(p).Maybe() mocks.endorsementGatherer.On("DomainContext").Return(mocks.domainContext).Maybe() + mocks.domainSmartContract.On("Domain").Return(mocks.domain).Maybe() mocks.domainSmartContract.On("Address").Return(*domainAddress).Maybe() + mocks.domainSmartContract.On("ContractConfig").Return(&prototk.ContractConfig{ + CoordinatorSelection: prototk.ContractConfig_COORDINATOR_ENDORSER, + }) + + mocks.stateStore.On("NewDomainContext", mock.Anything, mocks.domain, *domainAddress, mock.Anything).Return(mocks.domainContext).Maybe() + //mocks.domain.On("Configuration").Return(&prototk.DomainConfig{}).Maybe() - syncPoints := syncpoints.NewSyncPoints(ctx, &pldconf.FlushWriterConfig{}, p, mocks.txManager) - o := NewSequencer(ctx, mocks.privateTxManager, tktypes.RandHex(16), *domainAddress, &pldconf.PrivateTxManagerSequencerConfig{}, mocks.allComponents, mocks.domainSmartContract, mocks.endorsementGatherer, mocks.publisher, syncPoints, mocks.identityResolver, mocks.stateDistributer, mocks.preparedTransactionDistributer, mocks.transportWriter, 30*time.Second) + syncPoints := syncpoints.NewSyncPoints(ctx, &pldconf.FlushWriterConfig{}, p, mocks.txManager, mocks.pubTxManager) + o, err := NewSequencer(ctx, mocks.privateTxManager, tktypes.RandHex(16), *domainAddress, &pldconf.PrivateTxManagerSequencerConfig{}, mocks.allComponents, mocks.domainSmartContract, mocks.endorsementGatherer, mocks.publisher, syncPoints, mocks.identityResolver, mocks.stateDistributer, mocks.preparedTransactionDistributer, mocks.transportWriter, 30*time.Second, 0) + require.NoError(t, err) ocDone, err := o.Start(ctx) require.NoError(t, err) @@ -137,7 +151,7 @@ func TestNewSequencerProcessNewTransactionAssemblyFailed(t *testing.T) { waitForFinalize := make(chan bool, 1) dependencyMocks.domainSmartContract.On("AssembleTransaction", mock.Anything, mock.Anything, mock.Anything).Return(errors.New("fail assembly. Just happy that we got this far")) - dependencyMocks.publisher.On("PublishTransactionAssembleFailedEvent", mock.Anything, newTxID.String(), "mock.Anything").Return(nil).Run(func(args mock.Arguments) { + dependencyMocks.publisher.On("PublishTransactionAssembleFailedEvent", mock.Anything, newTxID.String(), mock.Anything, mock.Anything).Return(nil).Run(func(args mock.Arguments) { waitForFinalize <- true }) //As we are using a mock publisher, the assemble failed event never gets back onto the event loop to trigger the next step ( finalization ) diff --git a/core/go/internal/privatetxnmgr/syncpoints/delegate.go b/core/go/internal/privatetxnmgr/syncpoints/delegate.go deleted file mode 100644 index 90ad3e5c4..000000000 --- a/core/go/internal/privatetxnmgr/syncpoints/delegate.go +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright © 2024 Kaleido, Inc. -// -// SPDX-License-Identifier: Apache-2.0 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package syncpoints - -import ( - "context" - - "github.com/google/uuid" - "github.com/kaleido-io/paladin/core/internal/components" - "github.com/kaleido-io/paladin/toolkit/pkg/log" - "github.com/kaleido-io/paladin/toolkit/pkg/tktypes" - "gorm.io/gorm" - "gorm.io/gorm/clause" -) - -type delegateOperation struct { - ID uuid.UUID `json:"id"` - TransactionID uuid.UUID `json:"transaction_id"` - DelegateNodeID string `json:"delegate_node_id"` -} - -// DelegateTransaction writes a record to the local database recording that the given transaction has been delegated to the given delegate -// then triggers a reliable cross node handshake to transmit that delegation to the delegate node and record their acknowledgement -func (s *syncPoints) QueueDelegation(dCtx components.DomainContext, contractAddress tktypes.EthAddress, transactionID uuid.UUID, delegateNodeID string, onCommit func(context.Context), onRollback func(context.Context, error)) { - - op := s.writer.Queue(dCtx.Ctx(), &syncPointOperation{ - domainContext: dCtx, - contractAddress: contractAddress, - delegateOperation: &delegateOperation{ - ID: uuid.New(), - TransactionID: transactionID, - DelegateNodeID: delegateNodeID, - }, - }) - go func() { - if _, err := op.WaitFlushed(dCtx.Ctx()); err != nil { - onRollback(dCtx.Ctx(), err) - } else { - onCommit(dCtx.Ctx()) - } - }() -} - -func (s *syncPoints) writeDelegateOperations(ctx context.Context, dbTX *gorm.DB, delegateOperations []*delegateOperation) error { - - // For each operation in the batch, we simply need insert a row to the database - log.L(ctx).Debugf("Writing delegations %d", len(delegateOperations)) - err := dbTX. - Table("transaction_delegations"). - Clauses(clause.OnConflict{ - Columns: []clause.Column{ - {Name: "transaction_id"}, - {Name: "delegate_node_id"}, - }, - DoNothing: true, // immutable - }). - Create(delegateOperations). - Error - - if err != nil { - log.L(ctx).Errorf("Error persisting delegations: %s", err) - return err - } - - return nil -} diff --git a/core/go/internal/privatetxnmgr/syncpoints/delegate_test.go b/core/go/internal/privatetxnmgr/syncpoints/delegate_test.go deleted file mode 100644 index 3d0c9e327..000000000 --- a/core/go/internal/privatetxnmgr/syncpoints/delegate_test.go +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright © 2024 Kaleido, Inc. -// -// SPDX-License-Identifier: Apache-2.0 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package syncpoints - -import ( - "context" - "database/sql/driver" - "testing" - - "github.com/DATA-DOG/go-sqlmock" - "github.com/google/uuid" - "github.com/kaleido-io/paladin/toolkit/pkg/tktypes" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestWriteDelegateOperations(t *testing.T) { - ctx := context.Background() - - dc, flushResult := newTestDomainContextWithFlush(t) - - s, m := newSyncPointsForTesting(t) - testDelegateNodeA := "delegateNodeA" - testDelegateNodeB := "delegateNodeB" - testTxnID1 := uuid.New() - testTxnID2 := uuid.New() - testTxnID3 := uuid.New() - testContractAddress1 := tktypes.RandAddress() - testContractAddress2 := tktypes.RandAddress() - testSyncPointOperations := []*syncPointOperation{ - { - domainContext: dc, - contractAddress: *testContractAddress1, - delegateOperation: &delegateOperation{ - TransactionID: testTxnID1, - DelegateNodeID: testDelegateNodeA, - }, - }, - { - domainContext: dc, - contractAddress: *testContractAddress1, - delegateOperation: &delegateOperation{ - TransactionID: testTxnID2, - DelegateNodeID: testDelegateNodeB, - }, - }, - { - domainContext: dc, - contractAddress: *testContractAddress2, - delegateOperation: &delegateOperation{ - TransactionID: testTxnID3, - DelegateNodeID: testDelegateNodeB, - }, - }, - } - dbTX := m.persistence.P.DB() - m.persistence.Mock.ExpectExec("INSERT.*transaction_delegations").WithArgs( - sqlmock.AnyArg(), testTxnID1, testDelegateNodeA, - sqlmock.AnyArg(), testTxnID2, testDelegateNodeB, - sqlmock.AnyArg(), testTxnID3, testDelegateNodeB, - ).WillReturnResult(driver.ResultNoRows) - - dbResultCB, res, err := s.runBatch(ctx, dbTX, testSyncPointOperations) - assert.NoError(t, err) - require.Len(t, res, 3) - assert.NoError(t, m.persistence.Mock.ExpectationsWereMet()) - dbResultCB(nil) - require.NoError(t, <-flushResult) -} diff --git a/core/go/internal/privatetxnmgr/syncpoints/delegation_acks.go b/core/go/internal/privatetxnmgr/syncpoints/delegation_acks.go deleted file mode 100644 index 5520c3ff2..000000000 --- a/core/go/internal/privatetxnmgr/syncpoints/delegation_acks.go +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright © 2024 Kaleido, Inc. -// -// SPDX-License-Identifier: Apache-2.0 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package syncpoints - -import ( - "context" - - "github.com/google/uuid" - "github.com/kaleido-io/paladin/core/internal/components" - "github.com/kaleido-io/paladin/toolkit/pkg/log" - "github.com/kaleido-io/paladin/toolkit/pkg/tktypes" - "gorm.io/gorm" - "gorm.io/gorm/clause" -) - -type delegationAckOperation struct { - ID uuid.UUID `json:"id"` - DelegationID uuid.UUID `json:"delegation_id"` -} - -func (s *syncPoints) QueueDelegationAck(dCtx components.DomainContext, contractAddress tktypes.EthAddress, delegationID uuid.UUID, onCommit func(context.Context), onRollback func(context.Context, error)) { - - op := s.writer.Queue(dCtx.Ctx(), &syncPointOperation{ - domainContext: dCtx, - contractAddress: contractAddress, - delegationAckOperation: &delegationAckOperation{ - ID: uuid.New(), - DelegationID: delegationID, - }, - }) - go func() { - if _, err := op.WaitFlushed(dCtx.Ctx()); err != nil { - onRollback(dCtx.Ctx(), err) - } else { - onCommit(dCtx.Ctx()) - } - }() -} - -func (s *syncPoints) writeDelegationAckOperations(ctx context.Context, dbTX *gorm.DB, delegationAckOperations []*delegationAckOperation) error { - - // For each operation in the batch, we simply need insert a row to the database - log.L(ctx).Debugf("Writing delegations %d", len(delegationAckOperations)) - err := dbTX. - Table("transaction_delegation_acknowledgements"). - Clauses(clause.OnConflict{ - Columns: []clause.Column{ - {Name: "delegation"}, - }, - DoNothing: true, // immutable - }). - Create(delegationAckOperations). - Error - - if err != nil { - log.L(ctx).Errorf("Error persisting delegation acknowledgments: %s", err) - return err - } - - return nil -} diff --git a/core/go/internal/privatetxnmgr/syncpoints/delegation_acks_test.go b/core/go/internal/privatetxnmgr/syncpoints/delegation_acks_test.go deleted file mode 100644 index 41feb9592..000000000 --- a/core/go/internal/privatetxnmgr/syncpoints/delegation_acks_test.go +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright © 2024 Kaleido, Inc. -// -// SPDX-License-Identifier: Apache-2.0 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package syncpoints - -import ( - "context" - "database/sql/driver" - "testing" - - "github.com/DATA-DOG/go-sqlmock" - "github.com/google/uuid" - "github.com/kaleido-io/paladin/toolkit/pkg/tktypes" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestWriteDelegationAcksOperations(t *testing.T) { - ctx := context.Background() - - dc, flushResult := newTestDomainContextWithFlush(t) - - s, m := newSyncPointsForTesting(t) - testDelegationID1 := uuid.New() - testDelegationID2 := uuid.New() - testContractAddress1 := tktypes.RandAddress() - testContractAddress2 := tktypes.RandAddress() - testSyncPointOperations := []*syncPointOperation{ - { - domainContext: dc, - contractAddress: *testContractAddress1, - delegationAckOperation: &delegationAckOperation{ - DelegationID: testDelegationID1, - }, - }, - { - domainContext: dc, - contractAddress: *testContractAddress2, - delegationAckOperation: &delegationAckOperation{ - DelegationID: testDelegationID2, - }, - }, - } - dbTX := m.persistence.P.DB() - m.persistence.Mock.ExpectExec("INSERT.*transaction_delegation_acknowledgements").WithArgs( - sqlmock.AnyArg(), testDelegationID1, - sqlmock.AnyArg(), testDelegationID2, - ).WillReturnResult(driver.ResultNoRows) - - dbResultCB, res, err := s.runBatch(ctx, dbTX, testSyncPointOperations) - assert.NoError(t, err) - require.Len(t, res, 2) - assert.NoError(t, m.persistence.Mock.ExpectationsWereMet()) - dbResultCB(nil) - require.NoError(t, <-flushResult) - -} diff --git a/core/go/internal/privatetxnmgr/syncpoints/dispatch.go b/core/go/internal/privatetxnmgr/syncpoints/dispatch.go index a07299db8..16d059c96 100644 --- a/core/go/internal/privatetxnmgr/syncpoints/dispatch.go +++ b/core/go/internal/privatetxnmgr/syncpoints/dispatch.go @@ -18,12 +18,9 @@ package syncpoints import ( "context" - "fmt" "github.com/google/uuid" - "github.com/hyperledger/firefly-common/pkg/i18n" "github.com/kaleido-io/paladin/core/internal/components" - "github.com/kaleido-io/paladin/core/internal/msgs" "github.com/kaleido-io/paladin/core/internal/preparedtxdistribution" "github.com/kaleido-io/paladin/core/internal/statedistribution" "github.com/kaleido-io/paladin/toolkit/pkg/log" @@ -44,12 +41,12 @@ type DispatchPersisted struct { ID string `json:"id"` PrivateTransactionID string `json:"privateTransactionID"` PublicTransactionAddress tktypes.EthAddress `json:"publicTransactionAddress"` - PublicTransactionNonce uint64 `json:"publicTransactionNonce"` + PublicTransactionID uint64 `json:"publicTransactionID"` } // A dispatch sequence is a collection of private transactions that are submitted together for a given signing address in order type PublicDispatch struct { - PublicTxBatch components.PublicTxBatch + PublicTxs []*components.PublicTxSubmission PrivateTransactionDispatches []*DispatchPersisted } @@ -119,7 +116,7 @@ func (s *syncPoints) PersistDeployDispatchBatch(ctx context.Context, dispatchBat return err } -func (s *syncPoints) writeDispatchOperations(ctx context.Context, dbTX *gorm.DB, dispatchOperations []*dispatchOperation) error { +func (s *syncPoints) writeDispatchOperations(ctx context.Context, dbTX *gorm.DB, dispatchOperations []*dispatchOperation) (pubTXCbs []func(), err error) { // For each operation in the batch, we need to call the baseledger transaction manager to allocate its nonce // which it can only guaranteed to be gapless and unique if it is done during the database transaction that inserts the dispatch record. @@ -134,22 +131,13 @@ func (s *syncPoints) writeDispatchOperations(ctx context.Context, dbTX *gorm.DB, continue } - // Call the public transaction manager to allocate nonces for all transactions in the sequence - // and persist them to the database under the current transaction - pubBatch := dispatchSequenceOp.PublicTxBatch - err := pubBatch.Submit(ctx, dbTX) + // Call the public transaction manager persist to the database under the current transaction + pubTXCb, publicTxns, err := s.pubTxMgr.WriteNewTransactions(ctx, dbTX, dispatchSequenceOp.PublicTxs) if err != nil { - log.L(ctx).Errorf("Error submitting public transaction: %s", err) - // TODO this is a really bad situation because it will cause all dispatches in the flush to rollback - // Should we skip this dispatch ( or this mini batch of dispatches?) - return err - } - publicTxIDs := pubBatch.Accepted() - if len(publicTxIDs) != len(dispatchSequenceOp.PrivateTransactionDispatches) { - errorMessage := fmt.Sprintf("Expected %d public transaction IDs, got %d", len(dispatchSequenceOp.PrivateTransactionDispatches), len(publicTxIDs)) - log.L(ctx).Error(errorMessage) - return i18n.NewError(ctx, msgs.MsgPrivateTxManagerInternalError, errorMessage) + log.L(ctx).Errorf("Error submitting public transactions: %s", err) + return nil, err } + pubTXCbs = append(pubTXCbs, pubTXCb) //TODO this results in an `INSERT` for each dispatchSequence //Would it be more efficient to pass an array for the whole flush? @@ -158,9 +146,8 @@ func (s *syncPoints) writeDispatchOperations(ctx context.Context, dbTX *gorm.DB, for dispatchIndex, dispatch := range dispatchSequenceOp.PrivateTransactionDispatches { //fill in the foreign key before persisting in our dispatch table - dispatch.PublicTransactionAddress = publicTxIDs[dispatchIndex].PublicTx().From - dispatch.PublicTransactionNonce = publicTxIDs[dispatchIndex].PublicTx().Nonce.Uint64() - + dispatch.PublicTransactionAddress = publicTxns[dispatchIndex].From + dispatch.PublicTransactionID = *publicTxns[dispatchIndex].LocalID dispatch.ID = uuid.New().String() } @@ -172,7 +159,7 @@ func (s *syncPoints) writeDispatchOperations(ctx context.Context, dbTX *gorm.DB, Columns: []clause.Column{ {Name: "private_transaction_id"}, {Name: "public_transaction_address"}, - {Name: "public_transaction_nonce"}, + {Name: "public_transaction_id"}, }, DoNothing: true, // immutable }). @@ -181,7 +168,7 @@ func (s *syncPoints) writeDispatchOperations(ctx context.Context, dbTX *gorm.DB, if err != nil { log.L(ctx).Errorf("Error persisting dispatches: %s", err) - return err + return nil, err } } @@ -189,7 +176,7 @@ func (s *syncPoints) writeDispatchOperations(ctx context.Context, dbTX *gorm.DB, if len(op.privateDispatches) > 0 { if err := s.txMgr.UpsertInternalPrivateTxsFinalizeIDs(ctx, dbTX, op.privateDispatches); err != nil { log.L(ctx).Errorf("Error persisting private dispatches: %s", err) - return err + return nil, err } } @@ -198,7 +185,7 @@ func (s *syncPoints) writeDispatchOperations(ctx context.Context, dbTX *gorm.DB, if err := s.txMgr.WritePreparedTransactions(ctx, dbTX, op.preparedTransactions); err != nil { log.L(ctx).Errorf("Error persisting prepared transactions: %s", err) - return err + return nil, err } } @@ -221,7 +208,7 @@ func (s *syncPoints) writeDispatchOperations(ctx context.Context, dbTX *gorm.DB, if err != nil { log.L(ctx).Errorf("Error persisting prepared transaction distributions: %s", err) - return err + return nil, err } } @@ -243,10 +230,10 @@ func (s *syncPoints) writeDispatchOperations(ctx context.Context, dbTX *gorm.DB, if err != nil { log.L(ctx).Errorf("Error persisting state distributions: %s", err) - return err + return nil, err } } } - return nil + return pubTXCbs, nil } diff --git a/core/go/internal/privatetxnmgr/syncpoints/syncpoints.go b/core/go/internal/privatetxnmgr/syncpoints/syncpoints.go index a023e631a..9d30004a1 100644 --- a/core/go/internal/privatetxnmgr/syncpoints/syncpoints.go +++ b/core/go/internal/privatetxnmgr/syncpoints/syncpoints.go @@ -61,24 +61,20 @@ type SyncPoints interface { // the onCommit and onRollback callbacks are called, on a separate goroutine when the transaction is committed or rolled back QueueTransactionFinalize(ctx context.Context, domain string, contractAddress tktypes.EthAddress, transactionID uuid.UUID, failureMessage string, onCommit func(context.Context), onRollback func(context.Context, error)) - // DelegateTransaction writes a record to the local database recording that the given transaction has been delegated to the given delegate - // then triggers a reliable cross node handshake to transmit that delegation to the delegate node and record their acknowledgement - QueueDelegation(dCtx components.DomainContext, contractAddress tktypes.EthAddress, transactionID uuid.UUID, delegateNodeID string, onCommit func(context.Context), onRollback func(context.Context, error)) - - // DelegateTransaction writes a record to the local database recording that we have received acknowledgement from the delegate node - QueueDelegationAck(dCtx components.DomainContext, contractAddress tktypes.EthAddress, delegationID uuid.UUID, onCommit func(context.Context), onRollback func(context.Context, error)) Close() } type syncPoints struct { - started bool - writer flushwriter.Writer[*syncPointOperation, *noResult] - txMgr components.TXManager + started bool + writer flushwriter.Writer[*syncPointOperation, *noResult] + txMgr components.TXManager + pubTxMgr components.PublicTxManager } -func NewSyncPoints(ctx context.Context, conf *pldconf.FlushWriterConfig, p persistence.Persistence, txMgr components.TXManager) SyncPoints { +func NewSyncPoints(ctx context.Context, conf *pldconf.FlushWriterConfig, p persistence.Persistence, txMgr components.TXManager, pubTxMgr components.PublicTxManager) SyncPoints { s := &syncPoints{ - txMgr: txMgr, + txMgr: txMgr, + pubTxMgr: pubTxMgr, } s.writer = flushwriter.NewWriter(ctx, s.runBatch, p, conf, &WriterConfigDefaults) return s diff --git a/core/go/internal/privatetxnmgr/syncpoints/writer.go b/core/go/internal/privatetxnmgr/syncpoints/writer.go index 7510c9829..ae594624b 100644 --- a/core/go/internal/privatetxnmgr/syncpoints/writer.go +++ b/core/go/internal/privatetxnmgr/syncpoints/writer.go @@ -54,12 +54,10 @@ to atomically allocate and record the nonce under that same transaction. // but never more than one of these. We probably could make the mutually exclusive nature more explicit by using interfaces but its not worth the added complexity type syncPointOperation struct { - contractAddress tktypes.EthAddress - domainContext components.DomainContext - finalizeOperation *finalizeOperation - dispatchOperation *dispatchOperation - delegateOperation *delegateOperation - delegationAckOperation *delegationAckOperation + contractAddress tktypes.EthAddress + domainContext components.DomainContext + finalizeOperation *finalizeOperation + dispatchOperation *dispatchOperation } func (dso *syncPointOperation) WriteKey() string { @@ -72,8 +70,6 @@ func (s *syncPoints) runBatch(ctx context.Context, dbTX *gorm.DB, values []*sync finalizeOperations := make([]*finalizeOperation, 0, len(values)) dispatchOperations := make([]*dispatchOperation, 0, len(values)) - delegateOperations := make([]*delegateOperation, 0, len(values)) - delegationAckOperations := make([]*delegationAckOperation, 0, len(values)) domainContextsToFlush := make(map[uuid.UUID]components.DomainContext) for _, op := range values { @@ -86,23 +82,23 @@ func (s *syncPoints) runBatch(ctx context.Context, dbTX *gorm.DB, values []*sync if op.dispatchOperation != nil { dispatchOperations = append(dispatchOperations, op.dispatchOperation) } - if op.delegateOperation != nil { - delegateOperations = append(delegateOperations, op.delegateOperation) - } - if op.delegationAckOperation != nil { - delegationAckOperations = append(delegationAckOperations, op.delegationAckOperation) - } } // We flush all of the affected domain contexts first, as they might contain states we need to refer // to in the DB transaction below using foreign key relationships domainContextDBTXCallbacks := make([]func(err error), 0, len(domainContextsToFlush)) + var pubTXCbs []func() dbTXCallback := func(err error) { for _, dcTXCallback := range domainContextDBTXCallbacks { if dcTXCallback != nil { dcTXCallback(err) } } + if err == nil { + for _, publicTXCallback := range pubTXCbs { + publicTXCallback() + } + } } // We must track if we're returning an error with a nil callback, and ensure that in those cases // we call the dbTXCallback with the error for any contexts we've received back from a Flush() call @@ -112,8 +108,8 @@ func (s *syncPoints) runBatch(ctx context.Context, dbTX *gorm.DB, values []*sync dbTXCallback(err) } }() - log.L(ctx).Infof("SyncPoints flush-writer: domain=contexts=%d finalizeOperations=%d dispatchOperations=%d delegateOperations=%d delegationAckOperations=%d", - len(domainContextsToFlush), len(finalizeOperations), len(dispatchOperations), len(delegateOperations), len(delegationAckOperations)) + log.L(ctx).Infof("SyncPoints flush-writer: domain=contexts=%d finalizeOperations=%d dispatchOperations=%d", + len(domainContextsToFlush), len(finalizeOperations), len(dispatchOperations)) for _, dc := range domainContextsToFlush { var domainCB func(error) domainCB, err = dc.Flush(dbTX) // err variable must not be re-allocated @@ -132,15 +128,7 @@ func (s *syncPoints) runBatch(ctx context.Context, dbTX *gorm.DB, values []*sync } if err == nil && len(dispatchOperations) > 0 { - err = s.writeDispatchOperations(ctx, dbTX, dispatchOperations) // err variable must not be re-allocated - } - - if err == nil && len(delegateOperations) > 0 { - err = s.writeDelegateOperations(ctx, dbTX, delegateOperations) // err variable must not be re-allocated - } - - if err == nil && len(delegationAckOperations) > 0 { - err = s.writeDelegationAckOperations(ctx, dbTX, delegationAckOperations) // err variable must not be re-allocated + pubTXCbs, err = s.writeDispatchOperations(ctx, dbTX, dispatchOperations) // err variable must not be re-allocated } if err != nil { diff --git a/core/go/internal/privatetxnmgr/transaction_flow.go b/core/go/internal/privatetxnmgr/transaction_flow.go index d50d48ff5..662c84212 100644 --- a/core/go/internal/privatetxnmgr/transaction_flow.go +++ b/core/go/internal/privatetxnmgr/transaction_flow.go @@ -17,6 +17,7 @@ package privatetxnmgr import ( "context" + "sync" "time" "github.com/google/uuid" @@ -30,11 +31,29 @@ import ( "github.com/kaleido-io/paladin/toolkit/pkg/prototk" ) -func NewTransactionFlow(ctx context.Context, transaction *components.PrivateTransaction, nodeID string, components components.AllComponents, domainAPI components.DomainSmartContract, publisher ptmgrtypes.Publisher, endorsementGatherer ptmgrtypes.EndorsementGatherer, identityResolver components.IdentityResolver, syncPoints syncpoints.SyncPoints, transportWriter ptmgrtypes.TransportWriter, requestTimeout time.Duration) ptmgrtypes.TransactionFlow { +func NewTransactionFlow( + ctx context.Context, + transaction *components.PrivateTransaction, + nodeName string, + components components.AllComponents, + domainAPI components.DomainSmartContract, + domainContext components.DomainContext, + publisher ptmgrtypes.Publisher, + endorsementGatherer ptmgrtypes.EndorsementGatherer, + identityResolver components.IdentityResolver, + syncPoints syncpoints.SyncPoints, + transportWriter ptmgrtypes.TransportWriter, + requestTimeout time.Duration, + selectCoordinator ptmgrtypes.CoordinatorSelector, + assembleCoordinator ptmgrtypes.AssembleCoordinator, + environment ptmgrtypes.SequencerEnvironment, +) ptmgrtypes.TransactionFlow { + return &transactionFlow{ stageErrorRetry: 10 * time.Second, domainAPI: domainAPI, - nodeID: nodeID, + domainContext: domainContext, + nodeName: nodeName, components: components, publisher: publisher, endorsementGatherer: endorsementGatherer, @@ -47,21 +66,31 @@ func NewTransactionFlow(ctx context.Context, transaction *components.PrivateTran finalizePending: false, requestedVerifierResolution: false, requestedSignatures: false, - requestedEndorsementTimes: make(map[string]map[string]time.Time), + pendingEndorsementRequests: make(map[string]map[string]*endorsementRequest), complete: false, localCoordinator: true, - readyForSequencing: false, dispatched: false, + prepared: false, clock: ptmgrtypes.RealClock(), requestTimeout: requestTimeout, + selectCoordinator: selectCoordinator, + assembleCoordinator: assembleCoordinator, + environment: environment, } } +type endorsementRequest struct { + //time the request was made + requestTime time.Time + //unique string to identify the request (non unique across retries) + idempotencyKey string +} type transactionFlow struct { stageErrorRetry time.Duration components components.AllComponents - nodeID string + nodeName string domainAPI components.DomainSmartContract + domainContext components.DomainContext transaction *components.PrivateTransaction publisher ptmgrtypes.Publisher endorsementGatherer ptmgrtypes.EndorsementGatherer @@ -74,35 +103,38 @@ type transactionFlow struct { finalizeRevertReason string finalizeRequired bool finalizePending bool + delegatePending bool + pendingDelegationRequestID string + delegateRequestTime time.Time + delegateRequestBlockHeight int64 + delegated bool + delegateRequestTimer *time.Timer + assemblePending bool complete bool - requestedVerifierResolution bool //TODO add precision here so that we can track individual requests and implement retry as per endorsement - requestedSignatures bool //TODO add precision here so that we can track individual requests and implement retry as per endorsement - requestedEndorsementTimes map[string]map[string]time.Time //map of attestationRequest names to a map of parties to the time the most request was made + requestedVerifierResolution bool //TODO add precision here so that we can track individual requests and implement retry as per endorsement + requestedSignatures bool //TODO add precision here so that we can track individual requests and implement retry as per endorsement + pendingEndorsementRequests map[string]map[string]*endorsementRequest //map of attestationRequest names to a map of parties to a struct containing information about the active pending request localCoordinator bool - readyForSequencing bool dispatched bool + prepared bool clock ptmgrtypes.Clock requestTimeout time.Duration + selectCoordinator ptmgrtypes.CoordinatorSelector + assembleCoordinator ptmgrtypes.AssembleCoordinator + environment ptmgrtypes.SequencerEnvironment + statusLock sync.RWMutex // under normal conditions, there should be only one contender for this lock ( the Write side of it) - i.e. the sequencer event loop so it should not normally slow things down + // however, it is not safe for the API thread to read the in memory status while the even loop is writing so things will slow down on the event loop thread while an API consumer is reading the status } -func (tf *transactionFlow) GetTxStatus(ctx context.Context) (components.PrivateTxStatus, error) { - return components.PrivateTxStatus{ - TxID: tf.transaction.ID.String(), - Status: tf.status, - LatestEvent: tf.latestEvent, - LatestError: tf.latestError, - }, nil -} - -func (tf *transactionFlow) IsComplete() bool { +func (tf *transactionFlow) IsComplete(_ context.Context) bool { return tf.complete } -func (tf *transactionFlow) ReadyForSequencing() bool { - return tf.readyForSequencing +func (tf *transactionFlow) ReadyForSequencing(ctx context.Context) bool { + return tf.transaction.PostAssembly != nil } -func (tf *transactionFlow) Dispatched() bool { +func (tf *transactionFlow) Dispatched(_ context.Context) bool { return tf.dispatched } @@ -110,7 +142,7 @@ func (tf *transactionFlow) IsEndorsed(ctx context.Context) bool { return !tf.hasOutstandingEndorsementRequests(ctx) } -func (tf *transactionFlow) CoordinatingLocally() bool { +func (tf *transactionFlow) CoordinatingLocally(_ context.Context) bool { return tf.localCoordinator } @@ -122,7 +154,7 @@ func (tf *transactionFlow) PrepareTransaction(ctx context.Context, defaultSigner } readTX := tf.components.Persistence().DB() // no DB transaction required here - prepError := tf.domainAPI.PrepareTransaction(tf.endorsementGatherer.DomainContext(), readTX, tf.transaction) + prepError := tf.domainAPI.PrepareTransaction(tf.domainContext, readTX, tf.transaction) if prepError != nil { log.L(ctx).Errorf("Error preparing transaction: %s", prepError) tf.latestError = i18n.ExpandWithCode(ctx, i18n.MessageKey(msgs.MsgPrivateTxManagerPrepareError), prepError.Error()) @@ -147,7 +179,7 @@ func (tf *transactionFlow) GetStateDistributions(ctx context.Context) (*componen return newStateDistributionBuilder(tf.components, tf.transaction).Build(ctx) } -func (tf *transactionFlow) InputStateIDs() []string { +func (tf *transactionFlow) InputStateIDs(_ context.Context) []string { inputStateIDs := make([]string, len(tf.transaction.PostAssembly.InputStates)) for i, inputState := range tf.transaction.PostAssembly.InputStates { @@ -156,7 +188,7 @@ func (tf *transactionFlow) InputStateIDs() []string { return inputStateIDs } -func (tf *transactionFlow) OutputStateIDs() []string { +func (tf *transactionFlow) OutputStateIDs(_ context.Context) []string { //We use the output states here not the OutputStatesPotential because it is not possible for another transaction // to spend a state unless it has been written to the state store and at that point we have the state ID @@ -167,12 +199,12 @@ func (tf *transactionFlow) OutputStateIDs() []string { return outputStateIDs } -func (tf *transactionFlow) Signer() string { +func (tf *transactionFlow) Signer(_ context.Context) string { return tf.transaction.Signer } -func (tf *transactionFlow) ID() uuid.UUID { +func (tf *transactionFlow) ID(_ context.Context) uuid.UUID { return tf.transaction.ID } diff --git a/core/go/internal/privatetxnmgr/transaction_flow_actions.go b/core/go/internal/privatetxnmgr/transaction_flow_actions.go index 382913a81..36d36fc7e 100644 --- a/core/go/internal/privatetxnmgr/transaction_flow_actions.go +++ b/core/go/internal/privatetxnmgr/transaction_flow_actions.go @@ -29,10 +29,38 @@ import ( "github.com/kaleido-io/paladin/toolkit/pkg/tktypes" ) +func (tf *transactionFlow) logActionDebug(ctx context.Context, msg string) { + //centralize the debug logging to force a consistent format and make it easier to analyze the logs + log.L(ctx).Debugf("transactionFlow:Action TransactionID='%s' Status='%s' LatestEvent='%s' LatestError='%s' : %s", tf.transaction.ID, tf.status, tf.latestEvent, tf.latestError, msg) +} + +func (tf *transactionFlow) logActionDebugf(ctx context.Context, msg string, args ...interface{}) { + //centralize the debug logging to force a consistent format and make it easier to analyze the logs + log.L(ctx).Debugf("transactionFlow:Action TransactionID='%s' Status='%s' LatestEvent='%s' LatestError='%s' : %s", tf.transaction.ID, tf.status, tf.latestEvent, tf.latestError, fmt.Sprintf(msg, args...)) +} + +func (tf *transactionFlow) logActionInfo(ctx context.Context, msg string) { + //centralize the info logging to force a consistent format and make it easier to analyze the logs + log.L(ctx).Infof("transactionFlow:Action TransactionID='%s' Status='%s' LatestEvent='%s' LatestError='%s' : %s", tf.transaction.ID, tf.status, tf.latestEvent, tf.latestError, msg) +} + +func (tf *transactionFlow) logActionInfof(ctx context.Context, msg string, args ...interface{}) { + //centralize the debug logging to force a consistent format and make it easier to analyze the logs + log.L(ctx).Infof("transactionFlow:Action TransactionID='%s' Status='%s' LatestEvent='%s' LatestError='%s' : %s", tf.transaction.ID, tf.status, tf.latestEvent, tf.latestError, fmt.Sprintf(msg, args...)) +} + +func (tf *transactionFlow) logActionError(ctx context.Context, msg string, err error) { + //centralize the info logging to force a consistent format and make it easier to analyze the logs + log.L(ctx).Errorf("transactionFlow:Action TransactionID='%s' Status='%s' LatestEvent='%s' LatestError='%s' : %s. %s", tf.transaction.ID, tf.status, tf.latestEvent, tf.latestError, msg, err.Error()) +} + func (tf *transactionFlow) Action(ctx context.Context) { - log.L(ctx).Debug("transactionFlow:Action") + tf.statusLock.Lock() + defer tf.statusLock.Unlock() + + tf.logActionDebug(ctx, ">>") if tf.complete { - log.L(ctx).Infof("Transaction %s is complete", tf.transaction.ID.String()) + tf.logActionInfo(ctx, "Transaction is complete") return } @@ -40,7 +68,7 @@ func (tf *transactionFlow) Action(ctx context.Context) { // if the event handler has marked the transaction as failed, then we initiate the finalize sync point if tf.finalizeRequired { if tf.finalizePending { - log.L(ctx).Infof("Transaction %s finalize already pending", tf.transaction.ID.String()) + tf.logActionInfo(ctx, "finalize already pending") return } //we know we need to finalize but we are not currently waiting for a finalize to complete @@ -49,30 +77,37 @@ func (tf *transactionFlow) Action(ctx context.Context) { } if tf.dispatched { - log.L(ctx).Infof("Transaction %s is dispatched", tf.transaction.ID.String()) + tf.logActionInfo(ctx, "Transaction is dispatched") return } if tf.transaction.PreAssembly == nil { + tf.logActionDebug(ctx, "PreAssembly is nil") panic("PreAssembly is nil.") //This should never happen unless there is a serious programming error or the memory has been corrupted // PreAssembly is checked for nil after InitTransaction which is during the synchronous transaction request // and before it is added to the transaction processor / dispatched to the event loop } + //depending on the delegation policy, we may be able to decide to delegate before we have assembled the transaction + if !tf.delegateIfRequired(ctx) { + tf.logActionDebug(ctx, "no continue after pre assembly delegate check") + return + } + if tf.transaction.PostAssembly == nil { - log.L(ctx).Debug("not assembled yet - or was assembled and reverted") + tf.logActionDebug(ctx, "PostAssembly is nil") - //if we have not sent a request, or if the request has timed out or been invalided by a re-assembly, then send the request + //if we have not sent a request, or if the request has timed out or been invalidated by a re-assembly, then send the request tf.requestVerifierResolution(ctx) if tf.hasOutstandingVerifierRequests(ctx) { - log.L(ctx).Infof("Transaction %s not ready to assemble. Waiting for verifiers to be resolved", tf.transaction.ID.String()) + tf.logActionInfo(ctx, "Transaction not ready to assemble. Waiting for verifiers to be resolved") return } tf.requestAssemble(ctx) if tf.transaction.PostAssembly == nil { - log.L(ctx).Infof("Transaction %s not assembled. Waiting for assembler to return", tf.transaction.ID.String()) + tf.logActionInfo(ctx, "Transaction not assembled. Waiting for assembler to return") return } if tf.transaction.PostAssembly.AssemblyResult == prototk.AssembleTransactionResponse_REVERT { @@ -88,38 +123,14 @@ func (tf *transactionFlow) Action(ctx context.Context) { } tf.status = "signed" - tf.delegateIfRequired(ctx) - if tf.status == "delegating" { - log.L(ctx).Infof("Transaction %s is delegating", tf.transaction.ID.String()) + //depending on the delegation policy, we may not be able to decide to delegate until after we have assembled the transaction + if !tf.delegateIfRequired(ctx) { + tf.logActionDebug(ctx, "no continue after post assembly delegate check") return } - if tf.status == "delegated" { - // probably should not get here because the sequencer should have removed the transaction processor - log.L(ctx).Infof("Transaction %s has been delegated", tf.transaction.ID.String()) - return - } - - if tf.transaction.PostAssembly.OutputStatesPotential != nil && tf.transaction.PostAssembly.OutputStates == nil { - // We need to write the potential states to the domain before we can sign or endorse the transaction - // but there is no point in doing that until we are sure that the transaction is going to be coordinated locally - // so this is the earliest, and latest, point in the flow that we can do this - readTX := tf.components.Persistence().DB() // no DB transaction required here for the reads from the DB (writes happen on syncpoint flusher) - err := tf.domainAPI.WritePotentialStates(tf.endorsementGatherer.DomainContext(), readTX, tf.transaction) - if err != nil { - //Any error from WritePotentialStates is likely to be caused by an invalid init or assemble of the transaction - // which is most likely a programming error in the domain or the domain manager or privateTxManager - // not much we can do other than revert the transaction with an internal error - errorMessage := fmt.Sprintf("Failed to write potential states: %s", err) - log.L(ctx).Error(errorMessage) - //TODO publish an event that will cause the transaction to be reverted - //tf.revertTransaction(ctx, i18n.ExpandWithCode(ctx, i18n.MessageKey(msgs.MsgPrivateTxManagerInternalError), errorMessage)) - return - } - } - log.L(ctx).Debugf("Transaction %s is ready (outputStatesPotential=%d outputStates=%d)", + log.L(ctx).Debugf("transactionFlow:Action TransactionID='%s' is ready for sequencing (outputStatesPotential=%d outputStates=%d)", tf.transaction.ID.String(), len(tf.transaction.PostAssembly.OutputStatesPotential), len(tf.transaction.PostAssembly.OutputStates)) - tf.readyForSequencing = true //If we get here, we have an assembled transaction and have no intention of delegating it // so we are responsible for coordinating the endorsement flow @@ -127,6 +138,7 @@ func (tf *transactionFlow) Action(ctx context.Context) { tf.requestEndorsements(ctx) if tf.hasOutstandingEndorsementRequests(ctx) { + tf.logActionDebug(ctx, "Transaction not ready to dispatch. Waiting for endorsements to be resolved") return } tf.status = "endorsed" @@ -146,6 +158,8 @@ func (tf *transactionFlow) Action(ctx context.Context) { // TODO: We should re-delegate in this scenario tf.latestError = i18n.NewError(ctx, msgs.MsgPrivateReDelegationRequired).Error() } + tf.logActionDebug(ctx, "<<") + } func (tf *transactionFlow) setTransactionSigner(ctx context.Context) (reDelegate bool, err error) { @@ -212,7 +226,7 @@ func (tf *transactionFlow) setTransactionSigner(ctx context.Context) (reDelegate return false, i18n.WrapError(ctx, err, msgs.MsgDomainEndorserSubmitConfigClash, endorserSubmitSigner, contractConf.CoordinatorSelection, contractConf.SubmitterSelection) } - if node != tf.nodeID { + if node != tf.nodeName { log.L(ctx).Warnf("For transaction %s to be submitted, the coordinator must move to the node ENDORSER_MUST_SUBMIT constraint %s", tx.ID, endorserSubmitSigner) return true, nil @@ -234,7 +248,12 @@ func (tf *transactionFlow) revertTransaction(ctx context.Context, revertReason s } func (tf *transactionFlow) finalize(ctx context.Context) { - log.L(ctx).Errorf("finalize transaction %s: %s", tf.transaction.ID.String(), tf.finalizeRevertReason) + if tf.finalizeRevertReason == "" { + log.L(ctx).Debugf("finalize transaction %s", tf.transaction.ID.String()) + + } else { + log.L(ctx).Errorf("finalize transaction %s: %s", tf.transaction.ID.String(), tf.finalizeRevertReason) + } //flush that to the txmgr database // so that the user can see that it is reverted and so that we stop retrying to assemble and endorse it @@ -250,7 +269,7 @@ func (tf *transactionFlow) finalize(ctx context.Context) { log.L(ctx).Infof("Transaction %s finalize committed", tf.transaction.ID.String()) // Remove this transaction from our domain context on success - all changes are flushed to DB at this point - tf.endorsementGatherer.DomainContext().ResetTransactions(tf.transaction.ID) + tf.domainContext.ResetTransactions(tf.transaction.ID) go tf.publisher.PublishTransactionFinalizedEvent(ctx, tf.transaction.ID.String()) }, @@ -260,88 +279,147 @@ func (tf *transactionFlow) finalize(ctx context.Context) { log.L(ctx).Errorf("Transaction %s finalize rolled back: %s", tf.transaction.ID.String(), rollbackErr) // Reset the whole domain context on failure - tf.endorsementGatherer.DomainContext().Reset() + tf.domainContext.Reset() go tf.publisher.PublishTransactionFinalizeError(ctx, tf.transaction.ID.String(), tf.finalizeRevertReason, rollbackErr) }, ) } -func (tf *transactionFlow) delegateIfRequired(ctx context.Context) { - log.L(ctx).Debug("transactionFlow:delegateIfRequired") - contractConfig := tf.domainAPI.ContractConfig() - - // Calculate if we know a coordinator that must be the correct node - var knownCoordinator = "" - if contractConfig.CoordinatorSelection == prototk.ContractConfig_COORDINATOR_STATIC { +func (tf *transactionFlow) delegateIfRequired(ctx context.Context) (doContinue bool) { - // Simple decision here. The static coordinator is where the coordinator is located - if contractConfig.StaticCoordinator != nil { - knownCoordinator = *contractConfig.StaticCoordinator + if tf.delegatePending { + tf.logActionInfof(ctx, "Transaction is delegating since %s (block=%d)", tf.delegateRequestTime, tf.delegateRequestBlockHeight) + if tf.clock.Now().Before(tf.delegateRequestTime.Add(tf.requestTimeout)) { + tf.logActionDebug(ctx, "Delegation request not timed out") + return false } + tf.logActionDebug(ctx, "Delegation request timed out") - } else if tf.transaction.PostAssembly.AttestationPlan != nil { + } - numEndorsers := 0 - endorser := "" // will only be used if there is only one - for _, attRequest := range tf.transaction.PostAssembly.AttestationPlan { - if attRequest.AttestationType == prototk.AttestationType_ENDORSE { - numEndorsers = numEndorsers + len(attRequest.Parties) - endorser = attRequest.Parties[0] - } - } - //in the special case of a single endorsers in a domain with a submit mode of ENDORSER_SUBMISSION we delegate to that endorser - // It is most likely that this means we are in the noto domain and - // that single endorser is the notary and all transactions will be delegated there for endorsement - // and dispatch to base ledger so we might as well delegate the coordination to it so that - // it can maximize the optimistic spending of pending states - - if contractConfig.CoordinatorSelection == prototk.ContractConfig_COORDINATOR_ENDORSER && numEndorsers == 1 { - knownCoordinator = endorser - } + if tf.status == "delegated" { + // probably should not get here because the sequencer should have removed the transaction processor + tf.logActionInfo(ctx, "Transaction is delegated") + return false + } + // There may be a potential optimization we can add where, in certain domain configurations, we can optimistically proceed without delegation and only delegate once we detect + // potential contention with other active nodes. For now, we keep it simple and strictly abide by the configuration of the domain + blockHeight, coordinatorNode, err := tf.selectCoordinator.SelectCoordinatorNode(ctx, tf.transaction, tf.environment) + if err != nil { + // errors from here are most likely a problem resolving the node name from the parties in the attestation plan + // so there is no point retrying although if we redo the assemble stage, we may get a different result + //TODO should we make the error action ( revert, reassemble, retry) an explicit property of the error so that this assessment can be made closer + // to the source of the error? + tf.latestError = err.Error() + tf.transaction.PostAssembly = nil + tf.logActionError(ctx, "Failed to select coordinator node", err) + return false + } + + // TODO persist the delegation and send the request on the callback + if coordinatorNode == tf.nodeName || coordinatorNode == "" { + // we are the coordinator so we should continue + tf.logActionDebug(ctx, "Local coordinator") + return true + } + tf.localCoordinator = false + + //TODO if already `delegating` check how long we have been waiting for the ack and send again. + //Should probably do that earlier in the flow because if we have just decided not to delegate or if we have just selected a different delegate, \ + //then we need to either claw back that delegation or wait until the delegate has realized that they are no longer the coordinator and returns / forwards the responsibility for this transaction + tf.status = "delegating" + // ensure that the From field is fully qualified before sending to the delegate so that they can send assemble requests to the correct place + fullQualifiedFrom, err := tktypes.PrivateIdentityLocator(tf.transaction.Inputs.From).FullyQualified(ctx, tf.nodeName) + if err != nil { + //this can only mean that the From field is an invalid identity locator and we should never have got this far + // unless there is a serious programming error or the memory has been corrupted + panic("Failed to fully qualify the coordinator node") } + tf.transaction.Inputs.From = fullQualifiedFrom.String() + + delegationRequestID := uuid.New().String() + + err = tf.transportWriter.SendDelegationRequest( + ctx, + delegationRequestID, + coordinatorNode, + tf.transaction, + blockHeight, + ) + if err != nil { + tf.latestError = i18n.ExpandWithCode(ctx, i18n.MessageKey(msgs.MsgPrivateTxManagerInternalError), err.Error()) + tf.logActionError(ctx, "Failed to send delegation request", err) + } + tf.pendingDelegationRequestID = delegationRequestID + tf.delegatePending = true + tf.delegateRequestBlockHeight = blockHeight + tf.delegateRequestTime = tf.clock.Now() + tf.delegateRequestTimer = time.AfterFunc(tf.requestTimeout, func() { + tf.publisher.PublishNudgeEvent(ctx, tf.transaction.ID.String()) + }) + //we have initiated a delegation so we should not continue any further with the flow on this node + tf.logActionInfo(ctx, fmt.Sprintf("Delegating transaction to %s", coordinatorNode)) + return false - // If we have one, check we're on that node - if knownCoordinator != "" { - coordinatorNode, err := tktypes.PrivateIdentityLocator(knownCoordinator).Node(ctx, true) +} + +func (tf *transactionFlow) writeAndLockStates(ctx context.Context) { + //this needs to be carefully coordinated with the assemble requester thread and the sequencer event loop thread + // we are accessing the transactionFlow's PrivateTransaction object which is only safe to do on the sequencer thread + // but we need to make sure that these writes/locks are complete before the assemble requester thread can proceed to assemble + // the next transaction + if tf.transaction.PostAssembly.OutputStatesPotential != nil && tf.transaction.PostAssembly.OutputStates == nil { + // We need to write the potential states to the domain before we can sign or endorse the transaction + // but there is no point in doing that until we are sure that the transaction is going to be coordinated locally + // so this is the earliest, and latest, point in the flow that we can do this + readTX := tf.components.Persistence().DB() // no DB transaction required here for the reads from the DB (writes happen on syncpoint flusher) + err := tf.domainAPI.WritePotentialStates(tf.domainContext, readTX, tf.transaction) if err != nil { - log.L(ctx).Errorf("Failed to get node name from locator %s: %s", knownCoordinator, err) - tf.latestError = i18n.ExpandWithCode(ctx, i18n.MessageKey(msgs.MsgPrivateTxManagerInternalError), err.Error()) + //Any error from WritePotentialStates is likely to be caused by an invalid init or assemble of the transaction + // which is most likely a programming error in the domain or the domain manager or privateTxManager + // not much we can do other than revert the transaction with an internal error + errorMessage := fmt.Sprintf("Failed to write potential states: %s", err) + log.L(ctx).Error(errorMessage) + //TODO publish an event that will cause the transaction to be reverted + tf.revertTransaction(ctx, i18n.ExpandWithCode(ctx, i18n.MessageKey(msgs.MsgPrivateTxManagerInternalError), errorMessage)) return + } else { + tf.logActionDebugf(ctx, "Potential states written %s", tf.domainContext.Info().ID) } - if coordinatorNode != tf.nodeID && coordinatorNode != "" { - tf.localCoordinator = false - // TODO persist the delegation and send the request on the callback - tf.status = "delegating" - // TODO update to "delegated" once the ack has been received - err := tf.transportWriter.SendDelegationRequest( - ctx, - uuid.New().String(), - coordinatorNode, - tf.transaction, - ) - if err != nil { - tf.latestError = i18n.ExpandWithCode(ctx, i18n.MessageKey(msgs.MsgPrivateTxManagerInternalError), err.Error()) - } + } + if len(tf.transaction.PostAssembly.InputStates) > 0 { + readTX := tf.components.Persistence().DB() // no DB transaction required here for the reads from the DB (writes happen on syncpoint flusher) + + err := tf.domainAPI.LockStates(tf.domainContext, readTX, tf.transaction) + if err != nil { + errorMessage := fmt.Sprintf("Failed to lock states: %s", err) + log.L(ctx).Error(errorMessage) + tf.revertTransaction(ctx, i18n.ExpandWithCode(ctx, i18n.MessageKey(msgs.MsgPrivateTxManagerInternalError), errorMessage)) return + } else { + tf.logActionDebugf(ctx, "Input states locked %s: %s", tf.domainContext.Info().ID, tf.transaction.PostAssembly.InputStates[0].ID) } } - } - func (tf *transactionFlow) requestAssemble(ctx context.Context) { //Assemble may require a call to another node ( in the case we have been delegated to coordinate transaction for other nodes) //Usually, they will get sent to us already assembled but there may be cases where we need to re-assemble // so this needs to be an async step // however, there must be only one assemble in progress at a time or else there is a risk that 2 transactions could chose to spend the same state - // (TODO - maybe in future, we could further optimise this and allow multiple assembles to be in progress if we can assert that they are not presented with the same available states) + // (TODO - maybe in future, we could further optimize this and allow multiple assembles to be in progress if we can assert that they are not presented with the same available states) // However, before we do that, we really need to sort out the separation of concerns between the domain manager, state store and private transaction manager and where the responsibility to single thread the assembly stream(s) lies log.L(ctx).Debug("transactionFlow:requestAssemble") if tf.transaction.PostAssembly != nil { - log.L(ctx).Debug("already assembled") + tf.logActionDebug(ctx, "Already assembled") + return + } + + if tf.assemblePending { + tf.logActionDebug(ctx, "Assemble already pending") return } @@ -353,70 +431,24 @@ func (tf *transactionFlow) requestAssemble(ctx context.Context) { ctx, tf.transaction.ID.String(), i18n.ExpandWithCode(ctx, i18n.MessageKey(msgs.MsgPrivateTxManagerInternalError), "Failed to get node name from locator"), + "", ) return } - if assemblingNode == tf.nodeID || assemblingNode == "" { - //we are the node that is responsible for assembling this transaction - readTX := tf.components.Persistence().DB() // no DB transaction required here - err = tf.domainAPI.AssembleTransaction(tf.endorsementGatherer.DomainContext(), readTX, tf.transaction) - if err != nil { - log.L(ctx).Errorf("AssembleTransaction failed: %s", err) - tf.publisher.PublishTransactionAssembleFailedEvent(ctx, - tf.transaction.ID.String(), - i18n.ExpandWithCode(ctx, i18n.MessageKey(msgs.MsgPrivateTxManagerAssembleError), err.Error()), - ) - return - } - if tf.transaction.PostAssembly == nil { - // This is most likely a programming error in the domain - log.L(ctx).Errorf("PostAssembly is nil.") - tf.publisher.PublishTransactionAssembleFailedEvent( - ctx, - tf.transaction.ID.String(), - i18n.ExpandWithCode(ctx, i18n.MessageKey(msgs.MsgPrivateTxManagerInternalError), "AssembleTransaction returned nil PostAssembly"), - ) - return - } - - // Some validation that we are confident we can execute the given attestation plan - for _, attRequest := range tf.transaction.PostAssembly.AttestationPlan { - switch attRequest.AttestationType { - case prototk.AttestationType_ENDORSE: - case prototk.AttestationType_SIGN: - case prototk.AttestationType_GENERATE_PROOF: - errorMessage := "AttestationType_GENERATE_PROOF is not implemented yet" - log.L(ctx).Error(errorMessage) - tf.latestError = i18n.ExpandWithCode(ctx, i18n.MessageKey(msgs.MsgPrivateTxManagerInternalError), errorMessage) - tf.publisher.PublishTransactionAssembleFailedEvent(ctx, - tf.transaction.ID.String(), - i18n.ExpandWithCode(ctx, i18n.MessageKey(msgs.MsgPrivateTxManagerAssembleError), errorMessage), - ) - default: - errorMessage := fmt.Sprintf("Unsupported attestation type: %s", attRequest.AttestationType) - log.L(ctx).Error(errorMessage) - tf.latestError = i18n.ExpandWithCode(ctx, i18n.MessageKey(msgs.MsgPrivateTxManagerInternalError), errorMessage) - tf.publisher.PublishTransactionAssembleFailedEvent(ctx, - tf.transaction.ID.String(), - i18n.ExpandWithCode(ctx, i18n.MessageKey(msgs.MsgPrivateTxManagerAssembleError), errorMessage), - ) - } - } + //TODO is this safe or do we need a deep copy? + transactionInputsCopy := *tf.transaction.Inputs + preAssemblyCopy := *tf.transaction.PreAssembly - //TODO should probably include the assemble output in the event - // for now that is not necessary because this is a local assemble and the domain manager updates the transaction that we passed by reference - // need to decide if we want to continue with that style of interface to the domain manager and if so, - // we need to do something different when the assembling node is remote - tf.publisher.PublishTransactionAssembledEvent(ctx, - tf.transaction.ID.String(), - ) - return + tf.assembleCoordinator.QueueAssemble( + ctx, + assemblingNode, + tf.transaction.ID, + &transactionInputsCopy, + &preAssemblyCopy, + ) + tf.assemblePending = true - } else { - log.L(ctx).Debugf("Assembling transaction %s on node %s", tf.transaction.ID.String(), assemblingNode) - //TODO send a request to the node that is responsible for assembling this transaction - } } func (tf *transactionFlow) requestSignature(ctx context.Context, attRequest *prototk.AttestationRequest, partyName string) { @@ -425,7 +457,7 @@ func (tf *transactionFlow) requestSignature(ctx context.Context, attRequest *pro unqualifiedLookup := partyName signerNode, err := tktypes.PrivateIdentityLocator(partyName).Node(ctx, true) - if signerNode != "" && signerNode != tf.nodeID { + if signerNode != "" && signerNode != tf.nodeName { log.L(ctx).Debugf("Requesting signature from a remote identity %s for %s", partyName, attRequest.Name) err = i18n.NewError(ctx, msgs.MsgPrivateTxManagerSignRemoteError, partyName) } @@ -499,7 +531,7 @@ func (tf *transactionFlow) requestSignatures(ctx context.Context) { tf.requestedSignatures = true } -func (tf *transactionFlow) requestEndorsement(ctx context.Context, party string, attRequest *prototk.AttestationRequest) { +func (tf *transactionFlow) requestEndorsement(ctx context.Context, idempotencyKey string, party string, attRequest *prototk.AttestationRequest) { partyLocator := tktypes.PrivateIdentityLocator(party) partyNode, err := partyLocator.Node(ctx, true) @@ -509,7 +541,7 @@ func (tf *transactionFlow) requestEndorsement(ctx context.Context, party string, return } - if partyNode == tf.nodeID || partyNode == "" { + if partyNode == tf.nodeName || partyNode == "" { // This is a local party, so we can endorse it directly endorsement, revertReason, err := tf.endorsementGatherer.GatherEndorsement( ctx, @@ -530,6 +562,9 @@ func (tf *transactionFlow) requestEndorsement(ctx context.Context, party string, } tf.publisher.PublishTransactionEndorsedEvent(ctx, tf.transaction.ID.String(), + idempotencyKey, + party, + attRequest.Name, endorsement, revertReason, ) @@ -539,6 +574,7 @@ func (tf *transactionFlow) requestEndorsement(ctx context.Context, party string, err = tf.transportWriter.SendEndorsementRequest( ctx, + idempotencyKey, party, partyNode, tf.transaction.Inputs.To.String(), @@ -563,12 +599,15 @@ func (tf *transactionFlow) requestEndorsements(ctx context.Context) { // there is a request in the attestation plan and we do not have a response to match it // first lets see if we have recently sent a request for this endorsement and just need to be patient previousRequestTime := time.Time{} - if timesForAttRequest, ok := tf.requestedEndorsementTimes[outstandingEndorsementRequest.attRequest.Name]; ok { - if t, ok := timesForAttRequest[outstandingEndorsementRequest.party]; ok { - previousRequestTime = t + idempotencyKey := uuid.New().String() + previousIdempotencyKey := "" + if pendingRequestsForAttRequest, ok := tf.pendingEndorsementRequests[outstandingEndorsementRequest.attRequest.Name]; ok { + if r, ok := pendingRequestsForAttRequest[outstandingEndorsementRequest.party]; ok { + previousRequestTime = r.requestTime + previousIdempotencyKey = r.idempotencyKey } } else { - tf.requestedEndorsementTimes[outstandingEndorsementRequest.attRequest.Name] = make(map[string]time.Time) + tf.pendingEndorsementRequests[outstandingEndorsementRequest.attRequest.Name] = make(map[string]*endorsementRequest) } if !previousRequestTime.IsZero() && tf.clock.Now().Before(previousRequestTime.Add(tf.requestTimeout)) { @@ -581,8 +620,17 @@ func (tf *transactionFlow) requestEndorsements(ctx context.Context) { } else { log.L(ctx).Infof("Previous endorsement request for transaction:%s, attestation request:%s, party:%s sent at %v has timed out", tf.transaction.ID.String(), outstandingEndorsementRequest.attRequest.Name, outstandingEndorsementRequest.party, previousRequestTime) } - tf.requestEndorsement(ctx, outstandingEndorsementRequest.party, outstandingEndorsementRequest.attRequest) - tf.requestedEndorsementTimes[outstandingEndorsementRequest.attRequest.Name][outstandingEndorsementRequest.party] = tf.clock.Now() + if previousIdempotencyKey != "" { + tf.logActionDebug(ctx, fmt.Sprintf("Previous endorsement request timed out. Sending new request with same idempotency key %s", previousIdempotencyKey)) + idempotencyKey = previousIdempotencyKey + } + + tf.requestEndorsement(ctx, idempotencyKey, outstandingEndorsementRequest.party, outstandingEndorsementRequest.attRequest) + tf.pendingEndorsementRequests[outstandingEndorsementRequest.attRequest.Name][outstandingEndorsementRequest.party] = + &endorsementRequest{ + requestTime: tf.clock.Now(), + idempotencyKey: idempotencyKey, + } } } @@ -599,6 +647,7 @@ func (tf *transactionFlow) requestVerifierResolution(ctx context.Context) { tf.transaction.PreAssembly.Verifiers = make([]*prototk.ResolvedVerifier, 0, len(tf.transaction.PreAssembly.RequiredVerifiers)) } for _, v := range tf.transaction.PreAssembly.RequiredVerifiers { + tf.logActionDebugf(ctx, "Resolving verifier %s", v.Lookup) tf.identityResolver.ResolveVerifierAsync( ctx, v.Lookup, diff --git a/core/go/internal/privatetxnmgr/transaction_flow_mutators.go b/core/go/internal/privatetxnmgr/transaction_flow_mutators.go index 730fd37cf..9357c6990 100644 --- a/core/go/internal/privatetxnmgr/transaction_flow_mutators.go +++ b/core/go/internal/privatetxnmgr/transaction_flow_mutators.go @@ -26,6 +26,8 @@ import ( ) func (tf *transactionFlow) ApplyEvent(ctx context.Context, event ptmgrtypes.PrivateTransactionEvent) { + tf.statusLock.Lock() + defer tf.statusLock.Unlock() //First we update our in memory record of the transaction with the data from the event switch event := event.(type) { @@ -43,12 +45,14 @@ func (tf *transactionFlow) ApplyEvent(ctx context.Context, event ptmgrtypes.Priv tf.applyTransactionAssembleFailedEvent(ctx, event) case *ptmgrtypes.TransactionDispatchedEvent: tf.applyTransactionDispatchedEvent(ctx, event) + case *ptmgrtypes.TransactionPreparedEvent: + tf.applyTransactionPreparedEvent(ctx, event) case *ptmgrtypes.TransactionConfirmedEvent: tf.applyTransactionConfirmedEvent(ctx, event) case *ptmgrtypes.TransactionRevertedEvent: tf.applyTransactionRevertedEvent(ctx, event) - case *ptmgrtypes.TransactionDelegatedEvent: - tf.applyTransactionDelegatedEvent(ctx, event) + case *ptmgrtypes.TransactionDelegationAcknowledgedEvent: + tf.applyTransactionDelegationAcknowledgedEvent(ctx, event) case *ptmgrtypes.ResolveVerifierResponseEvent: tf.applyResolveVerifierResponseEvent(ctx, event) case *ptmgrtypes.ResolveVerifierErrorEvent: @@ -57,6 +61,10 @@ func (tf *transactionFlow) ApplyEvent(ctx context.Context, event ptmgrtypes.Priv tf.applyTransactionFinalizedEvent(ctx, event) case *ptmgrtypes.TransactionFinalizeError: tf.applyTransactionFinalizeError(ctx, event) + case *ptmgrtypes.TransactionNudgeEvent: + tf.applyTransactionNudgeEvent(ctx, event) + case *ptmgrtypes.DelegationForInFlightEvent: + tf.applyDelegationForInFlightEvent(ctx, event) default: log.L(ctx).Warnf("Unknown event type: %T", event) @@ -77,24 +85,52 @@ func (tf *transactionFlow) applyTransactionSwappedInEvent(ctx context.Context, _ } -func (tf *transactionFlow) applyTransactionAssembledEvent(ctx context.Context, _ *ptmgrtypes.TransactionAssembledEvent) { +func (tf *transactionFlow) applyTransactionAssembledEvent(ctx context.Context, event *ptmgrtypes.TransactionAssembledEvent) { + log.L(ctx).Debug("transactionFlow:applyTransactionAssembledEvent") + tf.latestEvent = "TransactionAssembledEvent" + tf.transaction.PostAssembly = event.PostAssembly + tf.assemblePending = false if tf.transaction.PostAssembly.AssemblyResult == prototk.AssembleTransactionResponse_REVERT { // Not sure if any domains actually use this but it is a valid response to indicate failure log.L(ctx).Errorf("AssemblyResult is AssembleTransactionResponse_REVERT") - tf.revertTransaction(ctx, i18n.ExpandWithCode(ctx, i18n.MessageKey(msgs.MsgPrivateTxManagerAssembleRevert))) + revertReason := "" + if event.PostAssembly.RevertReason != nil { + revertReason = *event.PostAssembly.RevertReason + } + tf.revertTransaction(ctx, i18n.ExpandWithCode(ctx, i18n.MessageKey(msgs.MsgPrivateTxManagerAssembleRevert), revertReason)) + return + } + if tf.transaction.PostAssembly.AssemblyResult == prototk.AssembleTransactionResponse_PARK { + + log.L(ctx).Infof("AssemblyResult is AssembleTransactionResponse_PARK") + tf.status = "parked" + tf.assemblePending = false return } tf.status = "assembled" + tf.writeAndLockStates(ctx) + //allow assembly thread to proceed + sds, err := tf.GetStateDistributions(ctx) + if err != nil { + log.L(ctx).Errorf("Error getting state distributions: %s", err) + // we need to proceed with unblocking the assembleCoordinator. It wont have a chance to distribute the states to the remote assembler nodes + // so they may fail to assemble or may assemble a transaction that does not get endorsed but that is always a possibility anyway and the + // engine's retry strategy and the eventually consistent distribution of states will mean we will eventually process + // all transactions if they are valid + + } + tf.assembleCoordinator.Complete(event.AssembleRequestID, sds.Remote) } func (tf *transactionFlow) applyTransactionAssembleFailedEvent(ctx context.Context, event *ptmgrtypes.TransactionAssembleFailedEvent) { - log.L(ctx).Debugf("transactionFlow:applyTransactionAssembleFailedEvent: %s", event.Error) + log.L(ctx).Debugf("transactionFlow:applyTransactionAssembleFailedEvent: RequestID: '%s' Error: %s ", event.AssembleRequestID, event.Error) tf.latestEvent = "TransactionAssembleFailedEvent" tf.latestError = event.Error - tf.finalizeRequired = true - tf.finalizeRevertReason = event.Error + // set assemblePending to false so that the transaction can be re-assembled + tf.assemblePending = false + tf.assembleCoordinator.Complete(event.AssembleRequestID, nil) } func (tf *transactionFlow) applyTransactionSignedEvent(ctx context.Context, event *ptmgrtypes.TransactionSignedEvent) { @@ -106,6 +142,27 @@ func (tf *transactionFlow) applyTransactionSignedEvent(ctx context.Context, even func (tf *transactionFlow) applyTransactionEndorsedEvent(ctx context.Context, event *ptmgrtypes.TransactionEndorsedEvent) { tf.latestEvent = "TransactionEndorsedEvent" + log.L(ctx).Debugf("transactionFlow:applyTransactionEndorsedEvent: TransactionID: '%s' IdempotencyKey: '%s' Party: %s ", event.TransactionID, event.IdempotencyKey, event.Party) + + //if this response does not match a pending request, then we ignore it + pendingRequestsForAttRequestName, ok := tf.pendingEndorsementRequests[event.AttestationRequestName] + if !ok { + log.L(ctx).Warnf("Received endorsement for unknown request name %s", event.AttestationRequestName) + return + } + pendingRequest, ok := pendingRequestsForAttRequestName[event.Party] + if !ok { + log.L(ctx).Warnf("Received endorsement for unknown party %s", event.Party) + return + } + + if pendingRequest.idempotencyKey != event.IdempotencyKey { + log.L(ctx).Debugf("Pending request idempotencyKey %s does not match endorsement idempotencyKey %s, assuming response from obsolete request", pendingRequest.idempotencyKey, event.IdempotencyKey) + return + } + //we have (had) a pending request for this endorsement but it is no longer pending because we now have a response + delete(pendingRequestsForAttRequestName, event.Party) + if event.RevertReason != nil { log.L(ctx).Infof("Endorsement for transaction %s was rejected: %s", tf.transaction.ID.String(), *event.RevertReason) // endorsement errors trigger a re-assemble @@ -116,6 +173,8 @@ func (tf *transactionFlow) applyTransactionEndorsedEvent(ctx context.Context, ev // we discard them when they do return. //only apply at this stage, action will be taken later tf.transaction.PostAssembly = nil + // remove all pending endorsement request records because they are no longer valid + tf.pendingEndorsementRequests = make(map[string]map[string]*endorsementRequest) } else { log.L(ctx).Infof("Adding endorsement from %s to transaction %s", event.Endorsement.Verifier.Lookup, tf.transaction.ID.String()) @@ -131,6 +190,13 @@ func (tf *transactionFlow) applyTransactionDispatchedEvent(ctx context.Context, tf.dispatched = true } +func (tf *transactionFlow) applyTransactionPreparedEvent(ctx context.Context, _ *ptmgrtypes.TransactionPreparedEvent) { + log.L(ctx).Debugf("transactionFlow:applyTransactionPreparedEvent transactionID:%s ", tf.transaction.ID.String()) + tf.latestEvent = "TransactionPreparedEvent" + tf.status = "prepared" + tf.prepared = true +} + func (tf *transactionFlow) applyTransactionConfirmedEvent(ctx context.Context, event *ptmgrtypes.TransactionConfirmedEvent) { log.L(ctx).Debugf("transactionFlow:applyTransactionConfirmedEvent transactionID:%s contractAddress: %s", tf.transaction.ID.String(), event.ContractAddress) tf.latestEvent = "TransactionConfirmedEvent" @@ -144,10 +210,20 @@ func (tf *transactionFlow) applyTransactionRevertedEvent(ctx context.Context, _ tf.status = "reverted" } -func (tf *transactionFlow) applyTransactionDelegatedEvent(ctx context.Context, _ *ptmgrtypes.TransactionDelegatedEvent) { - log.L(ctx).Debugf("transactionFlow:applyTransactionDelegatedEvent transactionID:%s", tf.transaction.ID.String()) - tf.latestEvent = "TransactionDelegatedEvent" +func (tf *transactionFlow) applyTransactionDelegationAcknowledgedEvent(ctx context.Context, event *ptmgrtypes.TransactionDelegationAcknowledgedEvent) { + log.L(ctx).Debugf("transactionFlow:applyTransactionDelegationAcknowledgedEvent transactionID:%s", tf.transaction.ID.String()) + if event.DelegationRequestID != tf.pendingDelegationRequestID { + log.L(ctx).Warnf("Received delegation acknowledgment for unknown request %s", event.DelegationRequestID) + return + } + tf.latestEvent = "TransactionDelegationAcknowledgedEvent" tf.status = "delegated" + tf.delegated = true + tf.delegatePending = false + if tf.delegateRequestTimer != nil { + tf.delegateRequestTimer.Stop() + } + tf.delegateRequestTimer = nil } func (tf *transactionFlow) applyResolveVerifierResponseEvent(ctx context.Context, event *ptmgrtypes.ResolveVerifierResponseEvent) { @@ -191,3 +267,27 @@ func (tf *transactionFlow) applyTransactionFinalizeError(ctx context.Context, ev tf.finalizeRequired = true tf.finalizePending = false } + +func (tf *transactionFlow) applyTransactionNudgeEvent(ctx context.Context, _ *ptmgrtypes.TransactionNudgeEvent) { + log.L(ctx).Infof("applyTransactionNudgeEvent transaction %s", tf.transaction.ID) + tf.latestEvent = "TransactionNudgeEvent" + + //nothing really changes here, we just trigger a re-evaluation of the transaction state and next actions + +} + +func (tf *transactionFlow) applyDelegationForInFlightEvent(ctx context.Context, event *ptmgrtypes.DelegationForInFlightEvent) { + log.L(ctx).Infof("applyDelegationForInFlightEvent transaction %s blockHeight=%d", tf.transaction.ID, event.BlockHeight) + tf.latestEvent = "DelegationForInFlightEvent" + if tf.delegatePending && event.BlockHeight > tf.delegateRequestBlockHeight { + tf.status = "new" + tf.delegated = false + tf.delegatePending = false + if tf.delegateRequestTimer != nil { + tf.delegateRequestTimer.Stop() + } + tf.delegateRequestTimer = nil + log.L(ctx).Infof("delegation cancelled for transaction %s", tf.transaction.ID) + } + +} diff --git a/core/go/internal/privatetxnmgr/transaction_flow_status.go b/core/go/internal/privatetxnmgr/transaction_flow_status.go index 62464bff4..a665a25b1 100644 --- a/core/go/internal/privatetxnmgr/transaction_flow_status.go +++ b/core/go/internal/privatetxnmgr/transaction_flow_status.go @@ -17,11 +17,41 @@ package privatetxnmgr import ( "context" + "time" + "github.com/kaleido-io/paladin/core/internal/components" "github.com/kaleido-io/paladin/toolkit/pkg/log" "github.com/kaleido-io/paladin/toolkit/pkg/prototk" ) +func (tf *transactionFlow) GetTxStatus(ctx context.Context) (components.PrivateTxStatus, error) { + tf.statusLock.RLock() + defer tf.statusLock.RUnlock() + endorsementRequirements := tf.endorsementRequirements(ctx) + endorsementStatus := make([]components.PrivateTxEndorsementStatus, len(endorsementRequirements)) + for i, requirement := range endorsementRequirements { + endorsementStatus[i] = components.PrivateTxEndorsementStatus{ + Party: requirement.party, + EndorsementReceived: true, + } + if parties, found := tf.pendingEndorsementRequests[requirement.attRequest.Name]; found { + if request, found := parties[requirement.party]; found { + endorsementStatus[i].EndorsementReceived = false + endorsementStatus[i].RequestTime = request.requestTime.Format(time.RFC3339Nano) + } + } + } + + return components.PrivateTxStatus{ + TxID: tf.transaction.ID.String(), + Status: tf.status, + LatestEvent: tf.latestEvent, + LatestError: tf.latestError, + Endorsements: endorsementStatus, + Transaction: tf.transaction, + }, nil +} + func (tf *transactionFlow) hasOutstandingVerifierRequests(ctx context.Context) bool { log.L(ctx).Debug("transactionFlow:hasOutstandingVerifierRequests") @@ -74,13 +104,17 @@ func (tf *transactionFlow) hasOutstandingEndorsementRequests(ctx context.Context return len(tf.outstandingEndorsementRequests(ctx)) > 0 } -type outstandingEndorsementRequest struct { +type endorsementRequirement struct { attRequest *prototk.AttestationRequest party string } -func (tf *transactionFlow) outstandingEndorsementRequests(ctx context.Context) []*outstandingEndorsementRequest { - outstandingEndorsementRequests := make([]*outstandingEndorsementRequest, 0) +func (tf *transactionFlow) outstandingEndorsementRequests(ctx context.Context) []*endorsementRequirement { + outstandingEndorsementRequests := make([]*endorsementRequirement, 0) + if tf.transaction.PostAssembly == nil { + log.L(ctx).Debugf("PostAssembly is nil so there are no outstanding endorsement requests") + return outstandingEndorsementRequests + } for _, attRequest := range tf.transaction.PostAssembly.AttestationPlan { if attRequest.AttestationType == prototk.AttestationType_ENDORSE { for _, party := range attRequest.Parties { @@ -101,10 +135,27 @@ func (tf *transactionFlow) outstandingEndorsementRequests(ctx context.Context) [ } if !found { log.L(ctx).Debugf("endorsement request for %s outstanding for transaction %s", party, tf.transaction.ID) - outstandingEndorsementRequests = append(outstandingEndorsementRequests, &outstandingEndorsementRequest{party: party, attRequest: attRequest}) + outstandingEndorsementRequests = append(outstandingEndorsementRequests, &endorsementRequirement{party: party, attRequest: attRequest}) } } } } return outstandingEndorsementRequests } + +func (tf *transactionFlow) endorsementRequirements(ctx context.Context) []*endorsementRequirement { + //utility function to fold all the attestation plan into a single list, filtered by type - Endorse + endorsementRequests := make([]*endorsementRequirement, 0) + if tf.transaction.PostAssembly == nil { + log.L(ctx).Debugf("PostAssembly is nil so there are no endorsement requests") + return endorsementRequests + } + for _, attRequest := range tf.transaction.PostAssembly.AttestationPlan { + if attRequest.AttestationType == prototk.AttestationType_ENDORSE { + for _, party := range attRequest.Parties { + endorsementRequests = append(endorsementRequests, &endorsementRequirement{party: party, attRequest: attRequest}) + } + } + } + return endorsementRequests +} diff --git a/core/go/internal/privatetxnmgr/transaction_flow_test.go b/core/go/internal/privatetxnmgr/transaction_flow_test.go index 381fe3c10..912e3af22 100644 --- a/core/go/internal/privatetxnmgr/transaction_flow_test.go +++ b/core/go/internal/privatetxnmgr/transaction_flow_test.go @@ -21,11 +21,13 @@ import ( "time" "github.com/google/uuid" + "github.com/kaleido-io/paladin/config/pkg/confutil" "github.com/kaleido-io/paladin/core/internal/components" "github.com/kaleido-io/paladin/core/internal/privatetxnmgr/ptmgrtypes" "github.com/kaleido-io/paladin/core/mocks/componentmocks" "github.com/kaleido-io/paladin/core/mocks/privatetxnmgrmocks" "github.com/kaleido-io/paladin/core/mocks/prvtxsyncpointsmocks" + "github.com/kaleido-io/paladin/core/mocks/statedistributionmocks" "github.com/kaleido-io/paladin/toolkit/pkg/algorithms" "github.com/kaleido-io/paladin/toolkit/pkg/prototk" "github.com/kaleido-io/paladin/toolkit/pkg/signpayloads" @@ -33,9 +35,10 @@ import ( "github.com/kaleido-io/paladin/toolkit/pkg/verifiers" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" ) -type transactionProcessorDepencyMocks struct { +type transactionFlowDepencyMocks struct { allComponents *componentmocks.AllComponents domainSmartContract *componentmocks.DomainSmartContract domainContext *componentmocks.DomainContext @@ -48,11 +51,15 @@ type transactionProcessorDepencyMocks struct { identityResolver *componentmocks.IdentityResolver syncPoints *prvtxsyncpointsmocks.SyncPoints transportWriter *privatetxnmgrmocks.TransportWriter + environment *privatetxnmgrmocks.SequencerEnvironment + coordinatorSelector *privatetxnmgrmocks.CoordinatorSelector + stateDistributer *statedistributionmocks.StateDistributer + localAssembler *privatetxnmgrmocks.LocalAssembler } -func newPaladinTransactionProcessorForTesting(t *testing.T, ctx context.Context, transaction *components.PrivateTransaction) (*transactionFlow, *transactionProcessorDepencyMocks) { +func newTransactionFlowForTesting(t *testing.T, ctx context.Context, transaction *components.PrivateTransaction, nodeName string) (*transactionFlow, *transactionFlowDepencyMocks) { - mocks := &transactionProcessorDepencyMocks{ + mocks := &transactionFlowDepencyMocks{ allComponents: componentmocks.NewAllComponents(t), domainSmartContract: componentmocks.NewDomainSmartContract(t), domainContext: componentmocks.NewDomainContext(t), @@ -65,6 +72,10 @@ func newPaladinTransactionProcessorForTesting(t *testing.T, ctx context.Context, identityResolver: componentmocks.NewIdentityResolver(t), syncPoints: prvtxsyncpointsmocks.NewSyncPoints(t), transportWriter: privatetxnmgrmocks.NewTransportWriter(t), + environment: privatetxnmgrmocks.NewSequencerEnvironment(t), + coordinatorSelector: privatetxnmgrmocks.NewCoordinatorSelector(t), + stateDistributer: statedistributionmocks.NewStateDistributer(t), + localAssembler: privatetxnmgrmocks.NewLocalAssembler(t), } contractAddress := tktypes.RandAddress() mocks.allComponents.On("StateManager").Return(mocks.stateStore).Maybe() @@ -81,7 +92,9 @@ func newPaladinTransactionProcessorForTesting(t *testing.T, ctx context.Context, domain.On("Configuration").Return(&prototk.DomainConfig{}).Maybe() mocks.domainSmartContract.On("Domain").Return(domain).Maybe() - tp := NewTransactionFlow(ctx, transaction, tktypes.RandHex(16), mocks.allComponents, mocks.domainSmartContract, mocks.publisher, mocks.endorsementGatherer, mocks.identityResolver, mocks.syncPoints, mocks.transportWriter, 1*time.Minute) + assembleCoordinator := NewAssembleCoordinator(ctx, nodeName, 1, mocks.allComponents, mocks.domainSmartContract, mocks.domainContext, mocks.transportWriter, *contractAddress, mocks.environment, 1*time.Second, mocks.stateDistributer, mocks.localAssembler) + + tp := NewTransactionFlow(ctx, transaction, nodeName, mocks.allComponents, mocks.domainSmartContract, mocks.domainContext, mocks.publisher, mocks.endorsementGatherer, mocks.identityResolver, mocks.syncPoints, mocks.transportWriter, 1*time.Minute, mocks.coordinatorSelector, assembleCoordinator, mocks.environment) return tp.(*transactionFlow), mocks } @@ -172,7 +185,7 @@ func TestHasOutstandingEndorsementRequestsMultipleRequestsIncomplete(t *testing. }, } - tp, _ := newPaladinTransactionProcessorForTesting(t, ctx, testTx) + tp, _ := newTransactionFlowForTesting(t, ctx, testTx, "node1") result := tp.hasOutstandingEndorsementRequests(ctx) assert.True(t, result) @@ -285,7 +298,7 @@ func TestHasOutstandingEndorsementRequestsMultipleRequestsComplete(t *testing.T) }, } - tp, _ := newPaladinTransactionProcessorForTesting(t, ctx, testTx) + tp, _ := newTransactionFlowForTesting(t, ctx, testTx, "node1") result := tp.hasOutstandingEndorsementRequests(ctx) assert.False(t, result) @@ -359,7 +372,7 @@ func TestHasOutstandingEndorsementRequestSingleRequestMultiplePartiesIncomplete( }, } - tp, _ := newPaladinTransactionProcessorForTesting(t, ctx, testTx) + tp, _ := newTransactionFlowForTesting(t, ctx, testTx, "node1") result := tp.hasOutstandingEndorsementRequests(ctx) assert.True(t, result) @@ -454,7 +467,7 @@ func TestHasOutstandingEndorsementRequestSingleRequestMultiplePartiesComplete(t }, } - tp, _ := newPaladinTransactionProcessorForTesting(t, ctx, testTx) + tp, _ := newTransactionFlowForTesting(t, ctx, testTx, "node1") result := tp.hasOutstandingEndorsementRequests(ctx) assert.False(t, result) @@ -547,7 +560,7 @@ func TestHasOutstandingEndorsementRequestSingleRequestMultiplePartiesDuplicate(t }, } - tp, _ := newPaladinTransactionProcessorForTesting(t, ctx, testTx) + tp, _ := newTransactionFlowForTesting(t, ctx, testTx, "node1") result := tp.hasOutstandingEndorsementRequests(ctx) assert.True(t, result) @@ -640,12 +653,12 @@ func TestHasOutstandingEndorsementRequestSingleRequestMultiplePartiesCompleteMix }, } - tp, _ := newPaladinTransactionProcessorForTesting(t, ctx, testTx) + tp, _ := newTransactionFlowForTesting(t, ctx, testTx, "node1") result := tp.hasOutstandingEndorsementRequests(ctx) assert.False(t, result) } -func TestRequestEndorsements(t *testing.T) { +func TestRequestRemoteEndorsements(t *testing.T) { ctx := context.Background() newTxID := uuid.New() @@ -712,8 +725,12 @@ func TestRequestEndorsements(t *testing.T) { }, } - tp, mocks := newPaladinTransactionProcessorForTesting(t, ctx, testTx) + sendingNodeName := "sendingNode" + + tp, mocks := newTransactionFlowForTesting(t, ctx, testTx, sendingNodeName) + mocks.coordinatorSelector.On("SelectCoordinatorNode", mock.Anything, mock.Anything, mock.Anything).Return(int64(0), sendingNodeName, nil) mocks.transportWriter.On("SendEndorsementRequest", + mock.Anything, mock.Anything, "alice@node1", "node1", @@ -728,6 +745,7 @@ func TestRequestEndorsements(t *testing.T) { mock.Anything, //InfoStates, ).Return(nil).Once() mocks.transportWriter.On("SendEndorsementRequest", + mock.Anything, mock.Anything, "bob@node2", "node2", @@ -742,6 +760,7 @@ func TestRequestEndorsements(t *testing.T) { mock.Anything, //InfoStates, ).Return(nil).Once() mocks.transportWriter.On("SendEndorsementRequest", + mock.Anything, mock.Anything, "carol@node2", "node2", @@ -764,6 +783,160 @@ func TestRequestEndorsements(t *testing.T) { } +func TestRequestLocalEndorsements(t *testing.T) { + ctx := context.Background() + newTxID := uuid.New() + //endorsers are on the same node as the sender + sendingNodeName := "sendingNode" + + aliceIdentityLocator := "alice@" + sendingNodeName + aliceVerifier := tktypes.RandAddress().String() + bobIdentityLocator := "bob@" + sendingNodeName + bobVerifier := tktypes.RandAddress().String() + carolIdentityLocator := "carol@" + sendingNodeName + carolVerifier := tktypes.RandAddress().String() + + testContractAddress := *tktypes.RandAddress() + // create a transaction as if we have already + // - resolved the verifiers + // - assembled it + // - signed it + // so next step is to request endorsements + testTx := &components.PrivateTransaction{ + ID: newTxID, + Inputs: &components.TransactionInputs{ + To: testContractAddress, + From: aliceIdentityLocator, + }, + PreAssembly: &components.TransactionPreAssembly{ + TransactionSpecification: &prototk.TransactionSpecification{ + From: aliceIdentityLocator, + TransactionId: newTxID.String(), + }, + Verifiers: []*prototk.ResolvedVerifier{ + { + Lookup: aliceIdentityLocator, + Algorithm: algorithms.ECDSA_SECP256K1, + VerifierType: verifiers.ETH_ADDRESS, + Verifier: aliceVerifier, + }, + { + Lookup: bobIdentityLocator, + Algorithm: algorithms.ECDSA_SECP256K1, + VerifierType: verifiers.ETH_ADDRESS, + Verifier: bobVerifier, + }, + { + Lookup: carolIdentityLocator, + Algorithm: algorithms.ECDSA_SECP256K1, + VerifierType: verifiers.ETH_ADDRESS, + Verifier: carolVerifier, + }, + }, + }, + PostAssembly: &components.TransactionPostAssembly{ + AttestationPlan: []*prototk.AttestationRequest{ + { + Name: "foo", + AttestationType: prototk.AttestationType_ENDORSE, + Algorithm: algorithms.ECDSA_SECP256K1, + VerifierType: verifiers.ETH_ADDRESS, + PayloadType: signpayloads.OPAQUE_TO_RSV, + Parties: []string{ + aliceIdentityLocator, + bobIdentityLocator, + carolIdentityLocator, + }, + }, + }, + }, + } + + tp, mocks := newTransactionFlowForTesting(t, ctx, testTx, sendingNodeName) + mocks.coordinatorSelector.On("SelectCoordinatorNode", mock.Anything, mock.Anything, mock.Anything).Return(int64(0), sendingNodeName, nil) + mocks.endorsementGatherer.On("GatherEndorsement", + mock.Anything, // context + mock.Anything, //TransactionSpecification, + mock.Anything, //Verifiers, + mock.Anything, //Signatures, + mock.Anything, //InputStates, + mock.Anything, //ReadStates, + mock.Anything, //OutputStates, + mock.Anything, //InfoStates, + "alice@"+sendingNodeName, + mock.Anything, //attRequest + ).Return( + &prototk.AttestationResult{ + Name: "foo", + Verifier: &prototk.ResolvedVerifier{ + Lookup: aliceIdentityLocator, + Algorithm: algorithms.ECDSA_SECP256K1, + Verifier: aliceVerifier, + VerifierType: verifiers.ETH_ADDRESS, + }, + }, + nil, + nil, + ).Once() + mocks.endorsementGatherer.On("GatherEndorsement", + mock.Anything, // context + mock.Anything, //TransactionSpecification, + mock.Anything, //Verifiers, + mock.Anything, //Signatures, + mock.Anything, //InputStates, + mock.Anything, //ReadStates, + mock.Anything, //OutputStates, + mock.Anything, //InfoStates, + "bob@"+sendingNodeName, + mock.Anything, //attRequest + ).Return( + &prototk.AttestationResult{ + Name: "foo", + Verifier: &prototk.ResolvedVerifier{ + Lookup: aliceIdentityLocator, + Algorithm: algorithms.ECDSA_SECP256K1, + Verifier: aliceVerifier, + VerifierType: verifiers.ETH_ADDRESS, + }, + }, + nil, + nil, + ).Once() + mocks.endorsementGatherer.On("GatherEndorsement", + mock.Anything, // context + mock.Anything, //TransactionSpecification, + mock.Anything, //Verifiers, + mock.Anything, //Signatures, + mock.Anything, //InputStates, + mock.Anything, //ReadStates, + mock.Anything, //OutputStates, + mock.Anything, //InfoStates, + "carol@"+sendingNodeName, + mock.Anything, //attRequest + ).Return( + &prototk.AttestationResult{ + Name: "foo", + Verifier: &prototk.ResolvedVerifier{ + Lookup: aliceIdentityLocator, + Algorithm: algorithms.ECDSA_SECP256K1, + Verifier: aliceVerifier, + VerifierType: verifiers.ETH_ADDRESS, + }, + }, + nil, + nil, + ).Once() + + mocks.publisher.On("PublishTransactionEndorsedEvent", mock.Anything, newTxID.String(), mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return().Times(3) + tp.Action(ctx) + + mocks.transportWriter.AssertExpectations(t) + + //Check that we don't send the same requests again (we specified Once in the mocks above) + tp.Action(ctx) + +} + func TestTimedOutEndorsementRequest(t *testing.T) { // This can happen if the remote node is slow to respond //or the network is unreliable and the request or the response has been lost @@ -829,7 +1002,6 @@ func TestTimedOutEndorsementRequest(t *testing.T) { VerifierType: verifiers.ETH_ADDRESS, PayloadType: signpayloads.OPAQUE_TO_RSV, Parties: []string{ - aliceIdentityLocator, bobIdentityLocator, carolIdentityLocator, }, @@ -838,14 +1010,41 @@ func TestTimedOutEndorsementRequest(t *testing.T) { }, } - tp, mocks := newPaladinTransactionProcessorForTesting(t, ctx, testTx) + tp, mocks := newTransactionFlowForTesting(t, ctx, testTx, "node1") + mocks.coordinatorSelector.On("SelectCoordinatorNode", mock.Anything, mock.Anything, mock.Anything).Return(int64(0), "node1", nil) fakeClock := &fakeClock{timePassed: 0} tp.clock = fakeClock - expectEndorsementRequest := func(party, node string) { + expectEndorsementRequest := func(idempotencyKey *string, party, node string) { + //set the idempotency key received to the pointer provided + mocks.transportWriter.On("SendEndorsementRequest", + mock.Anything, + mock.Anything, + party, + node, + testContractAddress.String(), + newTxID.String(), + mock.Anything, //attRequest + mock.Anything, //TransactionSpecification, + mock.Anything, //Verifiers, + mock.Anything, //Signatures, + mock.Anything, //InputStates, + mock.Anything, //OutputStates, + mock.Anything, //InfoStates, + ).Return(nil).Once().Run(func(args mock.Arguments) { + if idempotencyKey != nil { + receivedIdempotencyKey := args.Get(1).(string) + *idempotencyKey = receivedIdempotencyKey + } + }) + } + + expectIdempotentEndorsementRequest := func(idempotencyKey, party, node string) { + //asserts the given idempotency key is used in the request mocks.transportWriter.On("SendEndorsementRequest", mock.Anything, + idempotencyKey, party, node, testContractAddress.String(), @@ -860,9 +1059,11 @@ func TestTimedOutEndorsementRequest(t *testing.T) { ).Return(nil).Once() } - expectEndorsementRequest("alice@node1", "node1") - expectEndorsementRequest("bob@node2", "node2") - expectEndorsementRequest("carol@node2", "node2") + idempotencyKeyBob := "" + idempotencyKeyCarol := "" + + expectEndorsementRequest(&idempotencyKeyBob, "bob@node2", "node2") + expectEndorsementRequest(&idempotencyKeyCarol, "carol@node2", "node2") tp.Action(ctx) mocks.transportWriter.AssertExpectations(t) @@ -871,9 +1072,8 @@ func TestTimedOutEndorsementRequest(t *testing.T) { //simulate the passing of time fakeClock.timePassed = 1*time.Minute + 1*time.Second - expectEndorsementRequest("alice@node1", "node1") - expectEndorsementRequest("bob@node2", "node2") - expectEndorsementRequest("carol@node2", "node2") + expectIdempotentEndorsementRequest(idempotencyKeyBob, "bob@node2", "node2") + expectIdempotentEndorsementRequest(idempotencyKeyCarol, "carol@node2", "node2") tp.Action(ctx) mocks.transportWriter.AssertExpectations(t) @@ -895,31 +1095,21 @@ func TestTimedOutEndorsementRequest(t *testing.T) { VerifierType: verifiers.ETH_ADDRESS, }, }, + Party: bobIdentityLocator, + AttestationRequestName: "foo", + IdempotencyKey: idempotencyKeyBob, }) //simulate the passing of time fakeClock.timePassed = fakeClock.timePassed + 1*time.Minute + 1*time.Second - expectEndorsementRequest("alice@node1", "node1") - expectEndorsementRequest("carol@node2", "node2") + expectIdempotentEndorsementRequest(idempotencyKeyCarol, "carol@node2", "node2") tp.Action(ctx) } -func TestEndorsementRequestAfterReassemble(t *testing.T) { - // when we have re-assembled the transaction after sending an endorsement request - // we should resend the request and should ignore any responses that eventually come back for the - // original request - - //TODO skip for now while we think about his. - // the main reason for re-assembly would be a rejected endorsement because we are trying to spend a state that has ( unbeknown to us) been spent already - // in that situation ( most likely in pente) we actually want to trigger a delegation handover - // however, after delegation re-assembly is likely - t.Skip() - -} +func TestEndorsementResponseAfterRevert(t *testing.T) { + // We send out 2 endorsement requests , the first one back causes a revert + // the second one back should be ignored -func TestDuplicateEndorsementResponse(t *testing.T) { - // we effectively have an at least once delivery guarantee on the endorsement requests and responses - // so we need to be able to handle duplicate responses ctx := context.Background() newTxID := uuid.New() @@ -931,7 +1121,7 @@ func TestDuplicateEndorsementResponse(t *testing.T) { carolVerifier := tktypes.RandAddress().String() testContractAddress := *tktypes.RandAddress() - // create a transaction as if we have already: + // create a transaction as if we have already // - resolved the verifiers // - assembled it // - signed it @@ -977,7 +1167,6 @@ func TestDuplicateEndorsementResponse(t *testing.T) { VerifierType: verifiers.ETH_ADDRESS, PayloadType: signpayloads.OPAQUE_TO_RSV, Parties: []string{ - aliceIdentityLocator, bobIdentityLocator, carolIdentityLocator, }, @@ -986,14 +1175,16 @@ func TestDuplicateEndorsementResponse(t *testing.T) { }, } - tp, mocks := newPaladinTransactionProcessorForTesting(t, ctx, testTx) + tp, mocks := newTransactionFlowForTesting(t, ctx, testTx, "node1") + mocks.coordinatorSelector.On("SelectCoordinatorNode", mock.Anything, mock.Anything, mock.Anything).Return(int64(0), "node1", nil) fakeClock := &fakeClock{timePassed: 0} tp.clock = fakeClock - expectEndorsementRequest := func(party, node string) { + expectEndorsementRequest := func(idempotencyKey *string, party, node string) { mocks.transportWriter.On("SendEndorsementRequest", mock.Anything, + mock.Anything, //idempotency key party, node, testContractAddress.String(), @@ -1005,40 +1196,36 @@ func TestDuplicateEndorsementResponse(t *testing.T) { mock.Anything, //InputStates, mock.Anything, //OutputStates, mock.Anything, //InfoStates, - ).Return(nil).Once() + ).Return(nil).Once().Run(func(args mock.Arguments) { + if idempotencyKey != nil { + receivedIdempotencyKey := args.Get(1).(string) + *idempotencyKey = receivedIdempotencyKey + } + }) } - expectEndorsementRequest("alice@node1", "node1") - expectEndorsementRequest("bob@node2", "node2") - expectEndorsementRequest("carol@node2", "node2") + bobIdempotencyKey := "" + carolIdempotencyKey := "" + expectEndorsementRequest(&bobIdempotencyKey, "bob@node2", "node2") + expectEndorsementRequest(&carolIdempotencyKey, "carol@node2", "node2") tp.Action(ctx) mocks.transportWriter.AssertExpectations(t) - //Receive response from alice + //Receive revert response from bob tp.applyTransactionEndorsedEvent(ctx, &ptmgrtypes.TransactionEndorsedEvent{ PrivateTransactionEventBase: ptmgrtypes.PrivateTransactionEventBase{ TransactionID: newTxID.String(), ContractAddress: testContractAddress.String(), }, - Endorsement: &prototk.AttestationResult{ - Name: "foo", - Verifier: &prototk.ResolvedVerifier{ - Lookup: aliceIdentityLocator, - Algorithm: algorithms.ECDSA_SECP256K1, - Verifier: aliceVerifier, - VerifierType: verifiers.ETH_ADDRESS, - }, - }, + Party: bobIdentityLocator, + AttestationRequestName: "foo", + IdempotencyKey: bobIdempotencyKey, + RevertReason: confutil.P("bob refused to endorse"), }) - //simulate the passing of time - fakeClock.timePassed = 1*time.Minute + 1*time.Second - expectEndorsementRequest("bob@node2", "node2") - expectEndorsementRequest("carol@node2", "node2") tp.Action(ctx) - mocks.transportWriter.AssertExpectations(t) - //Receive both responses from carol + //Receive successful response from carol tp.applyTransactionEndorsedEvent(ctx, &ptmgrtypes.TransactionEndorsedEvent{ PrivateTransactionEventBase: ptmgrtypes.PrivateTransactionEventBase{ TransactionID: newTxID.String(), @@ -1053,7 +1240,166 @@ func TestDuplicateEndorsementResponse(t *testing.T) { VerifierType: verifiers.ETH_ADDRESS, }, }, + Party: carolIdentityLocator, + AttestationRequestName: "foo", + IdempotencyKey: carolIdempotencyKey, }) + + //transaction should be marked for reassemble + assert.False(t, tp.ReadyForSequencing(ctx)) + +} + +func TestEndorsementResponseAfterReassemble(t *testing.T) { + // Similar to TestEndorsementResponseAfterRevert: + // We send out 2 endorsement requests , the first one back causes a revert + // the second one back should be ignored + // however, in this case, we manage to reassemble the transaction and send out the endorsement request again + // before the second endorsement response comes back + // it should still be ignored because it has been made obsolete by the reassemble and a new endorsement request to that same party + ctx := context.Background() + newTxID := uuid.New() + + aliceIdentityLocator := "alice@node1" + aliceVerifier := tktypes.RandAddress().String() + bobIdentityLocator := "bob@node2" + bobVerifier := tktypes.RandAddress().String() + carolIdentityLocator := "carol@node2" + carolVerifier := tktypes.RandAddress().String() + + testContractAddress := *tktypes.RandAddress() + // create a transaction as if we have already + // - resolved the verifiers + // - assembled it + // - signed it + // so next step is to request endorsements + payloadFromAssemble1 := tktypes.RandBytes(32) + testTx := &components.PrivateTransaction{ + ID: newTxID, + Inputs: &components.TransactionInputs{ + To: testContractAddress, + From: aliceIdentityLocator, + }, + PreAssembly: &components.TransactionPreAssembly{ + TransactionSpecification: &prototk.TransactionSpecification{ + From: aliceIdentityLocator, + TransactionId: newTxID.String(), + }, + Verifiers: []*prototk.ResolvedVerifier{ + { + Lookup: aliceIdentityLocator, + Algorithm: algorithms.ECDSA_SECP256K1, + VerifierType: verifiers.ETH_ADDRESS, + Verifier: aliceVerifier, + }, + { + Lookup: bobIdentityLocator, + Algorithm: algorithms.ECDSA_SECP256K1, + VerifierType: verifiers.ETH_ADDRESS, + Verifier: bobVerifier, + }, + { + Lookup: carolIdentityLocator, + Algorithm: algorithms.ECDSA_SECP256K1, + VerifierType: verifiers.ETH_ADDRESS, + Verifier: carolVerifier, + }, + }, + }, + PostAssembly: &components.TransactionPostAssembly{ + AttestationPlan: []*prototk.AttestationRequest{ + { + Name: "foo", + AttestationType: prototk.AttestationType_ENDORSE, + Algorithm: algorithms.ECDSA_SECP256K1, + VerifierType: verifiers.ETH_ADDRESS, + PayloadType: signpayloads.OPAQUE_TO_RSV, + Payload: payloadFromAssemble1, + Parties: []string{ + bobIdentityLocator, + carolIdentityLocator, + }, + }, + }, + }, + } + + tp, mocks := newTransactionFlowForTesting(t, ctx, testTx, "node1") + mocks.coordinatorSelector.On("SelectCoordinatorNode", mock.Anything, mock.Anything, mock.Anything).Return(int64(0), "node1", nil) + + fakeClock := &fakeClock{timePassed: 0} + tp.clock = fakeClock + + expectEndorsementRequest := func(idempotencyKey *string, party, node string) { + mocks.transportWriter.On("SendEndorsementRequest", + mock.Anything, + mock.Anything, //idempotency key + party, + node, + testContractAddress.String(), + newTxID.String(), + mock.Anything, //attRequest + mock.Anything, //TransactionSpecification, + mock.Anything, //Verifiers, + mock.Anything, //Signatures, + mock.Anything, //InputStates, + mock.Anything, //OutputStates, + mock.Anything, //InfoStates, + ).Return(nil).Once().Run(func(args mock.Arguments) { + if idempotencyKey != nil { + receivedIdempotencyKey := args.Get(1).(string) + *idempotencyKey = receivedIdempotencyKey + } + }) + } + + bobIdempotencyKey := "" + carolIdempotencyKey := "" + expectEndorsementRequest(&bobIdempotencyKey, "bob@node2", "node2") + expectEndorsementRequest(&carolIdempotencyKey, "carol@node2", "node2") + tp.Action(ctx) + mocks.transportWriter.AssertExpectations(t) + + //Receive revert response from bob + tp.applyTransactionEndorsedEvent(ctx, &ptmgrtypes.TransactionEndorsedEvent{ + PrivateTransactionEventBase: ptmgrtypes.PrivateTransactionEventBase{ + TransactionID: newTxID.String(), + ContractAddress: testContractAddress.String(), + }, + Party: bobIdentityLocator, + AttestationRequestName: "foo", + RevertReason: confutil.P("bob refused to endorse"), + IdempotencyKey: bobIdempotencyKey, + }) + + tp.Action(ctx) + + //re-assemble the transaction + payloadFromAssemble2 := tktypes.RandBytes(32) + + tp.transaction.PostAssembly = &components.TransactionPostAssembly{ + AttestationPlan: []*prototk.AttestationRequest{ + { + Name: "foo", + AttestationType: prototk.AttestationType_ENDORSE, + Algorithm: algorithms.ECDSA_SECP256K1, + VerifierType: verifiers.ETH_ADDRESS, + PayloadType: signpayloads.OPAQUE_TO_RSV, + Payload: payloadFromAssemble2, + Parties: []string{ + bobIdentityLocator, + carolIdentityLocator, + }, + }, + }, + } + bobIdempotencyKey2 := "" + carolIdempotencyKey2 := "" + expectEndorsementRequest(&bobIdempotencyKey2, "bob@node2", "node2") + expectEndorsementRequest(&carolIdempotencyKey2, "carol@node2", "node2") + tp.Action(ctx) + + //Receive late successful response from carol tp.applyTransactionEndorsedEvent(ctx, &ptmgrtypes.TransactionEndorsedEvent{ PrivateTransactionEventBase: ptmgrtypes.PrivateTransactionEventBase{ TransactionID: newTxID.String(), @@ -1067,7 +1413,216 @@ func TestDuplicateEndorsementResponse(t *testing.T) { Verifier: carolVerifier, VerifierType: verifiers.ETH_ADDRESS, }, + PayloadType: confutil.P(signpayloads.OPAQUE_TO_RSV), + Payload: payloadFromAssemble1, }, + Party: carolIdentityLocator, + AttestationRequestName: "foo", + IdempotencyKey: carolIdempotencyKey, + }) + + // should still have 2 outstanding endorsement requests + assert.Len(t, tp.outstandingEndorsementRequests(ctx), 2) + +} + +func TestDuplicateEndorsementResponse(t *testing.T) { + // we effectively have an at least once delivery guarantee on the endorsement requests and responses + // so we need to be able to handle duplicate responses + ctx := context.Background() + newTxID := uuid.New() + + senderNodeName := "senderNode" + senderIdentityLocator := "sender@" + senderNodeName + aliceIdentityLocator := "alice@node1" + aliceVerifier := tktypes.RandAddress().String() + bobIdentityLocator := "bob@node2" + bobVerifier := tktypes.RandAddress().String() + carolIdentityLocator := "carol@node2" + carolVerifier := tktypes.RandAddress().String() + + testContractAddress := *tktypes.RandAddress() + // create a transaction as if we have already: + // - resolved the verifiers + // - assembled it + // - signed it + // so next step is to request endorsements + testTx := &components.PrivateTransaction{ + ID: newTxID, + Inputs: &components.TransactionInputs{ + To: testContractAddress, + From: senderIdentityLocator, + }, + PreAssembly: &components.TransactionPreAssembly{ + TransactionSpecification: &prototk.TransactionSpecification{ + From: senderIdentityLocator, + TransactionId: newTxID.String(), + }, + Verifiers: []*prototk.ResolvedVerifier{ + { + Lookup: senderIdentityLocator, + Algorithm: algorithms.ECDSA_SECP256K1, + VerifierType: verifiers.ETH_ADDRESS, + Verifier: aliceVerifier, + }, + { + Lookup: aliceIdentityLocator, + Algorithm: algorithms.ECDSA_SECP256K1, + VerifierType: verifiers.ETH_ADDRESS, + Verifier: aliceVerifier, + }, + { + Lookup: bobIdentityLocator, + Algorithm: algorithms.ECDSA_SECP256K1, + VerifierType: verifiers.ETH_ADDRESS, + Verifier: bobVerifier, + }, + { + Lookup: carolIdentityLocator, + Algorithm: algorithms.ECDSA_SECP256K1, + VerifierType: verifiers.ETH_ADDRESS, + Verifier: carolVerifier, + }, + }, + }, + PostAssembly: &components.TransactionPostAssembly{ + AttestationPlan: []*prototk.AttestationRequest{ + { + Name: "foo", + AttestationType: prototk.AttestationType_ENDORSE, + Algorithm: algorithms.ECDSA_SECP256K1, + VerifierType: verifiers.ETH_ADDRESS, + PayloadType: signpayloads.OPAQUE_TO_RSV, + Parties: []string{ + aliceIdentityLocator, + bobIdentityLocator, + carolIdentityLocator, + }, + }, + }, + }, + } + + tp, mocks := newTransactionFlowForTesting(t, ctx, testTx, senderNodeName) + mocks.coordinatorSelector.On("SelectCoordinatorNode", mock.Anything, mock.Anything, mock.Anything).Return(int64(0), senderNodeName, nil) + + fakeClock := &fakeClock{timePassed: 0} + tp.clock = fakeClock + + expectEndorsementRequest := func(idempotencyKey *string, party, node string) { + mocks.transportWriter.On("SendEndorsementRequest", + mock.Anything, + mock.Anything, //idempotency key + party, + node, + testContractAddress.String(), + newTxID.String(), + mock.Anything, //attRequest + mock.Anything, //TransactionSpecification, + mock.Anything, //Verifiers, + mock.Anything, //Signatures, + mock.Anything, //InputStates, + mock.Anything, //OutputStates, + mock.Anything, //InfoStates, + ).Return(nil).Once().Run(func(args mock.Arguments) { + if idempotencyKey != nil { + receivedIdempotencyKey := args.Get(1).(string) + *idempotencyKey = receivedIdempotencyKey + } + }) + } + + expectIdempotentEndorsementRequest := func(idempotencyKey, party, node string) { + //asserts the given idempotency key is used in the request + mocks.transportWriter.On("SendEndorsementRequest", + mock.Anything, + idempotencyKey, + party, + node, + testContractAddress.String(), + newTxID.String(), + mock.Anything, //attRequest + mock.Anything, //TransactionSpecification, + mock.Anything, //Verifiers, + mock.Anything, //Signatures, + mock.Anything, //InputStates, + mock.Anything, //OutputStates, + mock.Anything, //InfoStates, + ).Return(nil).Once() + } + + aliceIdempotencyKey := "" + bobIdempotencyKey := "" + carolIdempotencyKey := "" + expectEndorsementRequest(&aliceIdempotencyKey, "alice@node1", "node1") + expectEndorsementRequest(&bobIdempotencyKey, "bob@node2", "node2") + expectEndorsementRequest(&carolIdempotencyKey, "carol@node2", "node2") + tp.Action(ctx) + mocks.transportWriter.AssertExpectations(t) + + //Receive response from alice + tp.applyTransactionEndorsedEvent(ctx, &ptmgrtypes.TransactionEndorsedEvent{ + PrivateTransactionEventBase: ptmgrtypes.PrivateTransactionEventBase{ + TransactionID: newTxID.String(), + ContractAddress: testContractAddress.String(), + }, + Endorsement: &prototk.AttestationResult{ + Name: "foo", + Verifier: &prototk.ResolvedVerifier{ + Lookup: aliceIdentityLocator, + Algorithm: algorithms.ECDSA_SECP256K1, + Verifier: aliceVerifier, + VerifierType: verifiers.ETH_ADDRESS, + }, + }, + Party: aliceIdentityLocator, + AttestationRequestName: "foo", + IdempotencyKey: aliceIdempotencyKey, + }) + + //simulate the passing of time + fakeClock.timePassed = 1*time.Minute + 1*time.Second + expectIdempotentEndorsementRequest(bobIdempotencyKey, "bob@node2", "node2") + expectIdempotentEndorsementRequest(carolIdempotencyKey, "carol@node2", "node2") + tp.Action(ctx) + mocks.transportWriter.AssertExpectations(t) + + //Receive both responses from carol + tp.applyTransactionEndorsedEvent(ctx, &ptmgrtypes.TransactionEndorsedEvent{ + PrivateTransactionEventBase: ptmgrtypes.PrivateTransactionEventBase{ + TransactionID: newTxID.String(), + ContractAddress: testContractAddress.String(), + }, + Endorsement: &prototk.AttestationResult{ + Name: "foo", + Verifier: &prototk.ResolvedVerifier{ + Lookup: carolIdentityLocator, + Algorithm: algorithms.ECDSA_SECP256K1, + Verifier: carolVerifier, + VerifierType: verifiers.ETH_ADDRESS, + }, + }, + Party: carolIdentityLocator, + AttestationRequestName: "foo", + IdempotencyKey: carolIdempotencyKey, + }) + tp.applyTransactionEndorsedEvent(ctx, &ptmgrtypes.TransactionEndorsedEvent{ + PrivateTransactionEventBase: ptmgrtypes.PrivateTransactionEventBase{ + TransactionID: newTxID.String(), + ContractAddress: testContractAddress.String(), + }, + Endorsement: &prototk.AttestationResult{ + Name: "foo", + Verifier: &prototk.ResolvedVerifier{ + Lookup: carolIdentityLocator, + Algorithm: algorithms.ECDSA_SECP256K1, + Verifier: carolVerifier, + VerifierType: verifiers.ETH_ADDRESS, + }, + }, + Party: carolIdentityLocator, + AttestationRequestName: "foo", + IdempotencyKey: carolIdempotencyKey, }) // no further action because we are still waiting for a response from bob @@ -1075,6 +1630,201 @@ func TestDuplicateEndorsementResponse(t *testing.T) { tp.Action(ctx) } +func TestGetTxStatusPendingEndorsements(t *testing.T) { + ctx := context.Background() + newTxID := uuid.New() + + senderNodeName := "senderNode" + senderIdentityLocator := "sender@" + senderNodeName + aliceIdentityLocator := "alice@node1" + aliceVerifier := tktypes.RandAddress().String() + bobIdentityLocator := "bob@node2" + bobVerifier := tktypes.RandAddress().String() + carolIdentityLocator := "carol@node3" + carolVerifier := tktypes.RandAddress().String() + daveIdentityLocator := "dave@node4" + daveVerifier := tktypes.RandAddress().String() + + testContractAddress := *tktypes.RandAddress() + testTx := &components.PrivateTransaction{ + ID: newTxID, + Inputs: &components.TransactionInputs{ + To: testContractAddress, + From: senderIdentityLocator, + }, + PreAssembly: &components.TransactionPreAssembly{ + TransactionSpecification: &prototk.TransactionSpecification{ + From: senderIdentityLocator, + TransactionId: newTxID.String(), + }, + Verifiers: []*prototk.ResolvedVerifier{ + { + Lookup: senderIdentityLocator, + Algorithm: algorithms.ECDSA_SECP256K1, + VerifierType: verifiers.ETH_ADDRESS, + Verifier: aliceVerifier, + }, + { + Lookup: aliceIdentityLocator, + Algorithm: algorithms.ECDSA_SECP256K1, + VerifierType: verifiers.ETH_ADDRESS, + Verifier: aliceVerifier, + }, + { + Lookup: bobIdentityLocator, + Algorithm: algorithms.ECDSA_SECP256K1, + VerifierType: verifiers.ETH_ADDRESS, + Verifier: bobVerifier, + }, + { + Lookup: carolIdentityLocator, + Algorithm: algorithms.ECDSA_SECP256K1, + VerifierType: verifiers.ETH_ADDRESS, + Verifier: carolVerifier, + }, + { + Lookup: daveIdentityLocator, + Algorithm: algorithms.ECDSA_SECP256K1, + VerifierType: verifiers.ETH_ADDRESS, + Verifier: daveVerifier, + }, + }, + }, + PostAssembly: &components.TransactionPostAssembly{ + AttestationPlan: []*prototk.AttestationRequest{ + { + Name: "foo", + AttestationType: prototk.AttestationType_ENDORSE, + Algorithm: algorithms.ECDSA_SECP256K1, + VerifierType: verifiers.ETH_ADDRESS, + PayloadType: signpayloads.OPAQUE_TO_RSV, + Parties: []string{ + aliceIdentityLocator, + bobIdentityLocator, + }, + }, + { + Name: "bar", + AttestationType: prototk.AttestationType_ENDORSE, + Algorithm: algorithms.ECDSA_SECP256K1, + VerifierType: verifiers.ETH_ADDRESS, + PayloadType: signpayloads.OPAQUE_TO_RSV, + Parties: []string{ + carolIdentityLocator, + daveIdentityLocator, + }, + }, + }, + }, + } + + tp, mocks := newTransactionFlowForTesting(t, ctx, testTx, senderNodeName) + mocks.coordinatorSelector.On("SelectCoordinatorNode", mock.Anything, mock.Anything, mock.Anything).Return(int64(0), senderNodeName, nil) + + expectEndorsementRequest := func(idempotencyKey *string, party, node string) { + mocks.transportWriter.On("SendEndorsementRequest", + mock.Anything, + mock.Anything, //idempotency key + party, + node, + testContractAddress.String(), + newTxID.String(), + mock.Anything, //attRequest + mock.Anything, //TransactionSpecification, + mock.Anything, //Verifiers, + mock.Anything, //Signatures, + mock.Anything, //InputStates, + mock.Anything, //OutputStates, + mock.Anything, //InfoStates, + ).Return(nil).Once().Run(func(args mock.Arguments) { + if idempotencyKey != nil { + receivedIdempotencyKey := args.Get(1).(string) + *idempotencyKey = receivedIdempotencyKey + } + }) + } + aliceIdempotencyKey := "" + bobIdempotencyKey := "" + carolIdempotencyKey := "" + daveIdempotencyKey := "" + expectEndorsementRequest(&aliceIdempotencyKey, "alice@node1", "node1") + expectEndorsementRequest(&bobIdempotencyKey, "bob@node2", "node2") + expectEndorsementRequest(&carolIdempotencyKey, "carol@node3", "node3") + expectEndorsementRequest(&daveIdempotencyKey, "dave@node4", "node4") + + tp.Action(ctx) + tp.applyTransactionEndorsedEvent(ctx, &ptmgrtypes.TransactionEndorsedEvent{ + PrivateTransactionEventBase: ptmgrtypes.PrivateTransactionEventBase{ + TransactionID: newTxID.String(), + ContractAddress: testContractAddress.String(), + }, + IdempotencyKey: aliceIdempotencyKey, + Party: aliceIdentityLocator, + AttestationRequestName: "foo", + Endorsement: &prototk.AttestationResult{ + Name: "foo", + Verifier: &prototk.ResolvedVerifier{ + Lookup: aliceIdentityLocator, + Algorithm: algorithms.ECDSA_SECP256K1, + Verifier: tktypes.RandAddress().String(), + VerifierType: verifiers.ETH_ADDRESS, + }, + }, + }) + tp.applyTransactionEndorsedEvent(ctx, &ptmgrtypes.TransactionEndorsedEvent{ + PrivateTransactionEventBase: ptmgrtypes.PrivateTransactionEventBase{ + TransactionID: newTxID.String(), + ContractAddress: testContractAddress.String(), + }, + IdempotencyKey: daveIdempotencyKey, + Party: daveIdentityLocator, + AttestationRequestName: "bar", + Endorsement: &prototk.AttestationResult{ + Name: "bar", + Verifier: &prototk.ResolvedVerifier{ + Lookup: daveIdentityLocator, + Algorithm: algorithms.ECDSA_SECP256K1, + Verifier: tktypes.RandAddress().String(), + VerifierType: verifiers.ETH_ADDRESS, + }, + }, + }) + status, err := tp.GetTxStatus(ctx) + assert.NoError(t, err) + assert.Equal(t, newTxID.String(), status.TxID) + assert.Len(t, status.Endorsements, 4) + endorsementStatusForParty := func(party string) *components.PrivateTxEndorsementStatus { + for _, e := range status.Endorsements { + if e.Party == party { + return &e + } + } + return nil + } + require.NotNil(t, endorsementStatusForParty(aliceIdentityLocator)) + require.NotNil(t, endorsementStatusForParty(bobIdentityLocator)) + require.NotNil(t, endorsementStatusForParty(carolIdentityLocator)) + require.NotNil(t, endorsementStatusForParty(daveIdentityLocator)) + + assert.Empty(t, endorsementStatusForParty(aliceIdentityLocator).RequestTime) + assert.True(t, endorsementStatusForParty(aliceIdentityLocator).EndorsementReceived) + + assert.NotEmpty(t, endorsementStatusForParty(bobIdentityLocator).RequestTime) + assert.False(t, endorsementStatusForParty(bobIdentityLocator).EndorsementReceived) + + assert.NotEmpty(t, endorsementStatusForParty(carolIdentityLocator).RequestTime) + assert.False(t, endorsementStatusForParty(carolIdentityLocator).EndorsementReceived) + + assert.Empty(t, endorsementStatusForParty(daveIdentityLocator).RequestTime) + assert.True(t, endorsementStatusForParty(daveIdentityLocator).EndorsementReceived) + + // Status string `json:"status"` + // + // LatestEvent string `json:"latestEvent"` + // LatestError string `json:"latestError"` + // Endorsements []PrivateTxEndorsementStatus `json:"endorsements"` +} + type fakeClock struct { timePassed time.Duration } diff --git a/core/go/internal/privatetxnmgr/transport_receiver.go b/core/go/internal/privatetxnmgr/transport_receiver.go index 488bd6a6f..3d82092a4 100644 --- a/core/go/internal/privatetxnmgr/transport_receiver.go +++ b/core/go/internal/privatetxnmgr/transport_receiver.go @@ -22,12 +22,8 @@ import ( "github.com/kaleido-io/paladin/toolkit/pkg/log" ) -// If we had lots of these we would probably want to centralize the assignment of the constants to avoid duplication -// but currently there is only 2 ( the other being IDENTITY_RESOLVER_DESTINATION ) -const PRIVATE_TX_MANAGER_DESTINATION = "private-tx-manager" - func (p *privateTxManager) Destination() string { - return PRIVATE_TX_MANAGER_DESTINATION + return components.PRIVATE_TX_MANAGER_DESTINATION } func (p *privateTxManager) ReceiveTransportMessage(ctx context.Context, message *components.TransportMessage) { @@ -40,11 +36,23 @@ func (p *privateTxManager) ReceiveTransportMessage(ctx context.Context, message switch message.MessageType { case "EndorsementRequest": - go p.handleEndorsementRequest(ctx, messagePayload, replyToDestination) + go p.handleEndorsementRequest(p.ctx, messagePayload, replyToDestination) case "EndorsementResponse": - go p.handleEndorsementResponse(ctx, messagePayload) + go p.handleEndorsementResponse(p.ctx, messagePayload) case "DelegationRequest": - go p.handleDelegationRequest(ctx, messagePayload) + go p.handleDelegationRequest(p.ctx, messagePayload, replyToDestination) + case "DelegationRequestAcknowledgment": + go p.handleDelegationRequestAcknowledgment(p.ctx, messagePayload) + case "AssembleRequest": + go p.handleAssembleRequest(p.ctx, messagePayload, replyToDestination) + case "AssembleResponse": + go p.handleAssembleResponse(p.ctx, messagePayload) + case "AssembleError": + go p.handleAssembleError(p.ctx, messagePayload) + case "StateProducedEvent": + go p.handleStateProducedEvent(p.ctx, messagePayload, replyToDestination) + case "StateAcknowledgedEvent": + go p.stateDistributer.HandleStateAcknowledgedEvent(p.ctx, message.Payload) default: log.L(ctx).Errorf("Unknown message type: %s", message.MessageType) } diff --git a/core/go/internal/privatetxnmgr/transport_writer.go b/core/go/internal/privatetxnmgr/transport_writer.go index 7e71280c6..4c4612ee2 100644 --- a/core/go/internal/privatetxnmgr/transport_writer.go +++ b/core/go/internal/privatetxnmgr/transport_writer.go @@ -19,6 +19,7 @@ import ( "context" "encoding/json" + "github.com/google/uuid" "github.com/kaleido-io/paladin/core/internal/components" engineProto "github.com/kaleido-io/paladin/core/pkg/proto/engine" pb "github.com/kaleido-io/paladin/core/pkg/proto/engine" @@ -50,7 +51,7 @@ func (tw *transportWriter) SendDelegationRequest( delegationId string, delegateNodeId string, transaction *components.PrivateTransaction, - + blockHeight int64, ) error { transactionBytes, err := json.Marshal(transaction) @@ -64,6 +65,7 @@ func (tw *transportWriter) SendDelegationRequest( TransactionId: transaction.ID.String(), DelegateNodeId: delegateNodeId, PrivateTransaction: transactionBytes, + BlockHeight: blockHeight, } delegationRequestBytes, err := proto.Marshal(delegationRequest) if err != nil { @@ -74,7 +76,7 @@ func (tw *transportWriter) SendDelegationRequest( if err = tw.transportManager.Send(ctx, &components.TransportMessage{ MessageType: "DelegationRequest", Payload: delegationRequestBytes, - Component: PRIVATE_TX_MANAGER_DESTINATION, + Component: components.PRIVATE_TX_MANAGER_DESTINATION, Node: delegateNodeId, ReplyTo: tw.nodeID, }); err != nil { @@ -83,8 +85,41 @@ func (tw *transportWriter) SendDelegationRequest( return nil } +func (tw *transportWriter) SendDelegationRequestAcknowledgment( + ctx context.Context, + delegatingNodeName string, + delegationId string, + delegateNodeName string, + transactionID string, + +) error { + + delegationRequestAcknowledgment := &pb.DelegationRequestAcknowledgment{ + DelegationId: delegationId, + TransactionId: transactionID, + DelegateNodeId: delegateNodeName, + ContractAddress: tw.contractAddress.String(), + } + delegationRequestAcknowledgmentBytes, err := proto.Marshal(delegationRequestAcknowledgment) + if err != nil { + log.L(ctx).Errorf("Error marshalling delegationRequestAcknowledgment message: %s", err) + return err + } + + if err = tw.transportManager.Send(ctx, &components.TransportMessage{ + MessageType: "DelegationRequestAcknowledgment", + Payload: delegationRequestAcknowledgmentBytes, + Component: components.PRIVATE_TX_MANAGER_DESTINATION, + Node: delegatingNodeName, + ReplyTo: tw.nodeID, + }); err != nil { + return err + } + return nil +} + // TODO do we have duplication here? contractAddress and transactionID are in the transactionSpecification -func (tw *transportWriter) SendEndorsementRequest(ctx context.Context, party string, targetNode string, contractAddress string, transactionID string, attRequest *prototk.AttestationRequest, transactionSpecification *prototk.TransactionSpecification, verifiers []*prototk.ResolvedVerifier, signatures []*prototk.AttestationResult, inputStates []*components.FullState, outputStates []*components.FullState, infoStates []*components.FullState) error { +func (tw *transportWriter) SendEndorsementRequest(ctx context.Context, idempotencyKey string, party string, targetNode string, contractAddress string, transactionID string, attRequest *prototk.AttestationRequest, transactionSpecification *prototk.TransactionSpecification, verifiers []*prototk.ResolvedVerifier, signatures []*prototk.AttestationResult, inputStates []*components.FullState, outputStates []*components.FullState, infoStates []*components.FullState) error { attRequestAny, err := anypb.New(attRequest) if err != nil { log.L(ctx).Error("Error marshalling attestation request", err) @@ -149,6 +184,7 @@ func (tw *transportWriter) SendEndorsementRequest(ctx context.Context, party str } endorsementRequest := &engineProto.EndorsementRequest{ + IdempotencyKey: idempotencyKey, ContractAddress: contractAddress, TransactionId: transactionID, AttestationRequest: attRequestAny, @@ -169,9 +205,47 @@ func (tw *transportWriter) SendEndorsementRequest(ctx context.Context, party str err = tw.transportManager.Send(ctx, &components.TransportMessage{ MessageType: "EndorsementRequest", Node: targetNode, - Component: PRIVATE_TX_MANAGER_DESTINATION, + Component: components.PRIVATE_TX_MANAGER_DESTINATION, ReplyTo: tw.nodeID, Payload: endorsementRequestBytes, }) return err } + +func (tw *transportWriter) SendAssembleRequest(ctx context.Context, assemblingNode string, assembleRequestID string, txID uuid.UUID, contractAddress string, transactionInputs *components.TransactionInputs, preAssembly *components.TransactionPreAssembly, stateLocksJSON []byte, blockHeight int64) error { + + transactionInputsBytes, err := json.Marshal(transactionInputs) + if err != nil { + log.L(ctx).Error("Error marshalling transaction inputs", err) + return err + } + + preAssemblyBytes, err := json.Marshal(preAssembly) + if err != nil { + log.L(ctx).Error("Error marshalling preassembly", err) + return err + } + + assembleRequest := &engineProto.AssembleRequest{ + TransactionId: txID.String(), + AssembleRequestId: assembleRequestID, + ContractAddress: contractAddress, + TransactionInputs: transactionInputsBytes, + PreAssembly: preAssemblyBytes, + StateLocks: stateLocksJSON, + BlockHeight: blockHeight, + } + assembleRequestBytes, err := proto.Marshal(assembleRequest) + if err != nil { + log.L(ctx).Error("Error marshalling assemble request", err) + return err + } + err = tw.transportManager.Send(ctx, &components.TransportMessage{ + MessageType: "AssembleRequest", + Node: assemblingNode, + Component: components.PRIVATE_TX_MANAGER_DESTINATION, + ReplyTo: tw.nodeID, + Payload: assembleRequestBytes, + }) + return err +} diff --git a/core/go/internal/publictxmgr/balance_manager.go b/core/go/internal/publictxmgr/balance_manager.go index 8ed2e09ad..ba90a7369 100644 --- a/core/go/internal/publictxmgr/balance_manager.go +++ b/core/go/internal/publictxmgr/balance_manager.go @@ -234,7 +234,7 @@ func (af *BalanceManagerWithInMemoryTracking) TransferGasFromAutoFuelingSource(c } } if fuelingTx != nil { - completed, err := af.pubTxMgr.CheckTransactionCompleted(ctx, fuelingTx.From, fuelingTx.Nonce.Uint64()) + completed, err := af.pubTxMgr.CheckTransactionCompleted(ctx, *fuelingTx.LocalID) if err != nil { return nil, err } @@ -276,7 +276,7 @@ func (af *BalanceManagerWithInMemoryTracking) TransferGasFromAutoFuelingSource(c // 2) Perform transaction to transfer value to the dest address log.L(ctx).Debugf("TransferGasFromAutoFuelingSource submitting a fueling tx for destination address: %s ", destAddress) - submission, err := af.pubTxMgr.SingleTransactionSubmit(ctx, &components.PublicTxSubmission{ + fuelingTx, err = af.pubTxMgr.SingleTransactionSubmit(ctx, &components.PublicTxSubmission{ PublicTxInput: pldapi.PublicTxInput{ From: af.sourceAddress, To: &destAddress, @@ -290,7 +290,6 @@ func (af *BalanceManagerWithInMemoryTracking) TransferGasFromAutoFuelingSource(c log.L(ctx).Errorf("TransferGasFromAutoFuelingSource fueling tx submission for destination address: %s failed due to: %+v", destAddress, err) return nil, err } - fuelingTx = submission.PublicTx() log.L(ctx).Debugf("TransferGasFromAutoFuelingSource tracking fueling tx with from=%s nonce=%d, for destination address: %s ", fuelingTx.From, fuelingTx.Nonce, destAddress) // start tracking the new transactions af.trackedFuelingTransactions[destAddress] = fuelingTx diff --git a/core/go/internal/publictxmgr/balance_manager_test.go b/core/go/internal/publictxmgr/balance_manager_test.go index 7522df121..06d870120 100644 --- a/core/go/internal/publictxmgr/balance_manager_test.go +++ b/core/go/internal/publictxmgr/balance_manager_test.go @@ -264,24 +264,15 @@ func TestTopUpAddressNoOpScenarios(t *testing.T) { } -func generateExpectedFuelingTransaction(idx int, amountToTransfer uint64, from, to tktypes.EthAddress) *pldapi.PublicTx { - gas := tktypes.HexUint64(10) - return &pldapi.PublicTx{ - From: from, - To: &to, - Nonce: tktypes.HexUint64(mockBaseNonce) + tktypes.HexUint64(idx), // fixed mock when disableManagerStart set - PublicTxOptions: pldapi.PublicTxOptions{ - Gas: &gas, - Value: tktypes.Uint64ToUint256(amountToTransfer), - }, - } +func expectFuelingEqual(t *testing.T, fuelingTx *pldapi.PublicTx, amountToTransfer uint64, from, to tktypes.EthAddress) { + assert.Equal(t, from, fuelingTx.From) + assert.Equal(t, to, *fuelingTx.To) + assert.Equal(t, *tktypes.Uint64ToUint256(amountToTransfer), *fuelingTx.Value) } func mockAutoFuelTransactionSubmit(m *mocksAndTestControl, bm *BalanceManagerWithInMemoryTracking, uncachedBalance bool) { // Then insert of the auto-fueling transaction - m.db.ExpectBegin() m.db.ExpectExec("INSERT.*public_txns").WillReturnResult(driver.ResultNoRows) - m.db.ExpectCommit() if uncachedBalance { // Mock the sufficient balance on the auto-fueling source address, and the nonce assignment @@ -316,10 +307,9 @@ func TestTopUpWithNoAmountModificationWithMultipleFuelingTxs(t *testing.T) { mockAutoFuelTransactionSubmit(m, bm, true) expectedTopUpAmount := big.NewInt(100) - expectedFuelingTransaction1 := generateExpectedFuelingTransaction(0, expectedTopUpAmount.Uint64(), *bm.sourceAddress, testDestAddress) fuelingTx, err := bm.TopUpAccount(ctx, accountToTopUp) require.NoError(t, err) - assert.Equal(t, expectedFuelingTransaction1, fuelingTx) + expectFuelingEqual(t, fuelingTx, expectedTopUpAmount.Uint64(), *bm.sourceAddress, testDestAddress) // Test no new fueling transaction when the current one is pending accountToTopUp2 := &AddressAccount{ @@ -333,25 +323,24 @@ func TestTopUpWithNoAmountModificationWithMultipleFuelingTxs(t *testing.T) { // return not yet completed, so should return the existing pending transaction m.db.ExpectQuery("SELECT.*public_txns"). - WillReturnRows(sqlmock.NewRows([]string{"from", "nonce"}).AddRow( - expectedFuelingTransaction1.From, expectedFuelingTransaction1.Nonce, + WillReturnRows(sqlmock.NewRows([]string{"from"}).AddRow( + *bm.sourceAddress, )) newFuelingTx, err := bm.TopUpAccount(ctx, accountToTopUp2) require.NoError(t, err) - assert.Equal(t, expectedFuelingTransaction1, newFuelingTx) + expectFuelingEqual(t, newFuelingTx, expectedTopUpAmount.Uint64(), *bm.sourceAddress, testDestAddress) // current transaction completed, replace with new transaction expectedTopUpAmount2 := big.NewInt(50) - expectedFuelingTransaction2 := generateExpectedFuelingTransaction(1, expectedTopUpAmount2.Uint64(), *bm.sourceAddress, testDestAddress) - m.db.ExpectQuery("SELECT.*public_txns").WillReturnRows(sqlmock.NewRows([]string{"from", "nonce", `Completed__tx_hash`}). - AddRow(expectedFuelingTransaction1.From, expectedFuelingTransaction1.Nonce, tktypes.Bytes32(tktypes.RandBytes(32)))) + m.db.ExpectQuery("SELECT.*public_txns").WillReturnRows(sqlmock.NewRows([]string{"from", `Completed__tx_hash`}). + AddRow(*bm.sourceAddress, tktypes.Bytes32(tktypes.RandBytes(32)))) mockAutoFuelTransactionSubmit(m, bm, false) fuelingTx2, err := bm.TopUpAccount(ctx, accountToTopUp2) require.NoError(t, err) - assert.Equal(t, expectedFuelingTransaction2, fuelingTx2) + expectFuelingEqual(t, fuelingTx2, expectedTopUpAmount2.Uint64(), *bm.sourceAddress, testDestAddress) // test when couldn't record the result of the submitted transaction // also do a balance look up @@ -364,12 +353,11 @@ func TestTopUpWithNoAmountModificationWithMultipleFuelingTxs(t *testing.T) { MaxCost: big.NewInt(50), } expectedTopUpAmount3 := big.NewInt(50) - expectedFuelingTransaction3 := generateExpectedFuelingTransaction(2, expectedTopUpAmount3.Uint64(), *bm.sourceAddress, testDestAddress) bm.NotifyAddressBalanceChanged(ctx, *bm.sourceAddress) m.ethClient.On("GetBalance", mock.Anything, *bm.sourceAddress, "latest").Return(tktypes.Uint64ToUint256(50), nil).Once() - m.db.ExpectQuery("SELECT.*public_txns").WillReturnRows(sqlmock.NewRows([]string{"from", "nonce", `Completed__tx_hash`}). - AddRow(expectedFuelingTransaction2.From, expectedFuelingTransaction2.Nonce, tktypes.Bytes32(tktypes.RandBytes(32)))) + m.db.ExpectQuery("SELECT.*public_txns").WillReturnRows(sqlmock.NewRows([]string{"from", `Completed__tx_hash`}). + AddRow(*bm.sourceAddress, tktypes.Bytes32(tktypes.RandBytes(32)))) m.ethClient.On("EstimateGasNoResolve", mock.Anything, mock.Anything, mock.Anything). Return(ethclient.EstimateGasResult{}, fmt.Errorf("pop")).Once() @@ -382,14 +370,14 @@ func TestTopUpWithNoAmountModificationWithMultipleFuelingTxs(t *testing.T) { // test that we can recover if the transaction was actually registered in DB // also do a address balance re-lookup m.db.ExpectQuery("SELECT.*public_txns"). - WillReturnRows(sqlmock.NewRows([]string{"from", "nonce"}).AddRow( - expectedFuelingTransaction3.From, expectedFuelingTransaction3.Nonce, + WillReturnRows(sqlmock.NewRows([]string{"from", "to", "value"}).AddRow( + *bm.sourceAddress, testDestAddress, (*tktypes.HexUint256)(expectedTopUpAmount3), )) - m.db.ExpectQuery("SELECT.*public_txns").WillReturnRows(sqlmock.NewRows([]string{"from", "nonce", `Completed__tx_hash`}). - AddRow(expectedFuelingTransaction3.From, expectedFuelingTransaction3.Nonce, nil /* incomplete */)) + m.db.ExpectQuery("SELECT.*public_txns").WillReturnRows(sqlmock.NewRows([]string{"from", "to", "value", `Completed__tx_hash`}). + AddRow(*bm.sourceAddress, testDestAddress, (*tktypes.HexUint256)(expectedTopUpAmount3), nil /* incomplete */)) fuelingTx3, err := bm.TopUpAccount(ctx, accountToTopUp3) require.NoError(t, err) - assert.Equal(t, expectedFuelingTransaction3.Nonce, fuelingTx3.Nonce) + expectFuelingEqual(t, fuelingTx3, expectedTopUpAmount3.Uint64(), *bm.sourceAddress, testDestAddress) } func TestTopUpSuccessTopUpMinAheadUseMin(t *testing.T) { @@ -420,10 +408,9 @@ func TestTopUpSuccessTopUpMinAheadUseMin(t *testing.T) { // the expectTopUpAmount should include min Value (50) multiply 2 extra space we set expectedTopUpAmount := big.NewInt(200) - expectedFuelingTransaction1 := generateExpectedFuelingTransaction(0, expectedTopUpAmount.Uint64(), *bm.sourceAddress, testDestAddress) fuelingTx, err := bm.TopUpAccount(ctx, accountToTopUp) require.NoError(t, err) - assert.Equal(t, expectedFuelingTransaction1, fuelingTx) + expectFuelingEqual(t, fuelingTx, expectedTopUpAmount.Uint64(), *bm.sourceAddress, testDestAddress) } @@ -455,10 +442,9 @@ func TestTopUpSuccessTopUpMinAheadUseMax(t *testing.T) { // the expectTopUpAmount should include max Value (150) multiply 2 extra space we set expectedTopUpAmount := big.NewInt(400) - expectedFuelingTransaction1 := generateExpectedFuelingTransaction(0, expectedTopUpAmount.Uint64(), *bm.sourceAddress, testDestAddress) fuelingTx, err := bm.TopUpAccount(ctx, accountToTopUp) require.NoError(t, err) - assert.Equal(t, expectedFuelingTransaction1, fuelingTx) + expectFuelingEqual(t, fuelingTx, expectedTopUpAmount.Uint64(), *bm.sourceAddress, testDestAddress) } @@ -490,10 +476,9 @@ func TestTopUpSuccessTopUpMinAheadUseAvg(t *testing.T) { // the expectTopUpAmount should include avg Value (100) multiply 2 extra space we set expectedTopUpAmount := big.NewInt(300) - expectedFuelingTransaction1 := generateExpectedFuelingTransaction(0, expectedTopUpAmount.Uint64(), *bm.sourceAddress, testDestAddress) fuelingTx, err := bm.TopUpAccount(ctx, accountToTopUp) require.NoError(t, err) - assert.Equal(t, expectedFuelingTransaction1, fuelingTx) + expectFuelingEqual(t, fuelingTx, expectedTopUpAmount.Uint64(), *bm.sourceAddress, testDestAddress) } @@ -522,10 +507,9 @@ func TestTopUpSuccessUseMinDestBalance(t *testing.T) { bm.minDestBalance = big.NewInt(250) expectedTopUpAmount := big.NewInt(150) - expectedFuelingTransaction1 := generateExpectedFuelingTransaction(0, expectedTopUpAmount.Uint64(), *bm.sourceAddress, testDestAddress) fuelingTx, err := bm.TopUpAccount(ctx, accountToTopUp) require.NoError(t, err) - assert.Equal(t, expectedFuelingTransaction1, fuelingTx) + expectFuelingEqual(t, fuelingTx, expectedTopUpAmount.Uint64(), *bm.sourceAddress, testDestAddress) } func TestTopUpSuccessUseMaxDestBalance(t *testing.T) { @@ -553,10 +537,9 @@ func TestTopUpSuccessUseMaxDestBalance(t *testing.T) { bm.maxDestBalance = big.NewInt(150) expectedTopUpAmount := big.NewInt(50) - expectedFuelingTransaction1 := generateExpectedFuelingTransaction(0, expectedTopUpAmount.Uint64(), *bm.sourceAddress, testDestAddress) fuelingTx, err := bm.TopUpAccount(ctx, accountToTopUp) require.NoError(t, err) - assert.Equal(t, expectedFuelingTransaction1, fuelingTx) + expectFuelingEqual(t, fuelingTx, expectedTopUpAmount.Uint64(), *bm.sourceAddress, testDestAddress) } func TestTopUpNoOpAlreadyAboveMaxDestBalance(t *testing.T) { diff --git a/core/go/internal/publictxmgr/in_flight_transaction_stage_controller.go b/core/go/internal/publictxmgr/in_flight_transaction_stage_controller.go index c01dad409..7c6b1c1e1 100644 --- a/core/go/internal/publictxmgr/in_flight_transaction_stage_controller.go +++ b/core/go/internal/publictxmgr/in_flight_transaction_stage_controller.go @@ -315,7 +315,8 @@ func (it *inFlightTransactionStageController) ProduceLatestInFlightStageContext( rsc.InMemoryTx.GetGasPriceObject() gasPriceJSON, _ := json.Marshal(rsc.InMemoryTx.GetGasPriceObject()) rsc.StageOutputsToBePersisted.TxUpdates.NewSubmission = &DBPubTxnSubmission{ - SignerNonce: rsc.InMemoryTx.GetSignerNonce(), + from: rsc.InMemoryTx.GetFrom().String(), + PublicTxnID: rsc.InMemoryTx.GetPubTxnID(), Created: tktypes.TimestampNow(), TransactionHash: *rsc.StageOutput.SignOutput.TxHash, GasPricing: gasPriceJSON, diff --git a/core/go/internal/publictxmgr/in_flight_transaction_state_manager_test.go b/core/go/internal/publictxmgr/in_flight_transaction_state_manager_test.go index 14c9c3231..80847ca79 100644 --- a/core/go/internal/publictxmgr/in_flight_transaction_state_manager_test.go +++ b/core/go/internal/publictxmgr/in_flight_transaction_state_manager_test.go @@ -368,7 +368,7 @@ func TestStateManagerTxPersistenceManagementUpdateErrors(t *testing.T) { rsc.StageOutputsToBePersisted.TxUpdates = &BaseTXUpdates{ NewSubmission: &DBPubTxnSubmission{ - SignerNonce: "signer:12345", + from: "0x12345", TransactionHash: tktypes.Bytes32(tktypes.RandBytes(32)), }, } diff --git a/core/go/internal/publictxmgr/in_memory_tx_state.go b/core/go/internal/publictxmgr/in_memory_tx_state.go index 4f849fbfd..235ae37b5 100644 --- a/core/go/internal/publictxmgr/in_memory_tx_state.go +++ b/core/go/internal/publictxmgr/in_memory_tx_state.go @@ -17,6 +17,8 @@ package publictxmgr import ( "context" + "fmt" + "strconv" "time" "github.com/hyperledger/firefly-signer/pkg/ethsigner" @@ -109,8 +111,16 @@ func (imtxs *inMemoryTxState) ApplyInMemoryUpdates(ctx context.Context, txUpdate } } +func (imtxs *inMemoryTxState) GetPubTxnID() uint64 { + return imtxs.mtx.ptx.PublicTxnID +} + func (imtxs *inMemoryTxState) GetSignerNonce() string { - return imtxs.mtx.ptx.SignerNonce + nonceStr := "unassigned" + if imtxs.mtx.ptx.Nonce != nil { + nonceStr = strconv.FormatUint(*imtxs.mtx.ptx.Nonce, 10) + } + return fmt.Sprintf("%s:%s", imtxs.mtx.ptx.From, nonceStr) } func (imtxs *inMemoryTxState) GetCreatedTime() *tktypes.Timestamp { @@ -122,7 +132,7 @@ func (imtxs *inMemoryTxState) GetTransactionHash() *tktypes.Bytes32 { } func (imtxs *inMemoryTxState) GetNonce() uint64 { - return imtxs.mtx.ptx.Nonce + return *imtxs.mtx.ptx.Nonce } func (imtxs *inMemoryTxState) GetFrom() tktypes.EthAddress { @@ -142,7 +152,7 @@ func (imtxs *inMemoryTxState) BuildEthTX() *ethsigner.Transaction { ptx := imtxs.mtx.ptx return buildEthTX( ptx.From, - &ptx.Nonce, + ptx.Nonce, ptx.To, ptx.Data, &pldapi.PublicTxOptions{ diff --git a/core/go/internal/publictxmgr/in_memory_tx_state_test.go b/core/go/internal/publictxmgr/in_memory_tx_state_test.go index afaa454ef..a1f37186c 100644 --- a/core/go/internal/publictxmgr/in_memory_tx_state_test.go +++ b/core/go/internal/publictxmgr/in_memory_tx_state_test.go @@ -59,7 +59,7 @@ func NewTestInMemoryTxState(t *testing.T) InMemoryTxStateManager { Created: oldTime, From: *oldFrom, To: oldTo, - Nonce: oldNonce.Uint64(), + Nonce: (*uint64)(&oldNonce), Gas: oldGasLimit.Uint64(), Value: oldValue, Data: oldTransactionData, @@ -92,14 +92,13 @@ func TestSettersAndGetters(t *testing.T) { oldTransactionData := tktypes.MustParseHexBytes(testTransactionData) testManagedTx := &DBPublicTxn{ - SignerNonce: fmt.Sprintf("%s:%d", oldFrom, oldNonce), - Created: oldTime, - From: *oldFrom, - To: oldTo, - Nonce: oldNonce.Uint64(), - Gas: uint64(oldGasLimit), - Value: oldValue, - Data: tktypes.HexBytes(oldTransactionData), + Created: oldTime, + From: *oldFrom, + To: oldTo, + Nonce: (*uint64)(&oldNonce), + Gas: uint64(oldGasLimit), + Value: oldValue, + Data: tktypes.HexBytes(oldTransactionData), } imts := NewInMemoryTxStateManager(context.Background(), testManagedTx) diff --git a/core/go/internal/publictxmgr/nonces.go b/core/go/internal/publictxmgr/nonces.go deleted file mode 100644 index 735013a0e..000000000 --- a/core/go/internal/publictxmgr/nonces.go +++ /dev/null @@ -1,238 +0,0 @@ -/* - * Copyright © 2024 Kaleido, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ - -package publictxmgr - -import ( - "context" - "sync" - "time" - - "github.com/hyperledger/firefly-common/pkg/i18n" - "github.com/kaleido-io/paladin/core/internal/msgs" - "github.com/kaleido-io/paladin/toolkit/pkg/log" - "github.com/kaleido-io/paladin/toolkit/pkg/tktypes" -) - -type NextNonceCallback func(ctx context.Context, signer tktypes.EthAddress) (uint64, error) - -type NonceAssignmentIntent interface { - Complete(ctx context.Context) - AssignNextNonce(ctx context.Context) (uint64, error) - Address() tktypes.EthAddress - Rollback(ctx context.Context) -} - -type NonceCache interface { - IntentToAssignNonce(ctx context.Context, signer tktypes.EthAddress) (NonceAssignmentIntent, error) - Stop() -} - -type nonceCacheStruct struct { - nextNonceBySigner map[tktypes.EthAddress]*cachedNonce - nextNonceCB NextNonceCallback - nonceStateTimeout time.Duration - reaperLock sync.RWMutex //if this proves to be a bottleneck, we could maintain a finer grained lock on each cache entry but would be more complex and error prone - inserterLock sync.Mutex //we should only ever grab this lock if we have a reader lock on the reaperLock otherwise we could cause a deadlock - mapMux sync.Mutex // only held for a short time during setNextNonceBySigner and getNextNonceBySigner. never attempt to take any other lock while holding this one - stopChannel chan struct{} -} - -func (nc *nonceCacheStruct) Stop() { - close(nc.stopChannel) -} - -func newNonceCache(nonceStateTimeout time.Duration, nextNonceCB NextNonceCallback) NonceCache { - n := &nonceCacheStruct{ - nextNonceBySigner: make(map[tktypes.EthAddress]*cachedNonce), - nonceStateTimeout: nonceStateTimeout, - stopChannel: make(chan struct{}), - nextNonceCB: nextNonceCB, - } - if nonceStateTimeout > 0 { - go n.reapLoop() - } - return n -} - -type cachedNonce struct { - nonceMux sync.Mutex - signer tktypes.EthAddress - value uint64 - updatedTime time.Time -} - -func (nc *nonceCacheStruct) reapLoop() { - ctx := log.WithLogField(context.Background(), "role", "nonce_cache_reaper") - ticker := time.NewTicker(nc.nonceStateTimeout) - for { - select { - case <-nc.stopChannel: - ticker.Stop() - return - case <-ticker.C: - nc.reap(ctx) - } - } -} - -func (nc *nonceCacheStruct) reap(ctx context.Context) { - nc.reaperLock.Lock() - defer nc.reaperLock.Unlock() - now := time.Now() - reapCount := 0 - for signingAddress, cachedNonce := range nc.nextNonceBySigner { - if now.Sub(cachedNonce.updatedTime) > nc.nonceStateTimeout { - reapCount++ - delete(nc.nextNonceBySigner, signingAddress) - } - } - log.L(ctx).Debugf("nonce cache reaper completed on ticker - reap count %d", reapCount) -} - -func (nc *nonceCacheStruct) getNextNonceBySigner(signer tktypes.EthAddress) (*cachedNonce, bool) { - nc.mapMux.Lock() - defer nc.mapMux.Unlock() - result, found := nc.nextNonceBySigner[signer] - return result, found -} - -func (nc *nonceCacheStruct) setNextNonceBySigner(signer tktypes.EthAddress, record *cachedNonce) { - nc.mapMux.Lock() - defer nc.mapMux.Unlock() - nc.nextNonceBySigner[signer] = record -} - -// Declare an intent to assign a nonce. -// This will ensure that we have a fresh copy of the nonce in memory so that -// the caller can be sure that the nonce assignment step will not suffer the latency of reading the database or calling -// out to the block chain node -// It is the callers responsibility to call `Complete` on the returned object -// The nonce cache for the given signing address is guaranteed not to be reaped after IntentToAssignNonce returns and before `complete` -// NOTE: multiple readers can hold intents to assign concurrently so the nonce is not actually assigned at this point -// -// nonce assignment itself is protected by a mutex so only one reader can assign at a time but thanks to the pre intent declaration, the assignment is quick -func (nc *nonceCacheStruct) IntentToAssignNonce(ctx context.Context, signer tktypes.EthAddress) (NonceAssignmentIntent, error) { - - // take a read lock to block the reaper thread - nc.reaperLock.RLock() - - cachedNonceRecord, isCached := nc.getNextNonceBySigner(signer) - if !isCached { - //we only ever grab the inserterLock if we already have a read lock otherwise there could be a deadlock - nc.inserterLock.Lock() - defer nc.inserterLock.Unlock() - - //double check in case another thread managed to get in while we were waiting for the inserterLock - cachedNonceRecord, isCached = nc.getNextNonceBySigner(signer) - - if !isCached { - - nextNonce, err := nc.nextNonceCB(ctx, signer) - if err != nil { - log.L(ctx).Errorf("failed to get next nonce") - return nil, err - } - - cachedNonceRecord = &cachedNonce{ - value: nextNonce, - signer: signer, - updatedTime: time.Now(), - } - - nc.setNextNonceBySigner(signer, cachedNonceRecord) - } - - } - return &nonceAssignmentIntent{ - nc: nc, - addr: signer, - locked: false, - completed: false, - cachedNonce: cachedNonceRecord, - nonceCache: nc, - }, nil -} - -func (i *nonceAssignmentIntent) Address() tktypes.EthAddress { - return i.addr -} - -type nonceAssignmentIntent struct { - nc *nonceCacheStruct - addr tktypes.EthAddress - locked bool - completed bool - cachedNonce *cachedNonce - nonceCache *nonceCacheStruct - initialValue uint64 -} - -// AssignNextNonce returns the next nonce to be used by the caller and obtains a lock on the nonce -// the caller is responsible for calling Complete or Rollback when they are confident that they will or will not use that nonce -// (i.e. typically just after a database transaction has been committed or rolled back) -// This means that any other callers will be blocked until the current caller calls Complete or Rollback -// if there is a need to obtain multiple nonces on the same signing address as part of the same database transaction, -// then a single NonceAssigmentIntent shoudld be shared and AssignNextNonce should be called -// multiple times on that single intent -// It is invalid to call AssignNextNonce after calling Complete or Rollback -func (i *nonceAssignmentIntent) AssignNextNonce(ctx context.Context) (uint64, error) { - if i.completed { - return 0, i18n.NewError(ctx, msgs.MsgPublicBatchCompleted) - } - if !i.locked { - i.cachedNonce.nonceMux.Lock() - //once we have the lock, take a copy of the first value we see so that we can roll back to it if needed - i.initialValue = i.cachedNonce.value - i.locked = true - } - value := i.cachedNonce.value - i.cachedNonce.value = i.cachedNonce.value + 1 - return value, nil -} - -func (i *nonceAssignmentIntent) Complete(ctx context.Context) { - //If we never took the lock or if we have already completed, then this is a no-op - if !i.completed && i.locked { - i.cachedNonce.updatedTime = time.Now() - i.cachedNonce.nonceMux.Unlock() - } - if !i.completed { - i.nonceCache.reaperLock.RUnlock() - } - i.completed = true - if i.nc.nonceStateTimeout == 0 { - // need to reap immediately - i.nc.reap(ctx) - } -} - -// If Rollback is called after a Complete or another Rollback, it will be a no-op -// thus it is safe to defer Rollback as soon as you have the intent -func (i *nonceAssignmentIntent) Rollback(ctx context.Context) { - - //unlock rollback to previous value - - //If we never took the lock or if we have already completed, then this is a no-op - if !i.completed && i.locked { - i.cachedNonce.value = i.initialValue - i.cachedNonce.nonceMux.Unlock() - } - if !i.completed { - i.nonceCache.reaperLock.RUnlock() - } - i.completed = true - -} diff --git a/core/go/internal/publictxmgr/nonces_test.go b/core/go/internal/publictxmgr/nonces_test.go deleted file mode 100644 index 29b5c5dfa..000000000 --- a/core/go/internal/publictxmgr/nonces_test.go +++ /dev/null @@ -1,645 +0,0 @@ -/* - * Copyright © 2024 Kaleido, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - */ - -package publictxmgr - -import ( - "context" - "database/sql/driver" - "math/rand" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/kaleido-io/paladin/config/pkg/confutil" - "github.com/kaleido-io/paladin/config/pkg/pldconf" - "github.com/kaleido-io/paladin/core/internal/components" - - "github.com/kaleido-io/paladin/toolkit/pkg/pldapi" - "github.com/kaleido-io/paladin/toolkit/pkg/tktypes" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/require" - "gorm.io/gorm" -) - -func TestIntentToAssignNonce(t *testing.T) { - ctx := context.Background() - callbackHasBeenCalled := false - nonceCache := newNonceCacheForTesting(func(ctx context.Context, signer tktypes.EthAddress) (uint64, error) { - callbackHasBeenCalled = true - return uint64(42), nil - }) - defer nonceCache.Stop() - signer := tktypes.EthAddress(tktypes.RandBytes(20)) - intent, err := nonceCache.IntentToAssignNonce(ctx, signer) - require.NoError(t, err) - assert.True(t, callbackHasBeenCalled) - intent.Complete(ctx) - - //Check that the reapear lock is in a state where the reaper can grab it - gotLock := nonceCache.reaperLock.TryLock() - assert.True(t, gotLock) - if gotLock { - nonceCache.reaperLock.Unlock() - } -} - -func TestAssignNonce(t *testing.T) { - ctx := context.Background() - callbackHasBeenCalled := false - nonceCache := newNonceCacheForTesting(func(ctx context.Context, signer tktypes.EthAddress) (uint64, error) { - callbackHasBeenCalled = true - return uint64(42), nil - }) - defer nonceCache.Stop() - signer := tktypes.EthAddress(tktypes.RandBytes(20)) - intent, err := nonceCache.IntentToAssignNonce(ctx, signer) - require.NoError(t, err) - assert.True(t, callbackHasBeenCalled) - - nextNonce, err := intent.AssignNextNonce(ctx) - require.NoError(t, err) - - assert.Equal(t, uint64(42), nextNonce) - intent.Complete(ctx) - - callbackHasBeenCalled = false - - intent, err = nonceCache.IntentToAssignNonce(ctx, signer) - require.NoError(t, err) - assert.False(t, callbackHasBeenCalled) - - nextNonce, err = intent.AssignNextNonce(ctx) - require.NoError(t, err) - assert.Equal(t, uint64(43), nextNonce) - intent.Complete(ctx) - - //Check that the reapear lock is in a state where the reaper can grab it - gotLock := nonceCache.reaperLock.TryLock() - assert.True(t, gotLock) - if gotLock { - nonceCache.reaperLock.Unlock() - } - -} - -func TestIntentToAssignNonceRollbackNoAssign(t *testing.T) { - ctx := context.Background() - callbackHasBeenCalled := false - nonceCache := newNonceCacheForTesting(func(ctx context.Context, signer tktypes.EthAddress) (uint64, error) { - callbackHasBeenCalled = true - return uint64(42), nil - }) - defer nonceCache.Stop() - signer := tktypes.EthAddress(tktypes.RandBytes(20)) - intent, err := nonceCache.IntentToAssignNonce(ctx, signer) - require.NoError(t, err) - assert.True(t, callbackHasBeenCalled) - intent.Rollback(ctx) - - //check that the nonce is still in memory and if assigned, we get the correct nonce - callbackHasBeenCalled = false - intent, err = nonceCache.IntentToAssignNonce(ctx, signer) - require.NoError(t, err) - assert.False(t, callbackHasBeenCalled) - - nextNonce, err := intent.AssignNextNonce(ctx) - require.NoError(t, err) - assert.Equal(t, uint64(42), nextNonce) - intent.Complete(ctx) - - //Check that the reapear lock is in a state where the reaper can grab it - gotLock := nonceCache.reaperLock.TryLock() - assert.True(t, gotLock) - if gotLock { - nonceCache.reaperLock.Unlock() - } -} - -func TestIntentToAssignNonceCompleteNoAssign(t *testing.T) { - ctx := context.Background() - callbackHasBeenCalled := false - nonceCache := newNonceCacheForTesting(func(ctx context.Context, signer tktypes.EthAddress) (uint64, error) { - callbackHasBeenCalled = true - return uint64(42), nil - }) - defer nonceCache.Stop() - signer := tktypes.EthAddress(tktypes.RandBytes(20)) - intent, err := nonceCache.IntentToAssignNonce(ctx, signer) - require.NoError(t, err) - assert.True(t, callbackHasBeenCalled) - intent.Complete(ctx) - - callbackHasBeenCalled = false - intent, err = nonceCache.IntentToAssignNonce(ctx, signer) - require.NoError(t, err) - assert.False(t, callbackHasBeenCalled) - - nextNonce, err := intent.AssignNextNonce(ctx) - require.NoError(t, err) - assert.Equal(t, uint64(42), nextNonce) - intent.Complete(ctx) - - //Check that the reapear lock is in a state where the reaper can grab it - gotLock := nonceCache.reaperLock.TryLock() - assert.True(t, gotLock) - if gotLock { - nonceCache.reaperLock.Unlock() - } -} - -func TestAssignNonceMultipleNonces(t *testing.T) { - ctx := context.Background() - nonceCache := newNonceCacheForTesting() - defer nonceCache.Stop() - signer := tktypes.EthAddress(tktypes.RandBytes(20)) - intent, err := nonceCache.IntentToAssignNonce(ctx, signer) - require.NoError(t, err) - - nextNonce, err := intent.AssignNextNonce(ctx) - require.NoError(t, err) - - assert.Equal(t, uint64(42), nextNonce) - nextNonce, err = intent.AssignNextNonce(ctx) - require.NoError(t, err) - assert.Equal(t, uint64(43), nextNonce) - nextNonce, err = intent.AssignNextNonce(ctx) - require.NoError(t, err) - assert.Equal(t, uint64(44), nextNonce) - - intent.Complete(ctx) - - intent, err = nonceCache.IntentToAssignNonce(ctx, signer) - require.NoError(t, err) - - nextNonce, err = intent.AssignNextNonce(ctx) - require.NoError(t, err) - assert.Equal(t, uint64(45), nextNonce) - intent.Complete(ctx) - - //Check that the reapear lock is in a state where the reaper can grab it - gotLock := nonceCache.reaperLock.TryLock() - assert.True(t, gotLock) - if gotLock { - nonceCache.reaperLock.Unlock() - } - -} - -func TestAssignNonceRollback(t *testing.T) { - ctx := context.Background() - callbackHasBeenCalled := false - nonceCache := newNonceCacheForTesting(func(ctx context.Context, signer tktypes.EthAddress) (uint64, error) { - callbackHasBeenCalled = true - return uint64(42), nil - }) - defer nonceCache.Stop() - signer := tktypes.EthAddress(tktypes.RandBytes(20)) - intent, err := nonceCache.IntentToAssignNonce(ctx, signer) - require.NoError(t, err) - assert.True(t, callbackHasBeenCalled) - - nextNonce, err := intent.AssignNextNonce(ctx) - require.NoError(t, err) - - assert.Equal(t, uint64(42), nextNonce) - intent.Rollback(ctx) - - callbackHasBeenCalled = false - - intent, err = nonceCache.IntentToAssignNonce(ctx, signer) - require.NoError(t, err) - assert.False(t, callbackHasBeenCalled) - - nextNonce, err = intent.AssignNextNonce(ctx) - require.NoError(t, err) - //should get 42 as before given that the previous assignment got rolled back - assert.Equal(t, uint64(42), nextNonce) - intent.Complete(ctx) - - //Check that the reapear lock is in a state where the reaper can grab it - gotLock := nonceCache.reaperLock.TryLock() - assert.True(t, gotLock) - if gotLock { - nonceCache.reaperLock.Unlock() - } -} - -func TestAssignNonceMultipleNoncesRollback(t *testing.T) { - ctx := context.Background() - nonceCache := newNonceCacheForTesting() - defer nonceCache.Stop() - signer := tktypes.EthAddress(tktypes.RandBytes(20)) - intent, err := nonceCache.IntentToAssignNonce(ctx, signer) - require.NoError(t, err) - - nextNonce, err := intent.AssignNextNonce(ctx) - require.NoError(t, err) - - assert.Equal(t, uint64(42), nextNonce) - nextNonce, err = intent.AssignNextNonce(ctx) - require.NoError(t, err) - assert.Equal(t, uint64(43), nextNonce) - nextNonce, err = intent.AssignNextNonce(ctx) - require.NoError(t, err) - assert.Equal(t, uint64(44), nextNonce) - - intent.Rollback(ctx) - - intent, err = nonceCache.IntentToAssignNonce(ctx, signer) - require.NoError(t, err) - - nextNonce, err = intent.AssignNextNonce(ctx) - require.NoError(t, err) - assert.Equal(t, uint64(42), nextNonce) - intent.Complete(ctx) - - //Check that the reapear lock is in a state where the reaper can grab it - gotLock := nonceCache.reaperLock.TryLock() - assert.True(t, gotLock) - if gotLock { - nonceCache.reaperLock.Unlock() - } -} - -func TestAssignNonceAfterCompleteFail(t *testing.T) { - ctx := context.Background() - - nonceCache := newNonceCacheForTesting() - defer nonceCache.Stop() - signer := tktypes.EthAddress(tktypes.RandBytes(20)) - intent, err := nonceCache.IntentToAssignNonce(ctx, signer) - require.NoError(t, err) - - nextNonce, err := intent.AssignNextNonce(ctx) - require.NoError(t, err) - - assert.Equal(t, uint64(42), nextNonce) - intent.Complete(ctx) - - _, err = intent.AssignNextNonce(ctx) - assert.Error(t, err) - - //Check that the reapear lock is in a state where the reaper can grab it - gotLock := nonceCache.reaperLock.TryLock() - assert.True(t, gotLock) - if gotLock { - nonceCache.reaperLock.Unlock() - } -} - -func TestAssignNonceAfterRollbackFail(t *testing.T) { - ctx := context.Background() - nonceCache := newNonceCacheForTesting() - defer nonceCache.Stop() - signer := tktypes.EthAddress(tktypes.RandBytes(20)) - intent, err := nonceCache.IntentToAssignNonce(ctx, signer) - require.NoError(t, err) - - nextNonce, err := intent.AssignNextNonce(ctx) - require.NoError(t, err) - - assert.Equal(t, uint64(42), nextNonce) - intent.Rollback(ctx) - - _, err = intent.AssignNextNonce(ctx) - assert.Error(t, err) - - //Check that the reapear lock is in a state where the reaper can grab it - gotLock := nonceCache.reaperLock.TryLock() - assert.True(t, gotLock) - if gotLock { - nonceCache.reaperLock.Unlock() - } -} - -func TestAssignNonceMultiThreaded(t *testing.T) { - ctx := context.Background() - callbackCalled := 0 - firstNonce := uint64(42) - nonceCache := newNonceCacheForTesting(func(ctx context.Context, signer tktypes.EthAddress) (uint64, error) { - callbackCalled++ - return firstNonce, nil - }) - iterations := 100 - threads := 100 - rollbacks := 0 - results := make([][]uint64, threads) - for i := 0; i < threads; i++ { - results[i] = make([]uint64, iterations) - } - - signer := tktypes.EthAddress(tktypes.RandBytes(20)) - doneIt := make(chan struct{}, threads) - doIt := func(threadNumber int) { - for iteration := 0; iteration < iterations; iteration++ { - - intent, err := nonceCache.IntentToAssignNonce(ctx, signer) - require.NoError(t, err) - nextNonce, err := intent.AssignNextNonce(ctx) - require.NoError(t, err) - if rand.Intn(10) == 9 { - //rollback on average 10% of the time - results[threadNumber][iteration] = 1 // 1 is a special value meaning rolledback - rollbacks++ - intent.Rollback(ctx) - } else { - results[threadNumber][iteration] = nextNonce - intent.Complete(ctx) - } - } - doneIt <- struct{}{} - } - - for thread := 0; thread < threads; thread++ { - go doIt(thread) - } - threadsDone := 0 - for { - <-doneIt - threadsDone++ - if threadsDone == threads { - break - } - } - - highestNonce := firstNonce + (uint64(iterations) * uint64(threads)) - 1 - uint64(rollbacks) - - //we should have a non gapless set of results and within a given thread, they should be in order - haveSeenFirstNonce := false - haveSeenHighestNonce := false - seen := make([]bool, iterations*threads) - for threadNumber, threadResults := range results { - previousNonce := firstNonce - 1 - for iterationNumber, iterationResult := range threadResults { - if iterationResult != 1 { //wasn't one of the random rollbacks - assert.Greater(t, iterationResult, previousNonce, "nonce %d out of order - not greater than %d on thread %d iteration %d ", iterationResult, previousNonce, threadNumber, iterationNumber) - assert.False(t, seen[iterationResult-firstNonce], "nonce %d used twice on thread %d iteration %d ", iterationResult, threadNumber, iterationNumber) - seen[iterationResult-firstNonce] = true - if iterationResult == firstNonce { - haveSeenFirstNonce = true - } - if iterationResult == highestNonce { - haveSeenHighestNonce = true - } - } - } - } - //given that all nonces are unique and are greater or equal to the first nonce, - // if we have seen the first and highest nonce then it must be gapless - assert.True(t, haveSeenFirstNonce) - assert.True(t, haveSeenHighestNonce) - assert.Equal(t, 1, callbackCalled) - - //Check that the reapear lock is in a state where the reaper can grab it - gotLock := nonceCache.reaperLock.TryLock() - assert.True(t, gotLock) - if gotLock { - nonceCache.reaperLock.Unlock() - } -} - -func TestAssignNonceMultiThreadedMultiSigningAddresses(t *testing.T) { - tests := []struct { - name string - numSigningAddresses int - numThreadsPerSigningAddress int - numItterationsPerThread int - }{ - { - name: "10 signing addresses, 10 threads per signing address, 1000 iterations per thread", - numSigningAddresses: 10, - numThreadsPerSigningAddress: 10, - numItterationsPerThread: 1000, - }, - { - name: "2 signing addresses, 100 threads per signing address, 1000 iterations per thread", - numSigningAddresses: 2, - numThreadsPerSigningAddress: 100, - numItterationsPerThread: 1000, - }, - { - name: "100 signing addresses, 2 threads per signing address, 1000 iterations per thread", - numSigningAddresses: 100, - numThreadsPerSigningAddress: 2, - numItterationsPerThread: 1000, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - - ctx := context.Background() - firstNonce := uint64(42) - //assert that that callback is only called once per signing address - callbackCalled := make(map[tktypes.EthAddress]bool) - nonceCache := newNonceCacheForTesting(func(ctx context.Context, signer tktypes.EthAddress) (uint64, error) { - assert.False(t, callbackCalled[signer]) - callbackCalled[signer] = true - return firstNonce, nil - }) - - numSigningAddresses := tt.numSigningAddresses - numThreadsPerSigningAddress := tt.numThreadsPerSigningAddress - numItterationsPerThread := tt.numItterationsPerThread - - //generate a random hex string for each signing address - signingAddresses := make([]tktypes.EthAddress, numSigningAddresses) - for i := 0; i < numSigningAddresses; i++ { - signingAddresses[i] = tktypes.EthAddress(tktypes.RandBytes(20)) - } - - // we are going to keep a count of how many rollbacks there were for each signing address - //so that we can calculate which nonce number we expect to get up to per signign address - rollbacks := make([]int32, numSigningAddresses) - - //create a 3 dimensional matrix to store the results - results := make([][][]uint64, numSigningAddresses) - for s := 0; s < numSigningAddresses; s++ { - results[s] = make([][]uint64, numThreadsPerSigningAddress) - for t := 0; t < numThreadsPerSigningAddress; t++ { - results[s][t] = make([]uint64, numItterationsPerThread) - } - } - - // inner function to run through a number of iterations on a single thread - runItterationsForThread := func(threadNumber int, signingAddressIndex int, signingAddress tktypes.EthAddress) { - for iteration := 0; iteration < numItterationsPerThread; iteration++ { - intent, err := nonceCache.IntentToAssignNonce(ctx, signingAddress) - defer intent.Rollback(ctx) - require.NoError(t, err) - nextNonce, err := intent.AssignNextNonce(ctx) - require.NoError(t, err) - if rand.Intn(10) == 9 { - //rollback on average 10% of the time - results[signingAddressIndex][threadNumber][iteration] = 1 // 1 is a special value meaning rolledback - atomic.AddInt32(&rollbacks[signingAddressIndex], 1) - intent.Rollback(ctx) - } else { - results[signingAddressIndex][threadNumber][iteration] = nextNonce - intent.Complete(ctx) - } - } - } - - // function to start a number of threads for a given signing address and wait for each of them to complete - runThreadsForSigningAddress := func(signingAddressIndex int, signingAddress tktypes.EthAddress) { - var wg sync.WaitGroup - wg.Add(numThreadsPerSigningAddress) - for thread := 0; thread < numThreadsPerSigningAddress; thread++ { - threadIndex := thread - go func() { - runItterationsForThread(threadIndex, signingAddressIndex, signingAddress) - wg.Done() - }() - } - wg.Wait() - } - - var wg sync.WaitGroup - wg.Add(numSigningAddresses) - for signingAddressIndex, signingAddress := range signingAddresses { - saIndex, sA := signingAddressIndex, signingAddress - go func() { - runThreadsForSigningAddress(saIndex, sA) - wg.Done() - }() - } - wg.Wait() - - //all done, now analyze the results - for signingAddressNumber, signingAddressResults := range results { - highestNonce := firstNonce + (uint64(numItterationsPerThread) * uint64(numThreadsPerSigningAddress)) - 1 - uint64(rollbacks[signingAddressNumber]) - //we should have a non gapless set of results and within a given thread, they should be in order - haveSeenFirstNonce := false - haveSeenHighestNonce := false - seen := make([]bool, numItterationsPerThread*numThreadsPerSigningAddress) - - for threadNumber, threadResults := range signingAddressResults { - previousNonce := firstNonce - 1 - for iterationNumber, iterationResult := range threadResults { - if iterationResult != 1 { //wasn't one of the random rollbacks - assert.Greater(t, iterationResult, previousNonce, "nonce %d out of order - not greater than %d on signing address %d thread %d iteration %d ", iterationResult, signingAddressNumber, previousNonce, threadNumber, iterationNumber) - assert.False(t, seen[iterationResult-firstNonce], "nonce %d used twice on signing address %d thread %d iteration %d ", iterationResult, signingAddressNumber, threadNumber, iterationNumber) - seen[iterationResult-firstNonce] = true - if iterationResult == firstNonce { - haveSeenFirstNonce = true - } - if iterationResult == highestNonce { - haveSeenHighestNonce = true - } - previousNonce = iterationResult - } - } - } - //given that all nonces are unique and are greater or equal to the first nonce, - // if we have seen the first and highest nonce then it must be gapless - assert.True(t, haveSeenFirstNonce, "did not see first nonce %d for signing address %d", firstNonce, signingAddressNumber) - assert.True(t, haveSeenHighestNonce, "did not see highest %d nonce for signing address %d", highestNonce, signingAddressNumber) - assert.True(t, callbackCalled[signingAddresses[signingAddressNumber]], "callback was not called for signing address %d", signingAddressNumber) - } - //Check that the reapear lock is in a state where the reaper can grab it - gotLock := nonceCache.reaperLock.TryLock() - assert.True(t, gotLock) - if gotLock { - nonceCache.reaperLock.Unlock() - } - }) - } -} - -func newNonceCacheForTesting(cbFuncs ...NextNonceCallback) *nonceCacheStruct { - cbFunc := func(ctx context.Context, signer tktypes.EthAddress) (uint64, error) { - return uint64(42), nil - } - if len(cbFuncs) == 1 { - cbFunc = cbFuncs[0] - } - nonceCache := newNonceCache(10*time.Second, cbFunc) - return nonceCache.(*nonceCacheStruct) -} - -func TestBatchDoubleSubmit(t *testing.T) { - ctx, ble, mocks, done := newTestPublicTxManager(t, false, func(mocks *mocksAndTestControl, conf *pldconf.PublicTxManagerConfig) { - conf.Manager.NonceCacheTimeout = confutil.P("0") - }) - defer done() - - mocks.ethClient.On("GetTransactionCount", mock.Anything, mock.Anything). - Return(confutil.P(tktypes.HexUint64(1122334455)), nil).Once() - - addr := tktypes.RandAddress() - batch, err := ble.PrepareSubmissionBatch(ctx, []*components.PublicTxSubmission{ - { - PublicTxInput: pldapi.PublicTxInput{ - From: addr, - PublicTxOptions: pldapi.PublicTxOptions{ - Gas: confutil.P(tktypes.HexUint64(1223451)), - Value: tktypes.Uint64ToUint256(100), - }, - }, - }, - }) - require.NoError(t, err) - assert.Empty(t, batch.Rejected()) - assert.Len(t, batch.Accepted(), 1) - - mocks.db.ExpectBegin() - mocks.db.ExpectExec("INSERT.*public_txns").WillReturnResult(driver.ResultNoRows) - mocks.db.ExpectCommit() - mocks.db.ExpectBegin() - mocks.db.ExpectRollback() - - err = ble.p.DB().Transaction(func(dbTX *gorm.DB) error { - return batch.Submit(ctx, dbTX) - }) - require.NoError(t, err) - batch.Completed(ctx, true) // would normally be in a defer - - err = ble.p.DB().Transaction(func(dbTX *gorm.DB) error { - return batch.Submit(ctx, dbTX) - }) - assert.Regexp(t, "PD011933", err) - - // Check that we reaped with nonce timeout zero - nc := ble.nonceManager.(*nonceCacheStruct) - nc.reaperLock.Lock() - defer nc.reaperLock.Unlock() - assert.Len(t, nc.nextNonceBySigner, 0) - -} - -func TestReapLoop(t *testing.T) { - nc := newNonceCache(10*time.Millisecond, func(ctx context.Context, signer tktypes.EthAddress) (uint64, error) { - return 0, nil - }).(*nonceCacheStruct) - defer nc.Stop() - ian, err := nc.IntentToAssignNonce(context.Background(), *tktypes.RandAddress()) - require.NoError(t, err) - ian.Complete(context.Background()) - - for { - nc.reaperLock.Lock() - cacheLen := len(nc.nextNonceBySigner) - nc.reaperLock.Unlock() - if cacheLen == 0 { - break - } - time.Sleep(10 * time.Millisecond) - } - -} diff --git a/core/go/internal/publictxmgr/persisted_pub_txs.go b/core/go/internal/publictxmgr/persisted_pub_txs.go index ce8d6708a..e7a6b69fd 100644 --- a/core/go/internal/publictxmgr/persisted_pub_txs.go +++ b/core/go/internal/publictxmgr/persisted_pub_txs.go @@ -17,8 +17,6 @@ package publictxmgr import ( - "strings" - "github.com/google/uuid" "github.com/kaleido-io/paladin/toolkit/pkg/pldapi" "github.com/kaleido-io/paladin/toolkit/pkg/tktypes" @@ -26,20 +24,20 @@ import ( // public_transactions type DBPublicTxn struct { - SignerNonce string `gorm:"column:signer_nonce;primaryKey"` + PublicTxnID uint64 `gorm:"column:pub_txn_id;primaryKey"` From tktypes.EthAddress `gorm:"column:from"` - Nonce uint64 `gorm:"column:nonce"` + Nonce *uint64 `gorm:"column:nonce"` Created tktypes.Timestamp `gorm:"column:created;autoCreateTime:nano"` To *tktypes.EthAddress `gorm:"column:to"` Gas uint64 `gorm:"column:gas"` FixedGasPricing tktypes.RawJSON `gorm:"column:fixed_gas_pricing"` Value *tktypes.HexUint256 `gorm:"column:value"` Data tktypes.HexBytes `gorm:"column:data"` - Suspended bool `gorm:"column:suspended"` // excluded from processing because it's suspended by user - Completed *DBPublicTxnCompletion `gorm:"foreignKey:signer_nonce;references:signer_nonce"` // excluded from processing because it's done - Submissions []*DBPubTxnSubmission `gorm:"-"` // we do the aggregation, not GORM + Suspended bool `gorm:"column:suspended"` // excluded from processing because it's suspended by user + Completed *DBPublicTxnCompletion `gorm:"foreignKey:pub_txn_id;references:pub_txn_id"` // excluded from processing because it's done + Submissions []*DBPubTxnSubmission `gorm:"-"` // we do the aggregation, not GORM // Binding is used only on queries by transaction (GORM doesn't seem to allow us to define a separate struct for this) - Binding *DBPublicTxnBinding `gorm:"foreignKey:signer_nonce;references:signer_nonce;"` + Binding *DBPublicTxnBinding `gorm:"foreignKey:pub_txn_id;references:pub_txn_id;"` } func (DBPublicTxn) TableName() string { @@ -47,7 +45,7 @@ func (DBPublicTxn) TableName() string { } type DBPublicTxnBinding struct { - SignerNonce string `gorm:"column:signer_nonce;primaryKey"` + PublicTxnID uint64 `gorm:"column:pub_txn_id;primaryKey"` Transaction uuid.UUID `gorm:"column:transaction"` TransactionType tktypes.Enum[pldapi.TransactionType] `gorm:"column:tx_type"` } @@ -57,9 +55,10 @@ func (DBPublicTxnBinding) TableName() string { } type DBPubTxnSubmission struct { - SignerNonce string `gorm:"column:signer_nonce;primaryKey"` + from string `gorm:"-"` // just used to ensure we dispatch to same writer as the associated pubic TX + PublicTxnID uint64 `gorm:"column:pub_txn_id"` Created tktypes.Timestamp `gorm:"column:created;autoCreateTime:false"` // we set this as we track the record in memory too - TransactionHash tktypes.Bytes32 `gorm:"column:tx_hash"` + TransactionHash tktypes.Bytes32 `gorm:"column:tx_hash;primaryKey"` GasPricing tktypes.RawJSON `gorm:"column:gas_pricing"` // no filtering allowed on this field as it's complex JSON gasPrice/maxFeePerGas/maxPriorityFeePerGas calculation } @@ -68,7 +67,7 @@ func (DBPubTxnSubmission) TableName() string { } type DBPublicTxnCompletion struct { - SignerNonce string `gorm:"column:signer_nonce;primaryKey"` + PublicTxnID uint64 `gorm:"column:pub_txn_id;primaryKey"` Created tktypes.Timestamp `gorm:"column:created;autoCreateTime:nano"` TransactionHash tktypes.Bytes32 `gorm:"column:tx_hash"` Success bool `gorm:"column:success"` @@ -81,12 +80,12 @@ func (DBPublicTxnCompletion) TableName() string { func (s *DBPubTxnSubmission) WriteKey() string { // Just use the from address as the write key, so all submissions on the same signing address get batched together - return strings.Split(s.SignerNonce, ":")[0] + return s.from } type bindingsMatchingSubmission struct { DBPublicTxnBinding `gorm:"embedded"` - Submission *DBPubTxnSubmission `gorm:"foreignKey:signer_nonce;references:signer_nonce;"` + Submission *DBPubTxnSubmission `gorm:"foreignKey:pub_txn_id;references:pub_txn_id;"` } type txFromOnly struct { diff --git a/core/go/internal/publictxmgr/submission_writer.go b/core/go/internal/publictxmgr/submission_writer.go index af393b925..2a5cb10f4 100644 --- a/core/go/internal/publictxmgr/submission_writer.go +++ b/core/go/internal/publictxmgr/submission_writer.go @@ -48,6 +48,9 @@ func (sw *submissionWriter) runBatch(ctx context.Context, tx *gorm.DB, values [] }). Create(values). Error + if err != nil { + return nil, nil, err + } // We don't actually provide any result, so just build an array of nil results return nil, make([]flushwriter.Result[*noResult], len(values)), err } diff --git a/core/go/internal/publictxmgr/transaction_manager.go b/core/go/internal/publictxmgr/transaction_manager.go index 6082a1972..1de2914a2 100644 --- a/core/go/internal/publictxmgr/transaction_manager.go +++ b/core/go/internal/publictxmgr/transaction_manager.go @@ -18,9 +18,7 @@ package publictxmgr import ( "context" "encoding/json" - "fmt" "math/big" - "strings" "sync" "time" @@ -86,9 +84,6 @@ type pubTxManager struct { gasPriceClient GasPriceClient submissionWriter *submissionWriter - // nonce manager - nonceManager NonceCache - // a map of signing addresses and transaction engines inFlightOrchestrators map[tktypes.EthAddress]*orchestrator signingAddressesPausedUntil map[tktypes.EthAddress]time.Time @@ -107,7 +102,7 @@ type pubTxManager struct { nonceCacheTimeout time.Duration engineLoopDone chan struct{} - activityRecordCache cache.Cache[string, *txActivityRecords] + activityRecordCache cache.Cache[uint64, *txActivityRecords] maxActivityRecordsPerTx int // balance manager @@ -149,7 +144,7 @@ func NewPublicTransactionManager(ctx context.Context, conf *pldconf.PublicTxMana retry: retry.NewRetryIndefinite(&conf.Manager.Retry), gasPriceIncreaseMax: gasPriceIncreaseMax, gasPriceIncreasePercent: confutil.Int(conf.GasPrice.IncreasePercentage, *pldconf.PublicTxManagerDefaults.GasPrice.IncreasePercentage), - activityRecordCache: cache.NewCache[string, *txActivityRecords](&conf.Manager.ActivityRecords.CacheConfig, &pldconf.PublicTxManagerDefaults.Manager.ActivityRecords.CacheConfig), + activityRecordCache: cache.NewCache[uint64, *txActivityRecords](&conf.Manager.ActivityRecords.CacheConfig, &pldconf.PublicTxManagerDefaults.Manager.ActivityRecords.CacheConfig), maxActivityRecordsPerTx: confutil.Int(conf.Manager.ActivityRecords.RecordsPerTransaction, *pldconf.PublicTxManagerDefaults.Manager.ActivityRecords.RecordsPerTransaction), } } @@ -187,16 +182,6 @@ func (ble *pubTxManager) Start() error { // The client is assured to be started by this point and available ble.ethClient = ble.ethClientFactory.SharedWS() ble.gasPriceClient.Init(ctx, ble.ethClient) - ble.nonceManager = newNonceCache(ble.nonceCacheTimeout, func(ctx context.Context, signer tktypes.EthAddress) (uint64, error) { - log.L(ctx).Tracef("NonceFromChain getting next nonce for signing address ID %s", signer) - nextNonce, err := ble.ethClient.GetTransactionCount(ctx, signer) - if err != nil { - log.L(ctx).Errorf("NonceFromChain getting next nonce for signer %s failed: %+v", signer, err) - return 0, err - } - log.L(ctx).Tracef("NonceFromChain getting next nonce for signer %s succeeded: %s, converting to uint: %d", signer, nextNonce.String(), nextNonce.Uint64()) - return nextNonce.Uint64(), nil - }) if ble.engineLoopDone == nil { // only start once ble.engineLoopDone = make(chan struct{}) log.L(ctx).Debugf("Kicking off enterprise handler engine loop") @@ -213,160 +198,11 @@ func (ble *pubTxManager) Stop() { if ble.submissionWriter != nil { ble.submissionWriter.Shutdown() } - if ble.nonceManager != nil { - ble.nonceManager.Stop() - } if ble.engineLoopDone != nil { <-ble.engineLoopDone } } -type preparedTransaction struct { - bindings []*components.PaladinTXReference - tx *pldapi.PublicTx - rejectError error // only if rejected - revertData tktypes.HexBytes // only if rejected, and was available - nsi NonceAssignmentIntent // only if accepted -} - -type preparedTransactionBatch struct { - ble *pubTxManager - accepted []components.PublicTxAccepted - rejected []components.PublicTxRejected -} - -// Submit writes the prepared submission to the database using the provided context -// This is expected to be a lightweight operation involving not much more than writing to the database, as the heavy lifting should have been done in PrepareSubmission -// The database transaction will be coordinated by the caller -func (pb *preparedTransactionBatch) Submit(ctx context.Context, dbTX *gorm.DB) (err error) { - persistedTransactions := make([]*DBPublicTxn, len(pb.accepted)) - publicTxBindings := make([]*DBPublicTxnBinding, 0, len(pb.accepted)) - for i, accepted := range pb.accepted { - ptx := accepted.(*preparedTransaction) - persistedTransactions[i], err = pb.ble.finalizeNonceForPersistedTX(ctx, ptx) - if err != nil { - return err - } - for _, bnd := range ptx.bindings { - publicTxBindings = append(publicTxBindings, &DBPublicTxnBinding{ - Transaction: bnd.TransactionID, - TransactionType: bnd.TransactionType, - SignerNonce: persistedTransactions[i].SignerNonce, - }) - } - } - // All the nonce processing to this point should have ensured we do not have a conflict on nonces. - // It is the caller's responsibility to ensure we do not have a conflict on transaction+resubmit_idx. - if len(persistedTransactions) > 0 { - err = dbTX. - WithContext(ctx). - Table("public_txns"). - Create(persistedTransactions). - Error - } - if err == nil && len(publicTxBindings) > 0 { - err = dbTX. - WithContext(ctx). - Table("public_txn_bindings"). - Create(publicTxBindings). - Error - } - - return err -} - -func (pb *preparedTransactionBatch) Accepted() []components.PublicTxAccepted { return pb.accepted } -func (pb *preparedTransactionBatch) Rejected() []components.PublicTxRejected { return pb.rejected } - -func (pb *preparedTransactionBatch) Completed(ctx context.Context, committed bool) { - for _, pt := range pb.accepted { - if committed { - pt.(*preparedTransaction).nsi.Complete(ctx) - } else { - pt.(*preparedTransaction).nsi.Rollback(ctx) - } - } - if committed && len(pb.accepted) > 0 { - log.L(ctx).Debugf("%d transactions committed to DB", len(pb.accepted)) - pb.ble.MarkInFlightOrchestratorsStale() - } -} - -func (pt *preparedTransaction) Bindings() []*components.PaladinTXReference { - return pt.bindings -} - -func (pt *preparedTransaction) PublicTx() *pldapi.PublicTx { - return pt.tx -} - -func (pt *preparedTransaction) RejectedError() error { - return pt.rejectError -} - -func (pt *preparedTransaction) RevertData() tktypes.HexBytes { - return pt.revertData -} - -func (ble *pubTxManager) PrepareSubmissionBatch(ctx context.Context, transactions []*components.PublicTxSubmission) (components.PublicTxBatch, error) { - batch := &preparedTransactionBatch{ - ble: ble, - accepted: make([]components.PublicTxAccepted, 0, len(transactions)), - rejected: make([]components.PublicTxRejected, 0), - } - earlyReturn := true - defer func() { - if earlyReturn { - // Ensure we always cleanup if we fail (for error or panic) before we've - // delegated responsibility for calling this to our caller - batch.Completed(ctx, false) - } - }() - nonceAssigned := make([]*preparedTransaction, 0, len(transactions)) - for _, tx := range transactions { - preparedSubmission, err := ble.prepareSubmission(ctx, nonceAssigned, tx) - if err != nil { - return nil, err - } - if preparedSubmission.rejectError != nil { - batch.rejected = append(batch.rejected, preparedSubmission) - } else { - nonceAssigned = append(nonceAssigned, preparedSubmission) - batch.accepted = append(batch.accepted, preparedSubmission) - } - } - earlyReturn = false - return batch, nil -} - -// A one-and-done submission of a single transaction, used internally by auto-fueling, and demonstrating use of the -// public transaction interface for the special case of a single transaction that will succeed or fail. -// Other callers have to handle the Accepted()/Rejected() list to decide what they do for a split result. -func (ble *pubTxManager) SingleTransactionSubmit(ctx context.Context, transaction *components.PublicTxSubmission) (components.PublicTxAccepted, error) { - batch, err := ble.PrepareSubmissionBatch(ctx, []*components.PublicTxSubmission{transaction}) - if err != nil { - return nil, err - } - // Must call completed and tell it whether the allocation of the nonces committed or rolled back - committed := false - defer func() { - batch.Completed(ctx, committed) - }() - // Try to submit - if len(batch.Rejected()) > 0 { - return nil, batch.Rejected()[0].RejectedError() - } - err = ble.p.DB().Transaction(func(dbTX *gorm.DB) error { - return batch.Submit(ctx, dbTX) - }) - if err != nil { - return nil, err - } - // We committed - so the nonces are finalized as allocated - committed = true - return batch.Accepted()[0], nil -} - func buildEthTX( from tktypes.EthAddress, nonce *uint64, @@ -392,100 +228,138 @@ func buildEthTX( return ethTx } -// PrepareSubmission prepares and validates the transaction input data so that a later call to -// Submit can be made in the middle of a wider database transaction with minimal risk of error -func (ble *pubTxManager) prepareSubmission(ctx context.Context, batchSoFar []*preparedTransaction, txi *components.PublicTxSubmission) (preparedSubmission *preparedTransaction, err error) { - log.L(ctx).Tracef("PrepareSubmission transaction: %+v", txi) - - pt := &preparedTransaction{ - bindings: txi.Bindings, - tx: &pldapi.PublicTx{ - To: txi.To, - Data: txi.Data, - PublicTxOptions: txi.PublicTxOptions, - }, +func (ble *pubTxManager) SingleTransactionSubmit(ctx context.Context, txi *components.PublicTxSubmission) (tx *pldapi.PublicTx, err error) { + var txs []*pldapi.PublicTx + var cb func() + err = ble.ValidateTransaction(ctx, ble.p.DB(), txi) + if err == nil { + cb, txs, err = ble.WriteNewTransactions(ctx, ble.p.DB(), []*components.PublicTxSubmission{txi}) + } + if err == nil { + tx = txs[0] + cb() } + return +} + +func (ble *pubTxManager) ValidateTransaction(ctx context.Context, dbTX *gorm.DB, txi *components.PublicTxSubmission) error { + log.L(ctx).Tracef("PrepareSubmission transaction: %+v", txi) if txi.From == nil { - return nil, i18n.NewError(ctx, msgs.MsgInvalidTXMissingFromAddr) + return i18n.NewError(ctx, msgs.MsgInvalidTXMissingFromAddr) } - pt.tx.From = *txi.From prepareStart := time.Now() var txType InFlightTxOperation - rejected := false - if pt.tx.Gas == nil || *pt.tx.Gas == 0 { + if txi.Gas == nil || *txi.Gas == 0 { gasEstimateResult, err := ble.ethClient.EstimateGasNoResolve(ctx, buildEthTX( *txi.From, nil, /* nonce not assigned at this point */ - pt.tx.To, - pt.tx.Data, - &pt.tx.PublicTxOptions, + txi.To, + txi.Data, + &txi.PublicTxOptions, )) if err != nil { - log.L(ctx).Errorf("HandleNewTx <%s> error estimating gas for transaction: %+v, request: (%+v)", txType, err, pt.tx) + log.L(ctx).Errorf("HandleNewTx <%s> error estimating gas for transaction: %+v, request: (%+v)", txType, err, txi) ble.thMetrics.RecordOperationMetrics(ctx, string(txType), string(GenericStatusFail), time.Since(prepareStart).Seconds()) if ethclient.MapSubmissionRejected(err) { - // transaction is rejected, so no nonce will be assigned - but we have not failed in our task - pt.rejectError = err + // transaction is rejected. We can build a useful error message hopefully by processing the rejection info if len(gasEstimateResult.RevertData) > 0 { // we can use the error dictionary callback to TXManager to look up the ABI // Note: The ABI is already persisted before TXManager calls down into us. - pt.rejectError = ble.rootTxMgr.CalculateRevertError(ctx, ble.p.DB(), gasEstimateResult.RevertData) - log.L(ctx).Warnf("Estimate gas reverted (%s): %s", err, pt.rejectError) + err = ble.rootTxMgr.CalculateRevertError(ctx, dbTX, gasEstimateResult.RevertData) + log.L(ctx).Warnf("Estimate gas reverted (%s): %s", err, err) } - return pt, nil + return err } - return nil, err + return err } - pt.tx.Gas = &gasEstimateResult.GasLimit - log.L(ctx).Tracef("HandleNewTx <%s> using the estimated gas limit %s for transaction: %+v", txType, pt.tx.Gas, pt.tx) + txi.Gas = &gasEstimateResult.GasLimit + log.L(ctx).Tracef("HandleNewTx <%s> using the estimated gas limit %s for transaction: %+v", txType, txi.Gas, txi) } else { - log.L(ctx).Tracef("HandleNewTx <%s> using the provided gas limit %s for transaction: %+v", txType, pt.tx.Gas, pt.tx) + log.L(ctx).Tracef("HandleNewTx <%s> using the provided gas limit %s for transaction: %+v", txType, txi.Gas, txi) } - if !rejected { - // Need to check for an existing NSI for the address in the batch - for _, alreadyInBatch := range batchSoFar { - if alreadyInBatch.nsi != nil && alreadyInBatch.nsi.Address() == pt.tx.From { - pt.nsi = alreadyInBatch.nsi + ble.thMetrics.RecordOperationMetrics(ctx, string(txType), string(GenericStatusSuccess), time.Since(prepareStart).Seconds()) + log.L(ctx).Debugf("HandleNewTx <%s> transaction validated and nonce assignment intent created for %s", txType, txi.From) + return nil + +} + +func (ble *pubTxManager) WriteNewTransactions(ctx context.Context, dbTX *gorm.DB, transactions []*components.PublicTxSubmission) (postCommit func(), pubTxns []*pldapi.PublicTx, err error) { + persistedTransactions := make([]*DBPublicTxn, len(transactions)) + for i, txi := range transactions { + persistedTransactions[i] = &DBPublicTxn{ + From: *txi.From, // safe because validated in ValidateTransaction + To: txi.To, + Gas: txi.Gas.Uint64(), + Value: txi.Value, + Data: txi.Data, + FixedGasPricing: tktypes.JSONString(txi.PublicTxGasPricing), + } + } + // All the nonce processing to this point should have ensured we do not have a conflict on nonces. + // It is the caller's responsibility to ensure we do not have a conflict on transaction+resubmit_idx. + if len(persistedTransactions) > 0 { + err = dbTX. + WithContext(ctx). + Table("public_txns"). + Clauses(clause.Returning{Columns: []clause.Column{{Name: "pub_txn_id"}}}). + Create(persistedTransactions). + Error + } + if err == nil { + publicTxBindings := make([]*DBPublicTxnBinding, 0, len(transactions)) + for i, txi := range transactions { + pubTxnID := persistedTransactions[i].PublicTxnID + for _, bnd := range txi.Bindings { + publicTxBindings = append(publicTxBindings, &DBPublicTxnBinding{ + Transaction: bnd.TransactionID, + TransactionType: bnd.TransactionType, + PublicTxnID: pubTxnID, + }) } } - if pt.nsi == nil { - pt.nsi, err = ble.nonceManager.IntentToAssignNonce(ctx, pt.tx.From) + if len(publicTxBindings) > 0 { + err = dbTX. + WithContext(ctx). + Table("public_txn_bindings"). + Create(publicTxBindings). + Error } - if err != nil { - log.L(ctx).Errorf("HandleNewTx <%s> error assigning nonce for transaction: %+v, request: (%+v)", txType, err, pt.tx) - ble.thMetrics.RecordOperationMetrics(ctx, string(txType), string(GenericStatusFail), time.Since(prepareStart).Seconds()) - return nil, err + } + if err == nil { + pubTxns = make([]*pldapi.PublicTx, len(persistedTransactions)) + toNotify := make(map[tktypes.EthAddress]bool) + for i, ptx := range persistedTransactions { + pubTxns[i] = mapPersistedTransaction(ptx) + toNotify[ptx.From] = true } + postCommit = ble.postCommitNewTransactions(ctx, toNotify) } - ble.thMetrics.RecordOperationMetrics(ctx, string(txType), string(GenericStatusSuccess), time.Since(prepareStart).Seconds()) - log.L(ctx).Debugf("HandleNewTx <%s> transaction validated and nonce assignment intent created for %s", txType, pt.tx.From) - return pt, nil - + return postCommit, pubTxns, err } -func (ble *pubTxManager) finalizeNonceForPersistedTX(ctx context.Context, ptx *preparedTransaction) (*DBPublicTxn, error) { - nonce, err := ptx.nsi.AssignNextNonce(ctx) - if err != nil { - log.L(ctx).Errorf("Failed to assign nonce to public transaction %+v: %s", ptx, err) - return nil, err +func (ble *pubTxManager) postCommitNewTransactions(ctx context.Context, toNotify map[tktypes.EthAddress]bool) func() { + return func() { + // Mark any active orchestrators stale + inactive := false + for addr := range toNotify { + oc := ble.getOrchestratorForAddress(addr) + if oc != nil { + log.L(ctx).Debugf("Notified orchestrator %s to re-poll due to new transactions", &addr) + oc.MarkInFlightTxStale() + } else { + inactive = true + } + } + // And if there was an orchestrator un-loaded, then mark the main poll loop stale + if inactive { + ble.MarkInFlightOrchestratorsStale() + } } - tx := ptx.tx - tx.Nonce = tktypes.HexUint64(nonce) - log.L(ctx).Infof("Creating a new public transaction from=%s nonce=%d (%s)", tx.From, tx.Nonce /* number */, tx.Nonce /* hex */) - log.L(ctx).Tracef("payload: %+v", tx) - return &DBPublicTxn{ - SignerNonce: fmt.Sprintf("%s:%d", tx.From, tx.Nonce), // having a single key rather than compound key helps us simplify cross-table correlation, particularly for batch lookup - From: tx.From, - Nonce: tx.Nonce.Uint64(), - To: tx.To, - Gas: tx.Gas.Uint64(), - Data: tx.Data, - }, nil } func recoverGasPriceOptions(gpoJSON tktypes.RawJSON) (ptgp pldapi.PublicTxGasPricing) { @@ -542,7 +416,7 @@ func (ble *pubTxManager) queryPublicTxWithBinding(ctx context.Context, dbTX *gor for iSub, pSub := range ptx.Submissions { tx.Submissions[iSub] = mapPersistedSubmissionData(pSub) } - tx.Activity = ble.getActivityRecords(ptx.SignerNonce) + tx.Activity = ble.getActivityRecords(ptx.PublicTxnID) results[iTx] = &pldapi.PublicTxWithBinding{ PublicTx: tx, } @@ -557,15 +431,14 @@ func (ble *pubTxManager) queryPublicTxWithBinding(ctx context.Context, dbTX *gor return results, nil } -func (ble *pubTxManager) CheckTransactionCompleted(ctx context.Context, from tktypes.EthAddress, nonce uint64) (bool, error) { +func (ble *pubTxManager) CheckTransactionCompleted(ctx context.Context, pubTxnID uint64) (bool, error) { // Runs a DB query to see if the transaction is marked completed (for good or bad) // A non existent transaction results in false var ptxs []*DBPublicTxn err := ble.p.DB(). WithContext(ctx). Table("public_txns"). - Where("from = ?", from). - Where("nonce = ?", nonce). + Where(`"pub_txn_id" = ?`, pubTxnID). Joins("Completed"). Select(`"Completed"."tx_hash"`). Limit(1). @@ -575,7 +448,7 @@ func (ble *pubTxManager) CheckTransactionCompleted(ctx context.Context, from tkt return false, err } if len(ptxs) > 0 && ptxs[0].Completed != nil { - log.L(ctx).Debugf("CheckTransactionCompleted returned true for %s:%d", from, nonce) + log.L(ctx).Debugf("CheckTransactionCompleted returned true for %s:%d (pubTxnID=%d)", ptxs[0].From, ptxs[0].Nonce, pubTxnID) return true, nil } return false, nil @@ -592,8 +465,8 @@ func (ble *pubTxManager) GetPendingFuelingTransaction(ctx context.Context, sourc Joins("Completed"). Where(`"Completed"."tx_hash" IS NULL`). Joins("Binding"). - Where(`"Binding"."signer_nonce" IS NULL`). // no binding for auto fueling txns - Where("data IS NULL"). // they are simple transfers + Where(`"Binding"."pub_txn_id" IS NULL`). // no binding for auto fueling txns + Where("data IS NULL"). // they are simple transfers Limit(1). Find(&ptxs). Error @@ -601,7 +474,7 @@ func (ble *pubTxManager) GetPendingFuelingTransaction(ctx context.Context, sourc return nil, err } if len(ptxs) > 0 { - log.L(ctx).Debugf("GetPendingFuelingTransaction returned %s", ptxs[0].SignerNonce) + log.L(ctx).Debugf("GetPendingFuelingTransaction returned %d", ptxs[0].PublicTxnID) return mapPersistedTransaction(ptxs[0]), nil } return nil, nil @@ -620,18 +493,18 @@ func (ble *pubTxManager) runTransactionQuery(ctx context.Context, dbTX *gorm.DB, if err != nil { return nil, err } - signerNonceRefs := make([]string, len(ptxs)) + publicTxRefs := make([]uint64, len(ptxs)) for i, ptx := range ptxs { - signerNonceRefs[i] = ptx.SignerNonce + publicTxRefs[i] = ptx.PublicTxnID } - if len(signerNonceRefs) > 0 { - allSubs, err := ble.getTransactionSubmissions(ctx, dbTX, signerNonceRefs) + if len(publicTxRefs) > 0 { + allSubs, err := ble.getTransactionSubmissions(ctx, dbTX, publicTxRefs) if err != nil { return nil, err } for _, sub := range allSubs { for _, ptx := range ptxs { - if sub.SignerNonce == ptx.SignerNonce { + if sub.PublicTxnID == ptx.PublicTxnID { ptx.Submissions = append(ptx.Submissions, sub) } } @@ -642,10 +515,11 @@ func (ble *pubTxManager) runTransactionQuery(ctx context.Context, dbTX *gorm.DB, func mapPersistedTransaction(ptx *DBPublicTxn) *pldapi.PublicTx { tx := &pldapi.PublicTx{ + LocalID: &ptx.PublicTxnID, From: ptx.From, - Nonce: tktypes.HexUint64(ptx.Nonce), Created: ptx.Created, To: ptx.To, + Nonce: (*tktypes.HexUint64)(ptx.Nonce), Data: ptx.Data, PublicTxOptions: pldapi.PublicTxOptions{ Gas: (*tktypes.HexUint64)(&ptx.Gas), @@ -675,12 +549,12 @@ func mapPersistedSubmissionData(pSub *DBPubTxnSubmission) *pldapi.PublicTxSubmis } } -func (ble *pubTxManager) getTransactionSubmissions(ctx context.Context, dbTX *gorm.DB, signerNonceRefs []string) ([]*DBPubTxnSubmission, error) { +func (ble *pubTxManager) getTransactionSubmissions(ctx context.Context, dbTX *gorm.DB, pubTxnIDs []uint64) ([]*DBPubTxnSubmission, error) { var ptxs []*DBPubTxnSubmission err := dbTX. WithContext(ctx). Table("public_submissions"). - Where("signer_nonce IN (?)", signerNonceRefs). + Where("pub_txn_id IN (?)", pubTxnIDs). Order("created DESC"). Find(&ptxs). Error @@ -704,7 +578,7 @@ func (ble *pubTxManager) ResumeTransaction(ctx context.Context, from tktypes.Eth func (pte *pubTxManager) UpdateSubStatus(ctx context.Context, imtx InMemoryTxStateReadOnly, subStatus BaseTxSubStatus, action BaseTxAction, info *fftypes.JSONAny, err *fftypes.JSONAny, actionOccurred *tktypes.Timestamp) error { // TODO: Choose after testing the right way to treat these records - if text is right or not if err == nil { - pte.addActivityRecord(imtx.GetSignerNonce(), + pte.addActivityRecord(imtx.GetPubTxnID(), i18n.ExpandWithCode(ctx, i18n.MessageKey(msgs.MsgPublicTxHistoryInfo), imtx.GetFrom(), @@ -715,7 +589,7 @@ func (pte *pubTxManager) UpdateSubStatus(ctx context.Context, imtx InMemoryTxSta ), ) } else { - pte.addActivityRecord(imtx.GetSignerNonce(), + pte.addActivityRecord(imtx.GetPubTxnID(), i18n.ExpandWithCode(ctx, i18n.MessageKey(msgs.MsgPublicTxHistoryError), imtx.GetFrom(), @@ -731,14 +605,14 @@ func (pte *pubTxManager) UpdateSubStatus(ctx context.Context, imtx InMemoryTxSta } // add an activity record - this function assumes caller will not add multiple -func (pte *pubTxManager) addActivityRecord(signerNonce string, msg string) { +func (pte *pubTxManager) addActivityRecord(pubTxnID uint64, msg string) { if pte.maxActivityRecordsPerTx == 0 { return } - txr, _ := pte.activityRecordCache.Get(signerNonce) + txr, _ := pte.activityRecordCache.Get(pubTxnID) if txr == nil { txr = &txActivityRecords{} - pte.activityRecordCache.Set(signerNonce, txr) + pte.activityRecordCache.Set(pubTxnID, txr) } // We add to the front of the list (newest record first) and cap the size txr.lock.Lock() @@ -757,8 +631,8 @@ func (pte *pubTxManager) addActivityRecord(signerNonce string, msg string) { txr.records = newActivity } -func (pte *pubTxManager) getActivityRecords(signerNonce string) []pldapi.TransactionActivityRecord { - txr, _ := pte.activityRecordCache.Get(signerNonce) +func (pte *pubTxManager) getActivityRecords(pubTxID uint64) []pldapi.TransactionActivityRecord { + txr, _ := pte.activityRecordCache.Get(pubTxID) if txr != nil { // Snap the current activity array pointer in the lock and return it directly // (it does not get modified, only re-allocated on each update) @@ -770,19 +644,18 @@ func (pte *pubTxManager) getActivityRecords(signerNonce string) []pldapi.Transac } func (pte *pubTxManager) GetPublicTransactionForHash(ctx context.Context, dbTX *gorm.DB, hash tktypes.Bytes32) (*pldapi.PublicTxWithBinding, error) { - var signerNonces []string + var publicTxnIDs []uint64 var txns []*pldapi.PublicTxWithBinding err := dbTX. Table("public_submissions"). Model(DBPubTxnSubmission{}). Where(`tx_hash = ?`, hash). - Pluck("signer_nonce", &signerNonces). + Pluck("pub_txn_id", &publicTxnIDs). + Limit(1). Error - if err == nil && len(signerNonces) > 0 { - signerNonceSplit := strings.Split(signerNonces[0], ":") + if err == nil && len(publicTxnIDs) > 0 { txns, err = pte.QueryPublicTxWithBindings(ctx, dbTX, query.NewQueryBuilder(). - Equal("from", signerNonceSplit[0]). - Equal("nonce", signerNonceSplit[1]). + Equal("localId", publicTxnIDs[0]). Query()) } if err != nil || len(txns) == 0 { @@ -805,7 +678,7 @@ func (pte *pubTxManager) MatchUpdateConfirmedTransactions(ctx context.Context, d var lookups []*bindingsMatchingSubmission err := dbTX. Table("public_txn_bindings"). - Select(`"transaction"`, `"tx_type"`, `"Submission"."signer_nonce"`, `"Submission"."tx_hash"`). + Select(`"transaction"`, `"tx_type"`, `"Submission"."pub_txn_id"`, `"Submission"."tx_hash"`). Joins("Submission"). Where(`"Submission"."tx_hash" IN (?)`, txHashes). Find(&lookups). @@ -831,7 +704,7 @@ func (pte *pubTxManager) MatchUpdateConfirmedTransactions(ctx context.Context, d }) // completions to insert, in the order of the inputs completions = append(completions, &DBPublicTxnCompletion{ - SignerNonce: match.SignerNonce, + PublicTxnID: match.PublicTxnID, TransactionHash: txi.Hash, Success: txi.Result.V() == pldapi.TXResult_SUCCESS, RevertData: txi.RevertReason, @@ -846,7 +719,7 @@ func (pte *pubTxManager) MatchUpdateConfirmedTransactions(ctx context.Context, d err := dbTX. Table("public_completions"). Clauses(clause.OnConflict{ - Columns: []clause.Column{{Name: "signer_nonce"}}, + Columns: []clause.Column{{Name: "pub_txn_id"}}, DoNothing: true, // immutable }). Create(completions). diff --git a/core/go/internal/publictxmgr/transaction_manager_loop.go b/core/go/internal/publictxmgr/transaction_manager_loop.go index 640925ad9..36fea70e2 100644 --- a/core/go/internal/publictxmgr/transaction_manager_loop.go +++ b/core/go/internal/publictxmgr/transaction_manager_loop.go @@ -130,8 +130,8 @@ func (ble *pubTxManager) poll(ctx context.Context) (polled int, total int) { err := ble.retry.Do(ctx, func(attempt int) (retry bool, err error) { // (raw SQL as couldn't convince gORM to build this) const dbQueryBase = `SELECT DISTINCT t."from" FROM "public_txns" AS t ` + - `LEFT JOIN "public_completions" AS c ON t."signer_nonce" = c."signer_nonce" ` + - `WHERE c."signer_nonce" IS NULL AND "suspended" IS FALSE` + `LEFT JOIN "public_completions" AS c ON t."pub_txn_id" = c."pub_txn_id" ` + + `WHERE c."pub_txn_id" IS NULL AND "suspended" IS FALSE` const dbQueryNothingInFlight = dbQueryBase + ` LIMIT ?` if len(inFlightSigningAddresses) == 0 { diff --git a/core/go/internal/publictxmgr/transaction_manager_loop_test.go b/core/go/internal/publictxmgr/transaction_manager_loop_test.go index e4bbba727..a6ddbd498 100644 --- a/core/go/internal/publictxmgr/transaction_manager_loop_test.go +++ b/core/go/internal/publictxmgr/transaction_manager_loop_test.go @@ -62,7 +62,7 @@ func TestNewEnginePollingStoppingAnOrchestratorForFairnessControl(t *testing.T) } // we should stop the first, and swap in the second - m.db.ExpectQuery("SELECT.*public_txn").WillReturnRows(sqlmock.NewRows([]string{"from"}).AddRow(testSigningAddr2)) + m.db.ExpectQuery("SELECT.*public_txn").WillReturnRows(sqlmock.NewRows([]string{"from", "nonce"}).AddRow(testSigningAddr2, 12345)) ble.poll(ctx) existingOrchestrator.orchestratorLoopDone = make(chan struct{}) diff --git a/core/go/internal/publictxmgr/transaction_manager_test.go b/core/go/internal/publictxmgr/transaction_manager_test.go index 76b2188a6..3d9715b78 100644 --- a/core/go/internal/publictxmgr/transaction_manager_test.go +++ b/core/go/internal/publictxmgr/transaction_manager_test.go @@ -61,8 +61,6 @@ type mocksAndTestControl struct { txManager *componentmocks.TXManager } -const mockBaseNonce = 103342 - // const testDestAddress = "0x6cee73cf4d5b0ac66ce2d1c0617bec4bedd09f39" // const testMainSigningAddress = testDestAddress @@ -169,9 +167,6 @@ func newTestPublicTxManager(t *testing.T, realDBAndSigner bool, extraSetup ...fu require.NoError(t, err) if mocks.disableManagerStart { - pmgr.nonceManager = newNonceCache(1*time.Hour, func(ctx context.Context, signer tktypes.EthAddress) (uint64, error) { - return mockBaseNonce, nil - }) pmgr.ethClient = pmgr.ethClientFactory.SharedWS() pmgr.gasPriceClient.Init(ctx, pmgr.ethClient) } else { @@ -259,34 +254,41 @@ func TestTransactionLifecycleRealKeyMgrAndDB(t *testing.T) { Return(confutil.P(tktypes.HexUint64(baseNonce)), nil).Once() // For the first one we do a one-off - _, err = ble.SingleTransactionSubmit(ctx, txs[0]) + singleTx, err := ble.SingleTransactionSubmit(ctx, txs[0]) require.NoError(t, err) // The rest we submit as as batch - batch, err := ble.PrepareSubmissionBatch(ctx, txs[1:]) + for _, tx := range txs[1:] { + err := ble.ValidateTransaction(ctx, ble.p.DB(), tx) + require.NoError(t, err) + } + postCommit, batch, err := ble.WriteNewTransactions(ctx, ble.p.DB(), txs[1:]) require.NoError(t, err) - assert.Empty(t, batch.Rejected()) - assert.Len(t, batch.Accepted(), len(txs)-1) - err = ble.p.DB().Transaction(func(dbTX *gorm.DB) error { - return batch.Submit(ctx, dbTX) - }) + require.Len(t, batch, len(txs[1:])) + for _, tx := range batch { + require.Greater(t, *tx.LocalID, uint64(0)) + } + postCommit() + + // Get one back again by ID + txRead, err := ble.QueryPublicTxWithBindings(ctx, ble.p.DB(), query.NewQueryBuilder().Equal("localId", *batch[1].LocalID).Limit(1).Query()) require.NoError(t, err) - batch.Completed(ctx, true) // would normally be in a defer + require.Len(t, txRead, 1) + require.Equal(t, batch[1].Data, txRead[0].Data) // Record activity on one TX - for i := range txs { - ble.addActivityRecord(fmt.Sprintf("%s:%d", resolvedKey, int(baseNonce)+i), fmt.Sprintf("activity %d", i)) + for i, tx := range append([]*pldapi.PublicTx{singleTx}, batch...) { + ble.addActivityRecord(*tx.LocalID, fmt.Sprintf("activity %d", i)) } // Query to check we now have all of these queryTxs, err := ble.QueryPublicTxWithBindings(ctx, ble.p.DB(), - query.NewQueryBuilder().Sort("nonce").Query()) + query.NewQueryBuilder().Sort("localId").Query()) require.NoError(t, err) assert.Len(t, queryTxs, len(txs)) for i, qTX := range queryTxs { // We don't include the bindings on these queries assert.Equal(t, *resolvedKey, qTX.From) - assert.Equal(t, uint64(i)+baseNonce, qTX.Nonce.Uint64()) assert.Equal(t, txs[i].Data, qTX.Data) require.Greater(t, len(qTX.Activity), 0) } @@ -294,10 +296,9 @@ func TestTransactionLifecycleRealKeyMgrAndDB(t *testing.T) { // Query scoped to one TX byTxn, err := ble.QueryPublicTxForTransactions(ctx, ble.p.DB(), txIDs, nil) require.NoError(t, err) - for i, tx := range txs { + for _, tx := range txs { queryTxs := byTxn[tx.Bindings[0].TransactionID] require.Len(t, queryTxs, 1) - assert.Equal(t, baseNonce+uint64(i), queryTxs[0].Nonce.Uint64()) } // Check we can select to just see confirmed (which this isn't yet) @@ -444,18 +445,6 @@ func TestSubmitFailures(t *testing.T) { }) assert.Regexp(t, "mapped revert error", err) - // insert transaction next nonce error - m.ethClient.On("EstimateGasNoResolve", mock.Anything, mock.Anything, mock.Anything). - Return(ethclient.EstimateGasResult{GasLimit: tktypes.HexUint64(10)}, nil) - m.ethClient.On("GetTransactionCount", mock.Anything, mock.Anything). - Return(nil, fmt.Errorf("pop")).Once() - _, err = ble.SingleTransactionSubmit(ctx, &components.PublicTxSubmission{ - PublicTxInput: pldapi.PublicTxInput{ - From: tktypes.RandAddress(), - }, - }) - assert.NotNil(t, err) - assert.Regexp(t, "pop", err) } func TestAddActivityDisabled(t *testing.T) { @@ -464,38 +453,33 @@ func TestAddActivityDisabled(t *testing.T) { }) defer done() - ble.addActivityRecord("signer1:nonce", "message") + ble.addActivityRecord(12345, "message") - assert.Empty(t, ble.getActivityRecords("signer1:nonce")) + assert.Empty(t, ble.getActivityRecords(12345)) } func TestAddActivityWrap(t *testing.T) { _, ble, _, done := newTestPublicTxManager(t, false) defer done() - signerNonce := "signer1:nonce" for i := 0; i < 100; i++ { - ble.addActivityRecord(signerNonce, fmt.Sprintf("message %.2d", i)) + ble.addActivityRecord(12345, fmt.Sprintf("message %.2d", i)) } - activityRecords := ble.getActivityRecords(signerNonce) + activityRecords := ble.getActivityRecords(12345) assert.Equal(t, "message 99", activityRecords[0].Message) assert.Equal(t, "message 98", activityRecords[1].Message) assert.Len(t, activityRecords, ble.maxActivityRecordsPerTx) } -func mockForSubmitSuccess(mocks *mocksAndTestControl, conf *pldconf.PublicTxManagerConfig) { - mocks.ethClient.On("GetTransactionCount", mock.Anything, mock.Anything). - Return(confutil.P(tktypes.HexUint64(1122334455)), nil).Once() - mocks.db.ExpectBegin() - mocks.db.ExpectExec("INSERT.*public_txns").WillReturnResult(driver.ResultNoRows) - mocks.db.ExpectCommit() -} - func TestHandleNewTransactionTransferOnlyWithProvideGas(t *testing.T) { ctx := context.Background() - _, ble, _, done := newTestPublicTxManager(t, false, mockForSubmitSuccess) + _, ble, _, done := newTestPublicTxManager(t, false, func(mocks *mocksAndTestControl, conf *pldconf.PublicTxManagerConfig) { + mocks.db.MatchExpectationsInOrder(false) + mocks.db.ExpectQuery("SELECT.*public_txns").WillReturnRows(sqlmock.NewRows([]string{})) + mocks.db.ExpectExec("INSERT.*public_txns").WillReturnResult(driver.ResultNoRows) + }) defer done() // create transaction succeeded @@ -510,8 +494,8 @@ func TestHandleNewTransactionTransferOnlyWithProvideGas(t *testing.T) { }, }) assert.NoError(t, err) - assert.NotNil(t, tx.PublicTx().From) - assert.Equal(t, uint64(1223451), tx.PublicTx().Gas.Uint64()) + assert.NotNil(t, tx.From) + assert.Equal(t, uint64(1223451), tx.Gas.Uint64()) } diff --git a/core/go/internal/publictxmgr/transaction_orchestrator.go b/core/go/internal/publictxmgr/transaction_orchestrator.go index 1c7f6cab0..f211615fc 100644 --- a/core/go/internal/publictxmgr/transaction_orchestrator.go +++ b/core/go/internal/publictxmgr/transaction_orchestrator.go @@ -24,6 +24,7 @@ import ( "github.com/kaleido-io/paladin/config/pkg/confutil" "github.com/kaleido-io/paladin/config/pkg/pldconf" "github.com/kaleido-io/paladin/core/pkg/blockindexer" + "gorm.io/gorm" "github.com/kaleido-io/paladin/core/pkg/ethclient" "github.com/kaleido-io/paladin/toolkit/pkg/log" @@ -139,6 +140,9 @@ type orchestrator struct { staleTimeout time.Duration lastQueueUpdate time.Time + + lastNonceAlloc time.Time + nextNonce *uint64 } const veryShortMinimum = 50 * time.Millisecond @@ -187,6 +191,11 @@ func (oc *orchestrator) orchestratorLoop() { defer close(oc.orchestratorLoopDone) + if err := oc.initNextNonceFromDBRetry(ctx); err != nil { + log.L(ctx).Warnf("Context cancelled while obtaining highest nonce for %s: %s", oc.signingAddress, err) + return + } + ticker := time.NewTicker(oc.orchestratorPollingInterval) defer ticker.Stop() for { @@ -220,6 +229,102 @@ func (oc *orchestrator) getFirstInFlight() (ift *inFlightTransactionStageControl return } +func (oc *orchestrator) initNextNonceFromDBRetry(ctx context.Context) error { + return oc.retry.Do(ctx, func(attempt int) (retryable bool, err error) { + return true, oc.initNextNonceFromDB(ctx) + }) +} + +func (oc *orchestrator) initNextNonceFromDB(ctx context.Context) error { + var txns []*DBPublicTxn + err := oc.p.DB(). + WithContext(ctx). + Where(`"from" = ?`, oc.signingAddress). + Where("nonce IS NOT NULL"). + Order("nonce DESC"). + Limit(1). + Find(&txns). + Error + if err != nil || len(txns) == 0 { + return err + } + nextNonce := *txns[0].Nonce + 1 + oc.nextNonce = &nextNonce + log.L(ctx).Infof("Next nonce initialized from DB fro %s: %d", oc.signingAddress, nextNonce) + return nil +} + +func (oc *orchestrator) allocateNonces(ctx context.Context, txns []*DBPublicTxn) error { + + // Of the the transactions might have nonces already + toAlloc := make([]*DBPublicTxn, 0, len(txns)) + for _, tx := range txns { + if tx.Nonce == nil { + toAlloc = append(toAlloc, tx) + } + } + if len(toAlloc) == 0 { + // Nothing to do + return nil + } + + // We need to ensure we have the next nonce to allocate + if oc.nextNonce == nil || time.Since(oc.lastNonceAlloc) > oc.nonceCacheTimeout { + log.L(ctx).Debugf("no cached nonce, or nonce expired for %s (cached=%v)", oc.signingAddress, oc.lastNonceAlloc) + txCount, err := oc.ethClient.GetTransactionCount(ctx, oc.signingAddress) + if err != nil { + return err + } + // See if we have nonces in our DB that are ahead of the mempool. + if oc.nextNonce != nil && *oc.nextNonce >= txCount.Uint64() { + log.L(ctx).Infof("Next nonce for %s is %d (at or ahead of mempool %d)", oc.signingAddress, *oc.nextNonce, txCount.Uint64()) + } else { + // Otherwise take the node's answer + oc.nextNonce = (*uint64)(txCount) + log.L(ctx).Infof("Next nonce for %s set to %d (from eth_getTransactionCount)", oc.signingAddress, *oc.nextNonce) + } + } + + // Set up the list of nonces we'll allocated, but until it's in the DB we do NOT update the oc.nextNonce beyond the first in the list + newNextNonce := *oc.nextNonce + newNonces := make([]uint64, len(toAlloc)) + for i := range newNonces { + newNonces[i] = newNextNonce + newNextNonce++ + } + + // Run the DB TXN using a VALUES temp table to update multiple rows in a single operation + err := oc.p.DB().Transaction(func(dbTX *gorm.DB) error { + sqlQuery := `WITH nonce_updates ("pub_txn_id", "nonce") AS ( VALUES ` + values := make([]any, 0, len(toAlloc)*2) + for i, tx := range toAlloc { + if i > 0 { + sqlQuery += `, ` + } + sqlQuery += `( CAST (? AS BIGINT), CAST (? AS BIGINT) ) ` + values = append(values, tx.PublicTxnID) + values = append(values, newNonces[i]) + log.L(ctx).Debugf("assigning %s:%d (pubTxnId=%d)", oc.signingAddress, newNonces[i], tx.PublicTxnID) + } + sqlQuery += ` ) UPDATE "public_txns" SET "nonce" = nu."nonce" FROM ( SELECT "pub_txn_id", "nonce" FROM nonce_updates ) AS nu ` + + `WHERE "public_txns"."pub_txn_id" = nu."pub_txn_id";` + return dbTX.WithContext(ctx).Exec(sqlQuery, values...).Error + }) + if err != nil { + return err + } + + // Update the txns themselves, and our nextNonce + for i, tx := range toAlloc { + nonce := newNonces[i] + tx.Nonce = &nonce + } + oc.lastNonceAlloc = time.Now() + oc.nextNonce = &newNextNonce + + return nil +} + func (oc *orchestrator) pollAndProcess(ctx context.Context) (polled int, total int) { pollStart := time.Now() oc.inFlightTxsMux.Lock() @@ -235,11 +340,12 @@ func (oc *orchestrator) pollAndProcess(ctx context.Context) (polled int, total i stageCounts[stageName] = 0 } - var highestInFlightNonce uint64 + var highestInFlightNonce *uint64 // Run through copying across from the old InFlight list to the new one, those that aren't ready to be deleted for _, p := range oldInFlight { - if p.stateManager.GetNonce() > highestInFlightNonce { - highestInFlightNonce = p.stateManager.GetNonce() + if highestInFlightNonce == nil || p.stateManager.GetNonce() > *highestInFlightNonce { + newHighest := p.stateManager.GetNonce() + highestInFlightNonce = &newHighest } if p.stateManager.CanBeRemoved(ctx) { oc.totalCompleted = oc.totalCompleted + 1 @@ -273,13 +379,13 @@ func (oc *orchestrator) pollAndProcess(ctx context.Context) (polled int, total i Where(`"Completed"."tx_hash" IS NULL`). Where("suspended IS FALSE"). Where(`"from" = ?`, oc.signingAddress). - Order("nonce"). + Order(`"public_txns"."pub_txn_id"`). Limit(spaces) if len(oc.inFlightTxs) > 0 { // We don't want to see any of the ones we already have in flight. // The only way something leaves our in-flight list, is if we get a notification from the block indexer // that it committed a DB transaction that removed it from our list. - q = q.Where("nonce > ?", highestInFlightNonce) + q = q.Where("(nonce IS NULL OR nonce > ?)", highestInFlightNonce) } // Note we do not use an explicit DB transaction to coordinate the read of the // transactions table with the read of the submissions table, @@ -294,6 +400,17 @@ func (oc *orchestrator) pollAndProcess(ctx context.Context) (polled int, total i return -1, len(oc.inFlightTxs) } + // Synchronously we ensure that we have a nonce for all of these. + // This is an indefinite retry, as we MUST not proceed until a nonce has been allocated+stored for every one + // of these transactions. Otherwise we might re-order transactions compared to their DB commit order + // (which is unacceptable for strict TX ordering). + if err := oc.retry.Do(ctx, func(attempt int) (retryable bool, err error) { + return true, oc.allocateNonces(ctx, additional) + }); err != nil { + log.L(ctx).Warnf("Orchestrator context cancelled while allocating nonce: %s", err) + return + } + log.L(ctx).Debugf("Orchestrator poll and process: polled %d items, space: %d", len(additional), spaces) for _, ptx := range additional { queueUpdated = true @@ -304,7 +421,7 @@ func (oc *orchestrator) pollAndProcess(ctx context.Context) (polled int, total i txStage = InFlightTxStageQueued } stageCounts[string(txStage)] = stageCounts[string(txStage)] + 1 - log.L(ctx).Debugf("Orchestrator added transaction with ID: %s", ptx.SignerNonce) + log.L(ctx).Debugf("Orchestrator added transaction with PublicTxnID=%d From=%s", ptx.PublicTxnID, ptx.From) } total = len(oc.inFlightTxs) polled = total - oldLen diff --git a/core/go/internal/publictxmgr/transaction_orchestrator_test.go b/core/go/internal/publictxmgr/transaction_orchestrator_test.go index 6c9a70658..50599f9e2 100644 --- a/core/go/internal/publictxmgr/transaction_orchestrator_test.go +++ b/core/go/internal/publictxmgr/transaction_orchestrator_test.go @@ -52,11 +52,10 @@ func newTestOrchestrator(t *testing.T, cbs ...func(mocks *mocksAndTestControl, c func newInflightTransaction(o *orchestrator, nonce uint64, txMods ...func(tx *DBPublicTxn)) (*inFlightTransactionStageController, *inFlightTransactionState) { tx := &DBPublicTxn{ - SignerNonce: fmt.Sprintf("%s:%d", o.signingAddress, 1), - From: o.signingAddress, - Nonce: nonce, - Gas: 2000, - Created: tktypes.TimestampNow(), + From: o.signingAddress, + Nonce: &nonce, + Gas: 2000, + Created: tktypes.TimestampNow(), } for _, txMod := range txMods { txMod(tx) @@ -78,10 +77,12 @@ func TestNewOrchestratorLoadsSecondTxAndQueuesBalanceCheck(t *testing.T) { // Fill first slot with a stage controller o.inFlightTxs = []*inFlightTransactionStageController{mockIT} - // Return the next nonce - will fill up the orchestrator - m.db.ExpectQuery("SELECT.*public_txn").WillReturnRows(sqlmock.NewRows([]string{"from", "nonce"}).AddRow( - o.signingAddress, 2, - )) + // Return a single transaction - note there's a highest nonce query on startup before the first poll, so we query twice + for i := 0; i < 2; i++ { + m.db.ExpectQuery("SELECT.*public_txn").WillReturnRows(sqlmock.NewRows([]string{"from", "nonce"}).AddRow( + o.signingAddress, 2, + )) + } // Do not return any submissions for it m.db.ExpectQuery("SELECT.*public_submissions").WillReturnRows(sqlmock.NewRows([]string{})) @@ -139,8 +140,10 @@ func TestNewOrchestratorPollingRemoveCompleted(t *testing.T) { o.inFlightTxs = []*inFlightTransactionStageController{mockIT} o.state = OrchestratorStateRunning - // Just keep returning empty rows and we should go idle once we've flushed through the status update above - m.db.ExpectQuery("SELECT.*public_txn").WillReturnRows(sqlmock.NewRows([]string{})) + for i := 0; i < 2; i++ { + // return empty rows - once for max nonce calculation, and then again for the actual query + m.db.ExpectQuery("SELECT.*public_txn").WillReturnRows(sqlmock.NewRows([]string{})) + } ocDone, _ := o.Start(ctx) @@ -197,13 +200,13 @@ func TestOrchestratorTriggerTopUp(t *testing.T) { o.inFlightTxs = []*inFlightTransactionStageController{mockIT} o.state = OrchestratorStateRunning - // Mock no auto-fueling TX in flight - m.db.ExpectQuery("SELECT.*public_txns.*data IS NULL").WillReturnRows(sqlmock.NewRows([]string{})) + for i := 0; i < 2; i++ { + // return empty rows - once for max nonce calculation, and then again for the actual query + m.db.ExpectQuery("SELECT.*public_txn").WillReturnRows(sqlmock.NewRows([]string{})) + } // Then insert of the auto-fueling transaction - m.db.ExpectBegin() m.db.ExpectExec("INSERT.*public_txns").WillReturnResult(driver.ResultNoRows) - m.db.ExpectCommit() // Mock the insufficient balance on the account that's submitting m.ethClient.On("GetBalance", mock.Anything, o.signingAddress, "latest").Return(tktypes.Uint64ToUint256(0), nil) diff --git a/core/go/internal/publictxmgr/transaction_submission.go b/core/go/internal/publictxmgr/transaction_submission.go index 20ce1c955..edff1599f 100644 --- a/core/go/internal/publictxmgr/transaction_submission.go +++ b/core/go/internal/publictxmgr/transaction_submission.go @@ -114,6 +114,7 @@ func (it *inFlightTransactionStageController) submitTX(ctx context.Context, mtx submissionErrorReason = "" submissionOutcome = SubmissionOutcomeNonceTooLow default: + log.L(ctx).Errorf("Submission error for transaction ID %s with hash %s (requires retry): %s", mtx.GetSignerNonce(), txHash, submissionError) submissionOutcome = SubmissionOutcomeFailedRequiresRetry return true, submissionError } diff --git a/core/go/internal/publictxmgr/types.go b/core/go/internal/publictxmgr/types.go index 094a0cf4d..9320df9cf 100644 --- a/core/go/internal/publictxmgr/types.go +++ b/core/go/internal/publictxmgr/types.go @@ -257,6 +257,7 @@ type InMemoryTxStateReadOnly interface { GetCreatedTime() *tktypes.Timestamp // get the transaction receipt from the in-memory state (note: the returned value should not be modified) GetTransactionHash() *tktypes.Bytes32 + GetPubTxnID() uint64 GetNonce() uint64 GetFrom() tktypes.EthAddress GetTo() *tktypes.EthAddress diff --git a/core/go/internal/statedistribution/received_state_writer.go b/core/go/internal/statedistribution/received_state_writer.go index 6571abe7f..481ff83ba 100644 --- a/core/go/internal/statedistribution/received_state_writer.go +++ b/core/go/internal/statedistribution/received_state_writer.go @@ -113,7 +113,7 @@ func (rsw *receivedStateWriter) Stop() { } func (rsw *receivedStateWriter) QueueAndWait(ctx context.Context, domainName string, contractAddress tktypes.EthAddress, schemaID tktypes.Bytes32, stateDataJson tktypes.RawJSON, nullifier *components.NullifierUpsert) error { - log.L(ctx).Debugf("receivedStateWriter:QueueAndWait %s %s %s", domainName, contractAddress, schemaID) + log.L(ctx).Debugf("receivedStateWriter:QueueAndWait domainName=%s contractAddress=%s schemaID=%s", domainName, contractAddress, schemaID) op := rsw.flushWriter.Queue(ctx, &receivedStateWriteOperation{ DomainName: domainName, ContractAddress: contractAddress, diff --git a/core/go/internal/statedistribution/state_distributer.go b/core/go/internal/statedistribution/state_distributer.go index 795483d4a..3448dace6 100644 --- a/core/go/internal/statedistribution/state_distributer.go +++ b/core/go/internal/statedistribution/state_distributer.go @@ -24,6 +24,7 @@ import ( "github.com/kaleido-io/paladin/core/internal/components" "github.com/kaleido-io/paladin/core/internal/msgs" "github.com/kaleido-io/paladin/core/pkg/persistence" + pb "github.com/kaleido-io/paladin/core/pkg/proto/engine" "github.com/kaleido-io/paladin/toolkit/pkg/log" "github.com/kaleido-io/paladin/toolkit/pkg/retry" "github.com/kaleido-io/paladin/toolkit/pkg/tktypes" @@ -82,6 +83,8 @@ type StateDistributer interface { Stop(ctx context.Context) BuildNullifiers(ctx context.Context, stateDistributions []*components.StateDistribution) ([]*components.NullifierUpsert, error) DistributeStates(ctx context.Context, stateDistributions []*components.StateDistribution) + HandleStateProducedEvent(ctx context.Context, stateProducedEvent *pb.StateProducedEvent, distributingNode string) + HandleStateAcknowledgedEvent(ctx context.Context, messagePayload []byte) } type stateDistributer struct { @@ -106,12 +109,6 @@ func (sd *stateDistributer) Start(bgCtx context.Context) error { ctx := sd.runCtx log.L(ctx).Info("stateDistributer:Start") - err := sd.transportManager.RegisterClient(ctx, sd) - if err != nil { - log.L(ctx).Errorf("Error registering transport client: %s", err) - return err - } - sd.acknowledgementWriter.Start() sd.receivedStateWriter.Start() diff --git a/core/go/internal/statedistribution/state_receiver.go b/core/go/internal/statedistribution/state_receiver.go index 5f3369984..f460eb840 100644 --- a/core/go/internal/statedistribution/state_receiver.go +++ b/core/go/internal/statedistribution/state_receiver.go @@ -25,7 +25,7 @@ import ( ) func (sd *stateDistributer) sendStateAcknowledgement(ctx context.Context, domainName string, contractAddress string, stateId string, receivingParty string, distributingNode string, distributionID string) error { - log.L(ctx).Debugf("stateDistributer:sendStateAcknowledgement %s %s %s %s %s %s", domainName, contractAddress, stateId, receivingParty, distributingNode, distributionID) + log.L(ctx).Debugf("stateDistributer:sendStateAcknowledgement domainName=%s contractAddress=%s stateId=%s receivingParty=%s distributingNode=%s distributionID=%s", domainName, contractAddress, stateId, receivingParty, distributingNode, distributionID) stateAcknowledgedEvent := &pb.StateAcknowledgedEvent{ DomainName: domainName, ContractAddress: contractAddress, @@ -43,7 +43,7 @@ func (sd *stateDistributer) sendStateAcknowledgement(ctx context.Context, domain MessageType: "StateAcknowledgedEvent", Payload: stateAcknowledgedEventBytes, Node: distributingNode, - Component: STATE_DISTRIBUTER_DESTINATION, + Component: components.PRIVATE_TX_MANAGER_DESTINATION, ReplyTo: sd.localNodeName, }) if err != nil { diff --git a/core/go/internal/statedistribution/state_sender.go b/core/go/internal/statedistribution/state_sender.go index 78eb6f012..fd21598c9 100644 --- a/core/go/internal/statedistribution/state_sender.go +++ b/core/go/internal/statedistribution/state_sender.go @@ -74,7 +74,7 @@ func (sd *stateDistributer) sendState(ctx context.Context, stateDistribution *co MessageType: "StateProducedEvent", Payload: stateProducedEventBytes, Node: targetNode, - Component: STATE_DISTRIBUTER_DESTINATION, + Component: components.PRIVATE_TX_MANAGER_DESTINATION, ReplyTo: sd.localNodeName, }) if err != nil { diff --git a/core/go/internal/statedistribution/transport_client.go b/core/go/internal/statedistribution/transport_client.go index d554ae4df..d0221cc3a 100644 --- a/core/go/internal/statedistribution/transport_client.go +++ b/core/go/internal/statedistribution/transport_client.go @@ -25,36 +25,10 @@ import ( "google.golang.org/protobuf/proto" ) -const STATE_DISTRIBUTER_DESTINATION = "state-distributer" - -func (sd *stateDistributer) Destination() string { - return STATE_DISTRIBUTER_DESTINATION -} - -func (sd *stateDistributer) ReceiveTransportMessage(ctx context.Context, message *components.TransportMessage) { - log.L(ctx).Debugf("stateDistributer:ReceiveTransportMessage") - messagePayload := message.Payload - - switch message.MessageType { - case "StateProducedEvent": - distributingNode := message.ReplyTo - go sd.handleStateProducedEvent(ctx, messagePayload, distributingNode) - case "StateAcknowledgedEvent": - go sd.handleStateAcknowledgedEvent(ctx, message.Payload) - default: - log.L(ctx).Errorf("Unknown message type: %s", message.MessageType) - } -} - -func (sd *stateDistributer) handleStateProducedEvent(ctx context.Context, messagePayload []byte, distributingNode string) { +func (sd *stateDistributer) HandleStateProducedEvent(ctx context.Context, stateProducedEvent *pb.StateProducedEvent, distributingNode string) { log.L(ctx).Debugf("stateDistributer:handleStateProducedEvent") - stateProducedEvent := &pb.StateProducedEvent{} - err := proto.Unmarshal(messagePayload, stateProducedEvent) - if err != nil { - log.L(ctx).Errorf("Failed to unmarshal StateProducedEvent: %s", err) - return - } + var err error s := &components.StateDistribution{ ID: stateProducedEvent.DistributionId, StateID: stateProducedEvent.StateId, @@ -109,7 +83,7 @@ func (sd *stateDistributer) handleStateProducedEvent(ctx context.Context, messag } } -func (sd *stateDistributer) handleStateAcknowledgedEvent(ctx context.Context, messagePayload []byte) { +func (sd *stateDistributer) HandleStateAcknowledgedEvent(ctx context.Context, messagePayload []byte) { log.L(ctx).Debugf("stateDistributer:handleStateAcknowledgedEvent") stateAcknowledgedEvent := &pb.StateAcknowledgedEvent{} err := proto.Unmarshal(messagePayload, stateAcknowledgedEvent) diff --git a/core/go/internal/statemgr/domain_context.go b/core/go/internal/statemgr/domain_context.go index 6a0f6e96e..25971e57a 100644 --- a/core/go/internal/statemgr/domain_context.go +++ b/core/go/internal/statemgr/domain_context.go @@ -18,6 +18,7 @@ package statemgr import ( "context" + "encoding/json" "fmt" "sync" @@ -133,6 +134,7 @@ func (dc *domainContext) getUnFlushedSpends() (spending []tktypes.HexBytes, null } func (dc *domainContext) mergeUnFlushedApplyLocks(schema components.Schema, dbStates []*pldapi.State, query *query.QueryJSON, requireNullifier bool) (_ []*pldapi.State, err error) { + log.L(dc).Debugf("domainContext:mergeUnFlushedApplyLocks dc.txLocks: %d creatingStates: %d", len(dc.txLocks), len(dc.creatingStates)) dc.stateLock.Lock() defer dc.stateLock.Unlock() if flushErr := dc.checkResetInitUnFlushed(); flushErr != nil { @@ -248,7 +250,7 @@ func (dc *domainContext) mergeInMemoryMatches(schema components.Schema, states [ } func (dc *domainContext) FindAvailableStates(dbTX *gorm.DB, schemaID tktypes.Bytes32, query *query.QueryJSON) (components.Schema, []*pldapi.State, error) { - + log.L(dc.Context).Debug("domainContext:FindAvailableStates") // Build a list of spending states spending, _, _, err := dc.getUnFlushedSpends() if err != nil { @@ -260,9 +262,12 @@ func (dc *domainContext) FindAvailableStates(dbTX *gorm.DB, schemaID tktypes.Byt if err != nil { return nil, nil, err } + log.L(dc.Context).Debugf("domainContext:FindAvailableStates read %d states from DB", len(states)) // Merge in un-flushed states to results states, err = dc.mergeUnFlushedApplyLocks(schema, states, query, false) + log.L(dc.Context).Debugf("domainContext:FindAvailableStates mergeUnFlushedApplyLocks %d", len(states)) + return schema, states, err } @@ -572,3 +577,74 @@ func (dc *domainContext) checkResetInitUnFlushed() error { } return nil } + +// pldapi.StateLocks do not include the stateID in the serialized JSON so we need to define a new struct to include it +type exportableStateLock struct { + State tktypes.HexBytes `json:"stateID"` + Transaction uuid.UUID `json:"transaction"` + Type tktypes.Enum[pldapi.StateLockType] `json:"type"` +} + +// Return a snapshot of all currently known state locks as serialized JSON +func (dc *domainContext) ExportStateLocks() ([]byte, error) { + dc.stateLock.Lock() + defer dc.stateLock.Unlock() + if flushErr := dc.checkResetInitUnFlushed(); flushErr != nil { + return nil, flushErr + } + locks := make([]*exportableStateLock, 0, len(dc.txLocks)) + for _, l := range dc.txLocks { + locks = append(locks, &exportableStateLock{ + State: l.State, + Transaction: l.Transaction, + Type: l.Type, + }) + } + return json.Marshal(locks) +} + +// ImportStateLocks is used to restore the state of the domain context, by adding a set of locks +func (dc *domainContext) ImportStateLocks(stateLocksJSON []byte) error { + dc.stateLock.Lock() + defer dc.stateLock.Unlock() + if flushErr := dc.checkResetInitUnFlushed(); flushErr != nil { + return flushErr + } + locks := make([]*exportableStateLock, 0) + err := json.Unmarshal(stateLocksJSON, &locks) + if err != nil { + return i18n.WrapError(dc, err, msgs.MsgDomainContextImportInvalidJSON) + } + dc.creatingStates = make(map[string]*components.StateWithLabels) + dc.txLocks = make([]*pldapi.StateLock, 0, len(locks)) + for _, l := range locks { + dc.txLocks = append(dc.txLocks, &pldapi.StateLock{ + DomainName: dc.domainName, + State: l.State, + Transaction: l.Transaction, + Type: l.Type, + }) + //if it transpires that any of the states we already know about are created by these transactions, + // then we need to add them to the creatingStates map otherwise they will not be returned in queries + if l.Type == pldapi.StateLockTypeCreate.Enum() { + foundInUnflushed := false + for _, state := range dc.unFlushed.states { + if state.ID.String() == l.State.String() { + dc.creatingStates[state.ID.String()] = state + foundInUnflushed = true + } + } + if !foundInUnflushed { + // assuming this function is being used to copy a coordinators context to a delegate assembler's context + // this this if branch could mean one of two things: + // 1. the state distribution message hasn't' arrived yet but will arrive soon + // 2. the state distribution message is never going to arrive because we are not on the distribution list + // We can't tell the difference between these two cases so can't really fail here + // It is up to the domain to ensure that they ask for the transaction to be `Park`ed temporarily if they suspect `1` + log.L(dc).Infof("ImportStateLocks: state %s not found in unflushed states", l.State) + } + } + } + + return nil +} diff --git a/core/go/internal/statemgr/domain_context_test.go b/core/go/internal/statemgr/domain_context_test.go index 77a299d98..4bc206b76 100644 --- a/core/go/internal/statemgr/domain_context_test.go +++ b/core/go/internal/statemgr/domain_context_test.go @@ -676,6 +676,15 @@ func TestDomainContextFlushErrorCapture(t *testing.T) { err = dc.AddStateLocks(&pldapi.StateLock{}) assert.Regexp(t, "PD010119.*pop", err) // needs reset + _, err = dc.ExportStateLocks() + assert.Regexp(t, "PD010119.*pop", err) // needs reset + + err = dc.ImportStateLocks([]byte("{}")) + assert.Regexp(t, "PD010119.*pop", err) // needs reset + + err = dc.AddStateLocks(&pldapi.StateLock{}) + assert.Regexp(t, "PD010119.*pop", err) // needs reset + _, err = dc.Flush(ss.p.DB()) assert.Regexp(t, "pop", err) // the original error as it's a flush @@ -1076,3 +1085,217 @@ func TestCheckEvalGTTimestamp(t *testing.T) { assert.False(t, match) } + +func TestExportStateLocks(t *testing.T) { + + ctx, ss, _, _, done := newDBMockStateManager(t) + defer done() + + schema1, err := newABISchema(ctx, "domain1", testABIParam(t, fakeCoinABI)) + require.NoError(t, err) + ss.abiSchemaCache.Set(schemaCacheKey("domain1", schema1.ID()), schema1) + + schema2, err := newABISchema(ctx, "domain1", testABIParam(t, fakeCoinABI2)) + require.NoError(t, err) + ss.abiSchemaCache.Set(schemaCacheKey("domain1", schema2.ID()), schema2) + + contractAddress, dc := newTestDomainContext(t, ctx, ss, "domain1", false) + defer dc.Close() + + s1, err := schema1.ProcessState(ctx, contractAddress, tktypes.RawJSON(fmt.Sprintf( + `{"amount": 20, "owner": "0x615dD09124271D8008225054d85Ffe720E7a447A", "salt": "%s"}`, + tktypes.RandHex(32))), nil, dc.customHashFunction) + require.NoError(t, err) + + s2, err := schema2.ProcessState(ctx, contractAddress, tktypes.RawJSON(fmt.Sprintf( + `{"tokenUri": "%s", "owner": "0x615dD09124271D8008225054d85Ffe720E7a447A", "salt": "%s"}`, + tktypes.RandHex(32), tktypes.RandHex(32))), nil, dc.customHashFunction) + require.NoError(t, err) + + transactionID1 := uuid.New() + transactionID2 := uuid.New() + transactionID3 := uuid.New() + + _, err = dc.UpsertStates( + ss.p.DB(), + &components.StateUpsert{ + ID: s1.ID, + SchemaID: schema1.ID(), + Data: s1.Data, + CreatedBy: &transactionID1, + }, + &components.StateUpsert{ + ID: s2.ID, + SchemaID: schema2.ID(), + Data: s2.Data, + CreatedBy: &transactionID2, + }, + ) + require.NoError(t, err) + + err = dc.AddStateLocks( + &pldapi.StateLock{ + Type: pldapi.StateLockTypeSpend.Enum(), + State: s2.ID, + Transaction: transactionID3, + }, + ) + assert.NoError(t, err) + + json, err := dc.ExportStateLocks() + require.NoError(t, err) + assert.JSONEq(t, ` + [ + { + "stateID":"`+s1.ID.String()+`", + "transaction":"`+transactionID1.String()+`", + "type":"create" + }, + { + "stateID":"`+s2.ID.String()+`", + "transaction":"`+transactionID2.String()+`", + "type":"create" + }, + { + "stateID":"`+s2.ID.String()+`", + "transaction":"`+transactionID3.String()+`", + "type":"spend" + } + ]`, + string(json), + ) +} + +func TestImportStateLocks(t *testing.T) { + + ctx, ss, _, done := newDBTestStateManager(t) + defer done() + + schema1, err := newABISchema(ctx, "domain1", testABIParam(t, fakeCoinABI)) + require.NoError(t, err) + ss.abiSchemaCache.Set(schemaCacheKey("domain1", schema1.ID()), schema1) + + contractAddress, dc := newTestDomainContext(t, ctx, ss, "domain1", false) + defer dc.Close() + + s1, err := schema1.ProcessState(ctx, contractAddress, tktypes.RawJSON(fmt.Sprintf( + `{"amount": 20, "owner": "0x615dD09124271D8008225054d85Ffe720E7a447A", "salt": "%s"}`, + tktypes.RandHex(32))), nil, dc.customHashFunction) + require.NoError(t, err) + + s2, err := schema1.ProcessState(ctx, contractAddress, tktypes.RawJSON(fmt.Sprintf( + `{"amount": 20, "owner": "0x615dD09124271D8008225054d85Ffe720E7a447A", "salt": "%s"}`, + tktypes.RandHex(32))), nil, dc.customHashFunction) + require.NoError(t, err) + + s3ID := tktypes.RandHex(32) + + s4, err := schema1.ProcessState(ctx, contractAddress, tktypes.RawJSON(fmt.Sprintf( + `{"amount": 20, "owner": "0x615dD09124271D8008225054d85Ffe720E7a447A", "salt": "%s"}`, + tktypes.RandHex(32))), nil, dc.customHashFunction) + require.NoError(t, err) + + s5, err := schema1.ProcessState(ctx, contractAddress, tktypes.RawJSON(fmt.Sprintf( + `{"amount": 20, "owner": "0x615dD09124271D8008225054d85Ffe720E7a447A", "salt": "%s"}`, + tktypes.RandHex(32))), nil, dc.customHashFunction) + require.NoError(t, err) + + transactionID1 := uuid.New() + transactionID2 := uuid.New() + transactionID3 := uuid.New() + + _, err = dc.UpsertStates(ss.p.DB(), + &components.StateUpsert{ + ID: s1.ID, + SchemaID: schema1.ID(), + Data: s1.Data, + }, + &components.StateUpsert{ + ID: s2.ID, + SchemaID: schema1.ID(), + Data: s2.Data, + }, + &components.StateUpsert{ + ID: s4.ID, + SchemaID: schema1.ID(), + Data: s4.Data, + }, + &components.StateUpsert{ + ID: s5.ID, + SchemaID: schema1.ID(), + Data: s5.Data, + }, + ) + require.NoError(t, err) + + //imported locks include + // - state1 created by transaction1 for which we have the data + // - state2 created by transaction2 for which we have the data but has been spent by transaction3 + // - state3 created by transaction3 for which we do not have the data + // - state4 created by transaction3 for which we do have the data + // and does not include state5 even though we do have the data for that + // so after all that, the only available states should be state1 and state 4 + jsonToImport := fmt.Sprintf(` + [ + { + "stateID":"%s", + "transaction":"%s", + "type":"create" + }, + { + "stateID":"%s", + "transaction":"%s", + "type":"create" + }, + { + "stateID":"%s", + "transaction":"%s", + "type":"create" + }, + { + "stateID":"%s", + "transaction":"%s", + "type":"create" + }, + { + "stateID":"%s", + "transaction":"%s", + "type":"spend" + } + ]`, + s1.ID.String(), transactionID1.String(), + s2.ID.String(), transactionID2.String(), + s3ID, transactionID3.String(), + s4.ID.String(), transactionID3.String(), + s2.ID.String(), transactionID3.String(), + ) + + err = dc.ImportStateLocks([]byte(jsonToImport)) + require.NoError(t, err) + _, states, err := dc.FindAvailableStates(ss.p.DB(), schema1.ID(), query.NewQueryBuilder().Query()) + require.NoError(t, err) + require.Len(t, states, 2) + assert.Equal(t, s1.ID, states[0].ID) + assert.Equal(t, s4.ID, states[1].ID) + +} + +func TestImportStateLocksJSONError(t *testing.T) { + + ctx, ss, _, _, done := newDBMockStateManager(t) + defer done() + _, dc := newTestDomainContext(t, ctx, ss, "domain1", false) + defer dc.Close() + //valid JSON but wrong type for stateID + jsonToImport := ` + [ + { + "stateID":true + } + ]` + + err := dc.ImportStateLocks([]byte(jsonToImport)) + assert.Error(t, err) + assert.Regexp(t, "PD010132", err) + +} diff --git a/core/go/internal/txmgr/block_indexing_test.go b/core/go/internal/txmgr/block_indexing_test.go index b5a2b357d..2d5302964 100644 --- a/core/go/internal/txmgr/block_indexing_test.go +++ b/core/go/internal/txmgr/block_indexing_test.go @@ -22,9 +22,9 @@ import ( "github.com/google/uuid" "github.com/hyperledger/firefly-signer/pkg/abi" + "github.com/kaleido-io/paladin/config/pkg/confutil" "github.com/kaleido-io/paladin/config/pkg/pldconf" "github.com/kaleido-io/paladin/core/internal/components" - "github.com/kaleido-io/paladin/core/mocks/componentmocks" "github.com/kaleido-io/paladin/core/pkg/blockindexer" "github.com/kaleido-io/paladin/toolkit/pkg/pldapi" @@ -66,33 +66,36 @@ func TestPublicConfirmWithErrorDecodeRealDB(t *testing.T) { txi := newTestConfirm(revertData) var txID *uuid.UUID - ctx, txm, done := newTestTransactionManager(t, true, func(conf *pldconf.TxManagerConfig, mc *mockComponents) { - mockSubmissionBatch := componentmocks.NewPublicTxBatch(t) - mockSubmissionBatch.On("Rejected").Return([]components.PublicTxRejected{}) - mockSubmissionBatch.On("Submit", mock.Anything, mock.Anything).Return(nil) - mockSubmissionBatch.On("Completed", mock.Anything, true).Return(nil) - mc.publicTxMgr.On("PrepareSubmissionBatch", mock.Anything, mock.Anything).Return(mockSubmissionBatch, nil) + ctx, txm, done := newTestTransactionManager(t, true, + func(conf *pldconf.TxManagerConfig, mc *mockComponents) { + mockResolveKey(t, mc, "sender1", tktypes.RandAddress()) - mut := mc.publicTxMgr.On("MatchUpdateConfirmedTransactions", mock.Anything, mock.Anything, []*blockindexer.IndexedTransactionNotify{txi}) - mut.Run(func(args mock.Arguments) { - mut.Return([]*components.PublicTxMatch{ - { - PaladinTXReference: components.PaladinTXReference{ - TransactionID: *txID, // Transaction ID resolved by this point - TransactionType: pldapi.TransactionTypePublic.Enum(), - }, - IndexedTransactionNotify: txi, + mc.publicTxMgr.On("ValidateTransaction", mock.Anything, mock.Anything, mock.Anything).Return(nil) + mc.publicTxMgr.On("WriteNewTransactions", mock.Anything, mock.Anything, mock.Anything).Return( + func() {}, + []*pldapi.PublicTx{ + {LocalID: confutil.P(uint64(42))}, }, - }, nil) - }) - - mc.publicTxMgr.On("NotifyConfirmPersisted", mock.Anything, mock.MatchedBy(func(matches []*components.PublicTxMatch) bool { - return len(matches) == 1 && matches[0].TransactionID == *txID - })) + nil, + ) + + mut := mc.publicTxMgr.On("MatchUpdateConfirmedTransactions", mock.Anything, mock.Anything, []*blockindexer.IndexedTransactionNotify{txi}) + mut.Run(func(args mock.Arguments) { + mut.Return([]*components.PublicTxMatch{ + { + PaladinTXReference: components.PaladinTXReference{ + TransactionID: *txID, // Transaction ID resolved by this point + TransactionType: pldapi.TransactionTypePublic.Enum(), + }, + IndexedTransactionNotify: txi, + }, + }, nil) + }) - mc.keyManager.On("ResolveEthAddressBatchNewDatabaseTX", mock.Anything, []string{"sender1"}). - Return([]*tktypes.EthAddress{tktypes.RandAddress()}, nil) - }) + mc.publicTxMgr.On("NotifyConfirmPersisted", mock.Anything, mock.MatchedBy(func(matches []*components.PublicTxMatch) bool { + return len(matches) == 1 && matches[0].TransactionID == *txID + })) + }) defer done() abiRef, err := txm.storeABI(ctx, txm.p.DB(), testABI) diff --git a/core/go/internal/txmgr/persisted_receipt_test.go b/core/go/internal/txmgr/persisted_receipt_test.go index c0d5a2555..dffa421a3 100644 --- a/core/go/internal/txmgr/persisted_receipt_test.go +++ b/core/go/internal/txmgr/persisted_receipt_test.go @@ -114,10 +114,41 @@ func TestFinalizeTransactionsInsertFail(t *testing.T) { } +func mockKeyResolutionContextOk(t *testing.T) func(conf *pldconf.TxManagerConfig, mc *mockComponents) { + return func(conf *pldconf.TxManagerConfig, mc *mockComponents) { + _ = mockKeyResolver(t, mc) + } +} + +func mockKeyResolver(t *testing.T, mc *mockComponents) *componentmocks.KeyResolver { + krc := componentmocks.NewKeyResolutionContext(t) + kr := componentmocks.NewKeyResolver(t) + krc.On("KeyResolver", mock.Anything).Return(kr) + krc.On("PreCommit").Return(nil) + krc.On("Close", mock.Anything).Return() + mc.keyManager.On("NewKeyResolutionContext", mock.Anything).Return(krc) + return kr +} + +func mockKeyResolutionContextFail(t *testing.T) func(conf *pldconf.TxManagerConfig, mc *mockComponents) { + return func(conf *pldconf.TxManagerConfig, mc *mockComponents) { + _ = mockKeyResolverForFail(t, mc) + } +} + +func mockKeyResolverForFail(t *testing.T, mc *mockComponents) *componentmocks.KeyResolver { + krc := componentmocks.NewKeyResolutionContext(t) + kr := componentmocks.NewKeyResolver(t) + krc.On("KeyResolver", mock.Anything).Return(kr) + krc.On("Close", false).Return() + mc.keyManager.On("NewKeyResolutionContext", mock.Anything).Return(krc) + return kr +} + func TestFinalizeTransactionsInsertOkOffChain(t *testing.T) { - ctx, txm, done := newTestTransactionManager(t, true, func(conf *pldconf.TxManagerConfig, mc *mockComponents) { - mc.privateTxMgr.On("HandleNewTx", mock.Anything, mock.Anything).Return(nil) + ctx, txm, done := newTestTransactionManager(t, true, mockKeyResolutionContextOk(t), func(conf *pldconf.TxManagerConfig, mc *mockComponents) { + mc.privateTxMgr.On("HandleNewTx", mock.Anything, mock.Anything, mock.Anything).Return(nil) }) defer done() @@ -160,8 +191,8 @@ func TestFinalizeTransactionsInsertOkOffChain(t *testing.T) { func TestFinalizeTransactionsInsertOkEvent(t *testing.T) { - ctx, txm, done := newTestTransactionManager(t, true, func(conf *pldconf.TxManagerConfig, mc *mockComponents) { - mc.privateTxMgr.On("HandleNewTx", mock.Anything, mock.Anything).Return(nil) + ctx, txm, done := newTestTransactionManager(t, true, mockKeyResolutionContextOk(t), func(conf *pldconf.TxManagerConfig, mc *mockComponents) { + mc.privateTxMgr.On("HandleNewTx", mock.Anything, mock.Anything, mock.Anything).Return(nil) mc.stateMgr.On("GetTransactionStates", mock.Anything, mock.Anything, mock.Anything).Return( &pldapi.TransactionStates{None: true}, nil, diff --git a/core/go/internal/txmgr/prepared_transaction.go b/core/go/internal/txmgr/prepared_transaction.go index 12ec5f853..be33dc233 100644 --- a/core/go/internal/txmgr/prepared_transaction.go +++ b/core/go/internal/txmgr/prepared_transaction.go @@ -136,10 +136,7 @@ func (tm *txManager) WritePreparedTransactions(ctx context.Context, dbTX *gorm.D if len(preparedTxInserts) > 0 { err = dbTX.WithContext(ctx). - Clauses(clause.OnConflict{ - Columns: []clause.Column{{Name: "id"}}, - DoNothing: true, // immutable - }). + Clauses(clause.OnConflict{DoNothing: true /* immutable */}). Create(preparedTxInserts). Error } @@ -147,6 +144,7 @@ func (tm *txManager) WritePreparedTransactions(ctx context.Context, dbTX *gorm.D if err == nil && len(preparedTxStateInserts) > 0 { err = dbTX.WithContext(ctx). Omit("State"). + Clauses(clause.OnConflict{DoNothing: true /* immutable */}). Create(preparedTxStateInserts). Error } diff --git a/core/go/internal/txmgr/rpcmodule.go b/core/go/internal/txmgr/rpcmodule.go index 0a1409307..92e02a749 100644 --- a/core/go/internal/txmgr/rpcmodule.go +++ b/core/go/internal/txmgr/rpcmodule.go @@ -133,7 +133,7 @@ func (tm *txManager) rpcQueryTransactions() rpcserver.RPCHandler { return rpcserver.RPCMethod1(func(ctx context.Context, query query.QueryJSON, ) ([]*pldapi.Transaction, error) { - return tm.QueryTransactions(ctx, &query, false) + return tm.QueryTransactions(ctx, &query, tm.p.DB(), false) }) } @@ -141,7 +141,7 @@ func (tm *txManager) rpcQueryTransactionsFull() rpcserver.RPCHandler { return rpcserver.RPCMethod1(func(ctx context.Context, query query.QueryJSON, ) ([]*pldapi.TransactionFull, error) { - return tm.QueryTransactionsFull(ctx, &query, false) + return tm.QueryTransactionsFull(ctx, &query, tm.p.DB(), false) }) } @@ -151,9 +151,9 @@ func (tm *txManager) rpcQueryPendingTransactions() rpcserver.RPCHandler { full bool, ) (any, error) { if full { - return tm.QueryTransactionsFull(ctx, &query, true) + return tm.QueryTransactionsFull(ctx, &query, tm.p.DB(), true) } - return tm.QueryTransactions(ctx, &query, true) + return tm.QueryTransactions(ctx, &query, tm.p.DB(), true) }) } @@ -294,7 +294,7 @@ func (tm *txManager) rpcDebugTransactionStatus() rpcserver.RPCHandler { contractAddress string, id uuid.UUID, ) (components.PrivateTxStatus, error) { - return tm.privateTxMgr.GetTxStatus(ctx, contractAddress, id.String()) + return tm.privateTxMgr.GetTxStatus(ctx, contractAddress, id) }) } diff --git a/core/go/internal/txmgr/rpcmodule_test.go b/core/go/internal/txmgr/rpcmodule_test.go index 57f478406..d7eb4a05a 100644 --- a/core/go/internal/txmgr/rpcmodule_test.go +++ b/core/go/internal/txmgr/rpcmodule_test.go @@ -69,13 +69,29 @@ func newTestTransactionManagerWithRPC(t *testing.T, init ...func(*pldconf.TxMana } -func mockPublicSubmitTxOkOrReject(t *testing.T) func(conf *pldconf.TxManagerConfig, mc *mockComponents) { - return func(conf *pldconf.TxManagerConfig, mc *mockComponents) { - mockSubmissionBatch := componentmocks.NewPublicTxBatch(t) - mockSubmissionBatch.On("Rejected").Return([]components.PublicTxRejected{}) - mockSubmissionBatch.On("Submit", mock.Anything, mock.Anything).Return(nil) - mockSubmissionBatch.On("Completed", mock.Anything, mock.Anything).Return(nil) - mc.publicTxMgr.On("PrepareSubmissionBatch", mock.Anything, mock.Anything).Return(mockSubmissionBatch, nil) +func mockResolveKeyOKThenFail(t *testing.T, mc *mockComponents, identifier string, senderAddr *tktypes.EthAddress) { + kr := mockKeyResolverForFail(t, mc) + kr.On("ResolveKey", identifier, algorithms.ECDSA_SECP256K1, verifiers.ETH_ADDRESS). + Return(&pldapi.KeyMappingAndVerifier{Verifier: &pldapi.KeyVerifier{ + Verifier: senderAddr.String(), + }}, nil) +} + +func mockResolveKey(t *testing.T, mc *mockComponents, identifier string, senderAddr *tktypes.EthAddress) { + kr := mockKeyResolver(t, mc) + kr.On("ResolveKey", identifier, algorithms.ECDSA_SECP256K1, verifiers.ETH_ADDRESS). + Return(&pldapi.KeyMappingAndVerifier{Verifier: &pldapi.KeyVerifier{ + Verifier: senderAddr.String(), + }}, nil) +} + +func mockSubmitPublicTxOk(t *testing.T, senderAddr *tktypes.EthAddress) func(tmc *pldconf.TxManagerConfig, mc *mockComponents) { + return func(tmc *pldconf.TxManagerConfig, mc *mockComponents) { + mockResolveKey(t, mc, "sender1", senderAddr) + mc.publicTxMgr.On("ValidateTransaction", mock.Anything, mock.Anything, mock.Anything).Return(nil) + mc.publicTxMgr.On("WriteNewTransactions", mock.Anything, mock.Anything, mock.Anything).Return(func() {}, []*pldapi.PublicTx{ + {LocalID: confutil.P(uint64(12345))}, + }, nil) } } @@ -84,15 +100,13 @@ func TestPublicTransactionLifecycle(t *testing.T) { senderAddr := tktypes.RandAddress() var publicTxns map[uuid.UUID][]*pldapi.PublicTx ctx, url, tmr, done := newTestTransactionManagerWithRPC(t, - mockPublicSubmitTxOkOrReject(t), + mockSubmitPublicTxOk(t, senderAddr), mockQueryPublicTxForTransactions(func(ids []uuid.UUID, jq *query.QueryJSON) (map[uuid.UUID][]*pldapi.PublicTx, error) { return publicTxns, nil }), func(tmc *pldconf.TxManagerConfig, mc *mockComponents) { - mc.keyManager.On("ResolveEthAddressBatchNewDatabaseTX", mock.Anything, []string{"sender1"}). - Return([]*tktypes.EthAddress{senderAddr}, nil) - mc.keyManager.On("ResolveEthAddressNewDatabaseTX", mock.Anything, "sender1"). - Return(senderAddr, nil) + mc.keyManager.On("ResolveEthAddressNewDatabaseTX", mock.Anything, "sender1").Return(senderAddr, nil) // used in call + unconnected := ethclient.NewUnconnectedRPCClient(context.Background(), &pldconf.EthClientConfig{}, 0) mc.ethClientFactory.On("HTTPClient").Return(unconnected) }, @@ -142,7 +156,7 @@ func TestPublicTransactionLifecycle(t *testing.T) { tx1ID: { { From: *senderAddr, - Nonce: 111222333, + Nonce: confutil.P(tktypes.HexUint64(111222333)), }, }, } @@ -364,7 +378,7 @@ func TestPublicTransactionPassthroughQueries(t *testing.T) { tx := &pldapi.PublicTxWithBinding{ PublicTx: &pldapi.PublicTx{ From: tktypes.EthAddress(tktypes.RandBytes(20)), - Nonce: tktypes.HexUint64(nonce.Uint64()), + Nonce: confutil.P(tktypes.HexUint64(nonce.Uint64())), }, PublicTxBinding: pldapi.PublicTxBinding{Transaction: uuid.New(), TransactionType: pldapi.TransactionTypePublic.Enum()}, } @@ -496,12 +510,12 @@ func TestIdentityResolvePassthroughQueries(t *testing.T) { func TestDebugTransactionStatus(t *testing.T) { contractAddress := tktypes.RandAddress() - txID := uuid.New().String() + txID := uuid.New() ctx, url, _, done := newTestTransactionManagerWithRPC(t, func(tmc *pldconf.TxManagerConfig, mc *mockComponents) { mc.privateTxMgr.On("GetTxStatus", mock.Anything, contractAddress.String(), txID).Return(components.PrivateTxStatus{ - TxID: txID, + TxID: txID.String(), Status: "pending", LatestEvent: "submitted", LatestError: "some error message", @@ -516,7 +530,7 @@ func TestDebugTransactionStatus(t *testing.T) { var result components.PrivateTxStatus err = rpcClient.CallRPC(ctx, &result, "debug_getTransactionStatus", contractAddress.String(), txID) require.NoError(t, err) - assert.Equal(t, txID, result.TxID) + assert.Equal(t, txID.String(), result.TxID) assert.Equal(t, "pending", result.Status) assert.Equal(t, "submitted", result.LatestEvent) assert.Equal(t, "some error message", result.LatestError) @@ -545,8 +559,8 @@ func TestQueryPreparedTransactionsNotFound(t *testing.T) { func TestPrepareTransactions(t *testing.T) { - ctx, url, _, done := newTestTransactionManagerWithRPC(t, func(tmc *pldconf.TxManagerConfig, mc *mockComponents) { - mc.privateTxMgr.On("HandleNewTx", mock.Anything, mock.MatchedBy(func(tx *components.ValidatedTransaction) bool { + ctx, url, _, done := newTestTransactionManagerWithRPC(t, mockKeyResolutionContextOk(t), func(tmc *pldconf.TxManagerConfig, mc *mockComponents) { + mc.privateTxMgr.On("HandleNewTx", mock.Anything, mock.Anything, mock.MatchedBy(func(tx *components.ValidatedTransaction) bool { return tx.Transaction.SubmitMode.V() == pldapi.SubmitModeExternal })).Return(nil) }) diff --git a/core/go/internal/txmgr/transaction_query.go b/core/go/internal/txmgr/transaction_query.go index d34dfc598..d3d5a5800 100644 --- a/core/go/internal/txmgr/transaction_query.go +++ b/core/go/internal/txmgr/transaction_query.go @@ -72,7 +72,7 @@ func (tm *txManager) mapPersistedTXFull(pt *persistedTransaction) *pldapi.Transa return res } -func (tm *txManager) QueryTransactions(ctx context.Context, jq *query.QueryJSON, pending bool) ([]*pldapi.Transaction, error) { +func (tm *txManager) QueryTransactions(ctx context.Context, jq *query.QueryJSON, dbTX *gorm.DB, pending bool) ([]*pldapi.Transaction, error) { qw := &queryWrapper[persistedTransaction, pldapi.Transaction]{ p: tm.p, table: "transactions", @@ -90,15 +90,11 @@ func (tm *txManager) QueryTransactions(ctx context.Context, jq *query.QueryJSON, return mapPersistedTXBase(pt), nil }, } - return qw.run(ctx, nil) + return qw.run(ctx, dbTX) } -func (tm *txManager) QueryTransactionsFull(ctx context.Context, jq *query.QueryJSON, pending bool) (results []*pldapi.TransactionFull, err error) { - err = tm.p.DB().Transaction(func(dbTX *gorm.DB) error { - results, err = tm.QueryTransactionsFullTx(ctx, jq, dbTX, pending) - return err - }) - return +func (tm *txManager) QueryTransactionsFull(ctx context.Context, jq *query.QueryJSON, dbTX *gorm.DB, pending bool) (results []*pldapi.TransactionFull, err error) { + return tm.QueryTransactionsFullTx(ctx, jq, dbTX, pending) } func (tm *txManager) QueryTransactionsFullTx(ctx context.Context, jq *query.QueryJSON, dbTX *gorm.DB, pending bool) ([]*pldapi.TransactionFull, error) { @@ -146,7 +142,7 @@ func (tm *txManager) mergePublicTransactions(ctx context.Context, dbTX *gorm.DB, } func (tm *txManager) GetTransactionByIDFull(ctx context.Context, id uuid.UUID) (result *pldapi.TransactionFull, err error) { - ptxs, err := tm.QueryTransactionsFull(ctx, query.NewQueryBuilder().Limit(1).Equal("id", id).Query(), false) + ptxs, err := tm.QueryTransactionsFull(ctx, query.NewQueryBuilder().Limit(1).Equal("id", id).Query(), tm.p.DB(), false) if len(ptxs) == 0 || err != nil { return nil, err } @@ -154,7 +150,7 @@ func (tm *txManager) GetTransactionByIDFull(ctx context.Context, id uuid.UUID) ( } func (tm *txManager) GetTransactionByID(ctx context.Context, id uuid.UUID) (*pldapi.Transaction, error) { - ptxs, err := tm.QueryTransactions(ctx, query.NewQueryBuilder().Limit(1).Equal("id", id).Query(), false) + ptxs, err := tm.QueryTransactions(ctx, query.NewQueryBuilder().Limit(1).Equal("id", id).Query(), tm.p.DB(), false) if len(ptxs) == 0 || err != nil { return nil, err } @@ -162,7 +158,7 @@ func (tm *txManager) GetTransactionByID(ctx context.Context, id uuid.UUID) (*pld } func (tm *txManager) GetTransactionByIdempotencyKey(ctx context.Context, idempotencyKey string) (*pldapi.Transaction, error) { - ptxs, err := tm.QueryTransactions(ctx, query.NewQueryBuilder().Limit(1).Equal("idempotencyKey", idempotencyKey).Query(), false) + ptxs, err := tm.QueryTransactions(ctx, query.NewQueryBuilder().Limit(1).Equal("idempotencyKey", idempotencyKey).Query(), tm.p.DB(), false) if len(ptxs) == 0 || err != nil { return nil, err } diff --git a/core/go/internal/txmgr/transaction_query_test.go b/core/go/internal/txmgr/transaction_query_test.go index 654b9a377..59d98b8ad 100644 --- a/core/go/internal/txmgr/transaction_query_test.go +++ b/core/go/internal/txmgr/transaction_query_test.go @@ -30,9 +30,7 @@ import ( func TestGetTransactionByIDFullFail(t *testing.T) { ctx, txm, done := newTestTransactionManager(t, false, func(conf *pldconf.TxManagerConfig, mc *mockComponents) { - mc.db.ExpectBegin() mc.db.ExpectQuery("SELECT.*transactions").WillReturnError(fmt.Errorf("pop")) - mc.db.ExpectRollback() }) defer done() @@ -42,10 +40,8 @@ func TestGetTransactionByIDFullFail(t *testing.T) { func TestGetTransactionByIDFullPublicFail(t *testing.T) { ctx, txm, done := newTestTransactionManager(t, false, func(conf *pldconf.TxManagerConfig, mc *mockComponents) { - mc.db.ExpectBegin() mc.db.ExpectQuery("SELECT.*transactions").WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(uuid.New())) mc.db.ExpectQuery("SELECT.*transaction_deps").WillReturnRows(sqlmock.NewRows([]string{})) - mc.db.ExpectRollback() }, mockQueryPublicTxForTransactions(func(ids []uuid.UUID, jq *query.QueryJSON) (map[uuid.UUID][]*pldapi.PublicTx, error) { return nil, fmt.Errorf("pop") })) diff --git a/core/go/internal/txmgr/transaction_submission.go b/core/go/internal/txmgr/transaction_submission.go index 278fe3903..8fef237d9 100644 --- a/core/go/internal/txmgr/transaction_submission.go +++ b/core/go/internal/txmgr/transaction_submission.go @@ -28,11 +28,13 @@ import ( "github.com/kaleido-io/paladin/core/internal/components" "github.com/kaleido-io/paladin/core/internal/msgs" "github.com/kaleido-io/paladin/core/pkg/ethclient" + "github.com/kaleido-io/paladin/toolkit/pkg/algorithms" "github.com/kaleido-io/paladin/toolkit/pkg/log" "github.com/kaleido-io/paladin/toolkit/pkg/pldapi" "github.com/kaleido-io/paladin/toolkit/pkg/prototk" "github.com/kaleido-io/paladin/toolkit/pkg/query" "github.com/kaleido-io/paladin/toolkit/pkg/tktypes" + "github.com/kaleido-io/paladin/toolkit/pkg/verifiers" "gorm.io/gorm" "gorm.io/gorm/clause" ) @@ -355,15 +357,48 @@ func (tm *txManager) UpsertInternalPrivateTxsFinalizeIDs(ctx context.Context, db return nil } +func (tm *txManager) runWithDBTxnKeyResolver(ctx context.Context, fn func(dbTX *gorm.DB, kr components.KeyResolver) error) (err error) { + krc := tm.keyManager.NewKeyResolutionContext(ctx) + committed := false + defer func() { + krc.Close(committed) + }() + err = tm.p.DB().Transaction(func(dbTX *gorm.DB) error { + err := fn(dbTX, krc.KeyResolver(dbTX)) + if err == nil { + err = krc.PreCommit() + } + return err + }) + committed = (err == nil) + return err +} + func (tm *txManager) SendTransactions(ctx context.Context, txs []*pldapi.TransactionInput) (txIDs []uuid.UUID, err error) { - return tm.processNewTransactions(ctx, txs, pldapi.SubmitModeAuto) + var postCommit func() + err = tm.runWithDBTxnKeyResolver(ctx, func(dbTX *gorm.DB, kr components.KeyResolver) (err error) { + postCommit, txIDs, err = tm.processNewTransactions(ctx, dbTX, kr, txs, pldapi.SubmitModeAuto) + return + }) + if err == nil { + postCommit() + } + return } func (tm *txManager) PrepareTransactions(ctx context.Context, txs []*pldapi.TransactionInput) (txIDs []uuid.UUID, err error) { - return tm.processNewTransactions(ctx, txs, pldapi.SubmitModeExternal) + var postCommit func() + err = tm.runWithDBTxnKeyResolver(ctx, func(dbTX *gorm.DB, kr components.KeyResolver) (err error) { + postCommit, txIDs, err = tm.processNewTransactions(ctx, dbTX, kr, txs, pldapi.SubmitModeExternal) + return + }) + if err == nil { + postCommit() + } + return } -func (tm *txManager) processNewTransactions(ctx context.Context, txs []*pldapi.TransactionInput, submitMode pldapi.SubmitMode) (txIDs []uuid.UUID, err error) { +func (tm *txManager) processNewTransactions(ctx context.Context, dbTX *gorm.DB, kr components.KeyResolver, txs []*pldapi.TransactionInput, submitMode pldapi.SubmitMode) (postCommit func(), txIDs []uuid.UUID, err error) { // Public transactions need a signing address resolution and nonce allocation trackers // before we open the database transaction @@ -372,9 +407,9 @@ func (tm *txManager) processNewTransactions(ctx context.Context, txs []*pldapi.T txis := make([]*components.ValidatedTransaction, len(txs)) txIDs = make([]uuid.UUID, len(txs)) for i, tx := range txs { - txi, err := tm.resolveNewTransaction(ctx, tm.p.DB() /* no db tx for this part currently */, tx, submitMode) + txi, err := tm.resolveNewTransaction(ctx, dbTX, tx, submitMode) if err != nil { - return nil, err + return nil, nil, err } txID := *txi.Transaction.ID txis[i] = txi @@ -393,75 +428,60 @@ func (tm *txManager) processNewTransactions(ctx context.Context, txs []*pldapi.T } } - // Need to resolve the addresses for any public senders + // Public transactions need key resolution and validation if len(publicTxs) > 0 { - ethAddresses, err := tm.keyManager.ResolveEthAddressBatchNewDatabaseTX(ctx, publicTxSenders) - if err != nil { - return nil, err - } for i, ptx := range publicTxs { - ptx.From = ethAddresses[i] + resolvedKey, err := kr.ResolveKey(publicTxSenders[i], algorithms.ECDSA_SECP256K1, verifiers.ETH_ADDRESS) + if err == nil { + ptx.From, err = tktypes.ParseEthAddress(resolvedKey.Verifier.Verifier) + } + if err == nil { + err = tm.publicTxMgr.ValidateTransaction(ctx, dbTX, ptx) + } + if err != nil { + return nil, nil, err + } } } - // Perform pre-transaction processing in the public TX manager as required - var committed = false - var publicBatch components.PublicTxBatch - if len(publicTxs) > 0 { - publicBatch, err = tm.publicTxMgr.PrepareSubmissionBatch(ctx, publicTxs) - if err != nil { - return nil, err - } - // Must ensure we close the batch, now it's open (for good or bad) - defer func() { - publicBatch.Completed(ctx, committed) - }() - // TODO: don't support partial rejection currently - will be important when we introduce the flush writer - if len(publicBatch.Rejected()) > 0 { - return nil, publicBatch.Rejected()[0].RejectedError() - } + // Now we're ready to insert into the database + _, err = tm.insertTransactions(ctx, dbTX, txis, false /* all must succeed on this path - we map idempotency errors below */) + if err != nil { + _ = dbTX.Rollback() // so we can start a new TX in SQLite + return nil, nil, tm.checkIdempotencyKeys(ctx, err, txs) } - // Do in-transaction processing for our tables, and the public tables - insertedOK := false - err = tm.p.DB().Transaction(func(dbTX *gorm.DB) (err error) { - _, err = tm.insertTransactions(ctx, dbTX, txis, false /* all must succeed on this path - we map idempotency errors below */) - insertedOK = (err == nil) - if err == nil && publicBatch != nil { - err = publicBatch.Submit(ctx, dbTX) + // Insert any public txns (validated above) + postCommit = func() {} + if len(publicTxs) > 0 { + if postCommit, _, err = tm.publicTxMgr.WriteNewTransactions(ctx, dbTX, publicTxs); err != nil { + return nil, nil, err } - // TODO: private insertion too - return err - }) - if err != nil { - return nil, tm.checkIdempotencyKeys(ctx, err, insertedOK, txs) } - // From this point on we're committed, and need to tell the public tx manager as such - committed = true // TODO: Integrate with private TX manager persistence when available, as it will follow the // same pattern as public transactions above for _, txi := range txis { if txi.Transaction.Type.V() == pldapi.TransactionTypePrivate { - if err := tm.privateTxMgr.HandleNewTx(ctx, txi); err != nil { - return nil, err + if err := tm.privateTxMgr.HandleNewTx(ctx, dbTX, txi); err != nil { + return nil, nil, err } } } - return txIDs, err + return postCommit, txIDs, err } // Will either return the original error, or will return a special idempotency key error that can be used by the caller // to determine that they need to ask for the existing transactions (rather than fail) -func (tm *txManager) checkIdempotencyKeys(ctx context.Context, origErr error, insertedOK bool, txis []*pldapi.TransactionInput) error { +func (tm *txManager) checkIdempotencyKeys(ctx context.Context, origErr error, txis []*pldapi.TransactionInput) error { idempotencyKeys := make([]any, 0, len(txis)) for _, tx := range txis { if tx.IdempotencyKey != "" { idempotencyKeys = append(idempotencyKeys, tx.IdempotencyKey) } } - if !insertedOK && len(idempotencyKeys) > 0 { - existingTxs, lookupErr := tm.QueryTransactions(ctx, query.NewQueryBuilder().In("idempotencyKey", idempotencyKeys).Limit(len(idempotencyKeys)).Query(), false) + if len(idempotencyKeys) > 0 { + existingTxs, lookupErr := tm.QueryTransactions(ctx, query.NewQueryBuilder().In("idempotencyKey", idempotencyKeys).Limit(len(idempotencyKeys)).Query(), tm.p.DB(), false) if lookupErr != nil { log.L(ctx).Errorf("Failed to query for existing idempotencyKeys after insert error (returning original error): %s", lookupErr) } else if (len(existingTxs)) > 0 { @@ -478,6 +498,8 @@ func (tm *txManager) checkIdempotencyKeys(ctx context.Context, origErr error, in func (tm *txManager) resolveNewTransaction(ctx context.Context, dbTX *gorm.DB, tx *pldapi.TransactionInput, submitMode pldapi.SubmitMode) (*components.ValidatedTransaction, error) { txID := uuid.New() + // Useful to have a correlation from transactionID to idempotencyKey in the logs + log.L(ctx).Debugf("Resolving new transaction TransactionID: %s, idempotencyKey: %s ", txID, tx.IdempotencyKey) switch tx.Type.V() { case pldapi.TransactionTypePrivate: diff --git a/core/go/internal/txmgr/transaction_submission_test.go b/core/go/internal/txmgr/transaction_submission_test.go index 0abdf4810..95361e19c 100644 --- a/core/go/internal/txmgr/transaction_submission_test.go +++ b/core/go/internal/txmgr/transaction_submission_test.go @@ -26,21 +26,27 @@ import ( "github.com/kaleido-io/paladin/config/pkg/confutil" "github.com/kaleido-io/paladin/config/pkg/pldconf" "github.com/kaleido-io/paladin/core/internal/components" - "github.com/kaleido-io/paladin/core/mocks/componentmocks" "github.com/kaleido-io/paladin/core/pkg/ethclient" "gorm.io/gorm" + "github.com/kaleido-io/paladin/toolkit/pkg/algorithms" "github.com/kaleido-io/paladin/toolkit/pkg/pldapi" "github.com/kaleido-io/paladin/toolkit/pkg/pldclient" "github.com/kaleido-io/paladin/toolkit/pkg/query" "github.com/kaleido-io/paladin/toolkit/pkg/tktypes" + "github.com/kaleido-io/paladin/toolkit/pkg/verifiers" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) +func mockBeginRollback(conf *pldconf.TxManagerConfig, mc *mockComponents) { + mc.db.ExpectBegin() + mc.db.ExpectRollback() +} + func TestResolveFunctionABIAndDef(t *testing.T) { - ctx, txm, done := newTestTransactionManager(t, false) + ctx, txm, done := newTestTransactionManager(t, false, mockKeyResolutionContextFail(t), mockBeginRollback) defer done() _, err := txm.SendTransaction(ctx, &pldapi.TransactionInput{ @@ -54,7 +60,7 @@ func TestResolveFunctionABIAndDef(t *testing.T) { } func TestResolveFunctionNoABI(t *testing.T) { - ctx, txm, done := newTestTransactionManager(t, false) + ctx, txm, done := newTestTransactionManager(t, false, mockKeyResolutionContextFail(t), mockBeginRollback) defer done() _, err := txm.SendTransaction(ctx, &pldapi.TransactionInput{ @@ -68,7 +74,7 @@ func TestResolveFunctionNoABI(t *testing.T) { } func TestResolveFunctionBadABI(t *testing.T) { - ctx, txm, done := newTestTransactionManager(t, false) + ctx, txm, done := newTestTransactionManager(t, false, mockKeyResolutionContextFail(t), mockBeginRollback) defer done() _, err := txm.SendTransaction(ctx, &pldapi.TransactionInput{ @@ -82,12 +88,17 @@ func TestResolveFunctionBadABI(t *testing.T) { } func mockInsertABI(conf *pldconf.TxManagerConfig, mc *mockComponents) { + mc.db.ExpectBegin() + mockInsertABINoBegin(conf, mc) +} + +func mockInsertABINoBegin(conf *pldconf.TxManagerConfig, mc *mockComponents) { mc.db.ExpectExec("INSERT.*abis").WillReturnResult(driver.ResultNoRows) mc.db.ExpectExec("INSERT.*abi_entries").WillReturnResult(driver.ResultNoRows) } func TestResolveFunctionNamedWithNoTarget(t *testing.T) { - ctx, txm, done := newTestTransactionManager(t, false, mockInsertABI) + ctx, txm, done := newTestTransactionManager(t, false, mockInsertABI, mockKeyResolutionContextFail(t)) defer done() _, err := txm.SendTransaction(ctx, &pldapi.TransactionInput{ @@ -100,27 +111,15 @@ func TestResolveFunctionNamedWithNoTarget(t *testing.T) { assert.Regexp(t, "PD012204", err) } -func mockInsertABIAndTransactionOK(conf *pldconf.TxManagerConfig, mc *mockComponents) { - mc.db.ExpectExec("INSERT.*abis").WillReturnResult(driver.ResultNoRows) - mc.db.ExpectExec("INSERT.*abi_entries").WillReturnResult(driver.ResultNoRows) - mc.db.ExpectBegin() - mc.db.ExpectExec("INSERT.*transactions").WillReturnResult(driver.ResultNoRows) - mc.db.ExpectCommit() -} - -func mockPublicSubmitWithDataTxOk(t *testing.T) func(conf *pldconf.TxManagerConfig, mc *mockComponents) { +func mockInsertABIAndTransactionOK(commit bool) func(conf *pldconf.TxManagerConfig, mc *mockComponents) { return func(conf *pldconf.TxManagerConfig, mc *mockComponents) { - mockSubmissionBatch := componentmocks.NewPublicTxBatch(t) - mockSubmissionBatch.On("Rejected").Return([]components.PublicTxRejected{}) - mockSubmissionBatch.On("Submit", mock.Anything, mock.Anything).Return(nil) - mockSubmissionBatch.On("Completed", mock.Anything, true).Return(nil) - mc.publicTxMgr.On("PrepareSubmissionBatch", mock.Anything, mock.Anything).Return(mockSubmissionBatch, nil). - Run(func(args mock.Arguments) { - publicTxs := args[1].([]*components.PublicTxSubmission) - for _, tx := range publicTxs { - require.NotEmpty(t, tx.Data) - } - }) + mc.db.ExpectBegin() + mc.db.ExpectExec("INSERT.*abis").WillReturnResult(driver.ResultNoRows) + mc.db.ExpectExec("INSERT.*abi_entries").WillReturnResult(driver.ResultNoRows) + mc.db.ExpectExec("INSERT.*transactions").WillReturnResult(driver.ResultNoRows) + if commit { + mc.db.ExpectCommit() + } } } @@ -156,10 +155,11 @@ func mockGetPublicTransactionForHash(cb func(hash tktypes.Bytes32) (*pldapi.Publ func TestSubmitBadFromAddr(t *testing.T) { ctx, txm, done := newTestTransactionManager(t, false, func(conf *pldconf.TxManagerConfig, mc *mockComponents) { + kr := mockKeyResolverForFail(t, mc) + kr.On("ResolveKey", "sender1", algorithms.ECDSA_SECP256K1, verifiers.ETH_ADDRESS).Return(nil, fmt.Errorf("bad address")) + mc.db.ExpectBegin() mc.db.ExpectExec("INSERT.*abis").WillReturnResult(driver.ResultNoRows) mc.db.ExpectExec("INSERT.*abi_entries").WillReturnResult(driver.ResultNoRows) - mc.keyManager.On("ResolveEthAddressBatchNewDatabaseTX", mock.Anything, []string{"sender1"}). - Return(nil, fmt.Errorf("bad address")) }) defer done() @@ -181,10 +181,10 @@ func TestSubmitBadFromAddr(t *testing.T) { } func TestResolveFunctionHexInputOK(t *testing.T) { - ctx, txm, done := newTestTransactionManager(t, false, mockInsertABIAndTransactionOK, mockPublicSubmitWithDataTxOk(t), func(conf *pldconf.TxManagerConfig, mc *mockComponents) { - mc.keyManager.On("ResolveEthAddressBatchNewDatabaseTX", mock.Anything, []string{"sender1"}). - Return([]*tktypes.EthAddress{tktypes.RandAddress()}, nil) - }) + ctx, txm, done := newTestTransactionManager(t, false, + mockInsertABIAndTransactionOK(true), + mockSubmitPublicTxOk(t, tktypes.RandAddress()), + ) defer done() exampleABI := abi.ABI{{Type: abi.Function, Name: "doIt"}} @@ -205,7 +205,7 @@ func TestResolveFunctionHexInputOK(t *testing.T) { } func TestResolveFunctionHexInputFail(t *testing.T) { - ctx, txm, done := newTestTransactionManager(t, false, mockInsertABI) + ctx, txm, done := newTestTransactionManager(t, false, mockInsertABI, mockKeyResolutionContextFail(t)) defer done() exampleABI := abi.ABI{{Type: abi.Function, Name: "doIt", Inputs: abi.ParameterArray{{Type: "uint256"}}}} @@ -223,7 +223,7 @@ func TestResolveFunctionHexInputFail(t *testing.T) { } func TestResolveFunctionUnsupportedInput(t *testing.T) { - ctx, txm, done := newTestTransactionManager(t, false, mockInsertABI) + ctx, txm, done := newTestTransactionManager(t, false, mockInsertABI, mockKeyResolutionContextFail(t)) defer done() exampleABI := abi.ABI{{Type: abi.Function, Name: "doIt", Inputs: abi.ParameterArray{{Type: "uint256"}}}} @@ -241,11 +241,7 @@ func TestResolveFunctionUnsupportedInput(t *testing.T) { } func TestResolveFunctionPlainNameOK(t *testing.T) { - ctx, txm, done := newTestTransactionManager(t, false, mockInsertABIAndTransactionOK, mockPublicSubmitWithDataTxOk(t), - func(conf *pldconf.TxManagerConfig, mc *mockComponents) { - mc.keyManager.On("ResolveEthAddressBatchNewDatabaseTX", mock.Anything, []string{"sender1"}). - Return([]*tktypes.EthAddress{tktypes.RandAddress()}, nil) - }) + ctx, txm, done := newTestTransactionManager(t, false, mockInsertABIAndTransactionOK(true), mockSubmitPublicTxOk(t, tktypes.RandAddress())) defer done() exampleABI := abi.ABI{{Type: abi.Function, Name: "doIt"}} @@ -266,9 +262,10 @@ func TestResolveFunctionPlainNameOK(t *testing.T) { } func TestSendTransactionPrivateDeploy(t *testing.T) { - ctx, txm, done := newTestTransactionManager(t, false, mockInsertABIAndTransactionOK, func(conf *pldconf.TxManagerConfig, mc *mockComponents) { - mc.privateTxMgr.On("HandleNewTx", mock.Anything, mock.Anything).Return(nil) - }) + ctx, txm, done := newTestTransactionManager(t, false, mockInsertABIAndTransactionOK(true), mockKeyResolutionContextOk(t), + func(conf *pldconf.TxManagerConfig, mc *mockComponents) { + mc.privateTxMgr.On("HandleNewTx", mock.Anything, mock.Anything, mock.Anything).Return(nil) + }) defer done() exampleABI := abi.ABI{{Type: abi.Constructor}} @@ -288,9 +285,10 @@ func TestSendTransactionPrivateDeploy(t *testing.T) { } func TestSendTransactionPrivateInvoke(t *testing.T) { - ctx, txm, done := newTestTransactionManager(t, false, mockInsertABIAndTransactionOK, func(conf *pldconf.TxManagerConfig, mc *mockComponents) { - mc.privateTxMgr.On("HandleNewTx", mock.Anything, mock.Anything).Return(nil) - }) + ctx, txm, done := newTestTransactionManager(t, false, mockInsertABIAndTransactionOK(true), mockKeyResolutionContextOk(t), + func(conf *pldconf.TxManagerConfig, mc *mockComponents) { + mc.privateTxMgr.On("HandleNewTx", mock.Anything, mock.Anything, mock.Anything).Return(nil) + }) defer done() exampleABI := abi.ABI{{Type: abi.Function, Name: "doIt"}} @@ -312,9 +310,10 @@ func TestSendTransactionPrivateInvoke(t *testing.T) { } func TestSendTransactionPrivateInvokeFail(t *testing.T) { - ctx, txm, done := newTestTransactionManager(t, false, mockInsertABIAndTransactionOK, func(conf *pldconf.TxManagerConfig, mc *mockComponents) { - mc.privateTxMgr.On("HandleNewTx", mock.Anything, mock.Anything).Return(fmt.Errorf("pop")) - }) + ctx, txm, done := newTestTransactionManager(t, false, mockInsertABIAndTransactionOK(false), mockKeyResolutionContextFail(t), + func(conf *pldconf.TxManagerConfig, mc *mockComponents) { + mc.privateTxMgr.On("HandleNewTx", mock.Anything, mock.Anything, mock.Anything).Return(fmt.Errorf("pop")) + }) defer done() exampleABI := abi.ABI{{Type: abi.Function, Name: "doIt"}} @@ -336,11 +335,7 @@ func TestSendTransactionPrivateInvokeFail(t *testing.T) { } func TestResolveFunctionOnlyOneToMatch(t *testing.T) { - ctx, txm, done := newTestTransactionManager(t, false, mockInsertABIAndTransactionOK, mockPublicSubmitWithDataTxOk(t), - func(conf *pldconf.TxManagerConfig, mc *mockComponents) { - mc.keyManager.On("ResolveEthAddressBatchNewDatabaseTX", mock.Anything, []string{"sender1"}). - Return([]*tktypes.EthAddress{tktypes.RandAddress()}, nil) - }) + ctx, txm, done := newTestTransactionManager(t, false, mockInsertABIAndTransactionOK(true), mockSubmitPublicTxOk(t, tktypes.RandAddress())) defer done() exampleABI := abi.ABI{{Type: abi.Function, Name: "doIt"}} @@ -360,7 +355,7 @@ func TestResolveFunctionOnlyOneToMatch(t *testing.T) { } func TestResolveFunctionOnlyDuplicateMatch(t *testing.T) { - ctx, txm, done := newTestTransactionManager(t, false, mockInsertABI) + ctx, txm, done := newTestTransactionManager(t, false, mockInsertABI, mockKeyResolutionContextFail(t)) defer done() exampleABI := abi.ABI{ @@ -382,7 +377,7 @@ func TestResolveFunctionOnlyDuplicateMatch(t *testing.T) { } func TestResolveFunctionNoMatch(t *testing.T) { - ctx, txm, done := newTestTransactionManager(t, false, mockInsertABI) + ctx, txm, done := newTestTransactionManager(t, false, mockInsertABI, mockKeyResolutionContextFail(t)) defer done() exampleABI := abi.ABI{ @@ -404,7 +399,7 @@ func TestResolveFunctionNoMatch(t *testing.T) { } func TestParseInputsBadTxType(t *testing.T) { - ctx, txm, done := newTestTransactionManager(t, false) + ctx, txm, done := newTestTransactionManager(t, false, mockKeyResolutionContextFail(t), mockBeginRollback) defer done() exampleABI := abi.ABI{ @@ -424,7 +419,7 @@ func TestParseInputsBadTxType(t *testing.T) { } func TestParseInputsBadFromRemoteNode(t *testing.T) { - ctx, txm, done := newTestTransactionManager(t, false, mockInsertABI) + ctx, txm, done := newTestTransactionManager(t, false, mockInsertABI, mockKeyResolutionContextFail(t)) defer done() exampleABI := abi.ABI{ @@ -446,7 +441,7 @@ func TestParseInputsBadFromRemoteNode(t *testing.T) { } func TestParseInputsBytecodeNonConstructor(t *testing.T) { - ctx, txm, done := newTestTransactionManager(t, false, mockInsertABI) + ctx, txm, done := newTestTransactionManager(t, false, mockInsertABI, mockKeyResolutionContextFail(t)) defer done() exampleABI := abi.ABI{ @@ -468,7 +463,7 @@ func TestParseInputsBytecodeNonConstructor(t *testing.T) { } func TestParseInputsBytecodeMissingConstructor(t *testing.T) { - ctx, txm, done := newTestTransactionManager(t, false, mockInsertABI) + ctx, txm, done := newTestTransactionManager(t, false, mockInsertABI, mockKeyResolutionContextFail(t)) defer done() exampleABI := abi.ABI{ @@ -488,7 +483,7 @@ func TestParseInputsBytecodeMissingConstructor(t *testing.T) { } func TestParseInputsBadDataJSON(t *testing.T) { - ctx, txm, done := newTestTransactionManager(t, false, mockInsertABI) + ctx, txm, done := newTestTransactionManager(t, false, mockInsertABI, mockKeyResolutionContextFail(t)) defer done() exampleABI := abi.ABI{ @@ -507,7 +502,7 @@ func TestParseInputsBadDataJSON(t *testing.T) { } func TestParseInputsBadDataForFunction(t *testing.T) { - ctx, txm, done := newTestTransactionManager(t, false, mockInsertABI) + ctx, txm, done := newTestTransactionManager(t, false, mockInsertABI, mockKeyResolutionContextFail(t)) defer done() exampleABI := abi.ABI{ @@ -526,7 +521,7 @@ func TestParseInputsBadDataForFunction(t *testing.T) { } func TestParseInputsBadByteString(t *testing.T) { - ctx, txm, done := newTestTransactionManager(t, false, mockInsertABI) + ctx, txm, done := newTestTransactionManager(t, false, mockInsertABI, mockKeyResolutionContextFail(t)) defer done() exampleABI := abi.ABI{ @@ -544,26 +539,16 @@ func TestParseInputsBadByteString(t *testing.T) { assert.Regexp(t, "PD012208", err) } -func mockPublicSubmitTxRollback(t *testing.T) func(conf *pldconf.TxManagerConfig, mc *mockComponents) { - return func(conf *pldconf.TxManagerConfig, mc *mockComponents) { - mockSubmissionBatch := componentmocks.NewPublicTxBatch(t) - mockSubmissionBatch.On("Rejected").Return([]components.PublicTxRejected{}) - mockSubmissionBatch.On("Completed", mock.Anything, false).Return(nil) - mc.publicTxMgr.On("PrepareSubmissionBatch", mock.Anything, mock.Anything).Return(mockSubmissionBatch, nil) - } -} func TestInsertTransactionFail(t *testing.T) { ctx, txm, done := newTestTransactionManager(t, false, func(conf *pldconf.TxManagerConfig, mc *mockComponents) { + mc.db.ExpectBegin() mc.db.ExpectExec("INSERT.*abis").WillReturnResult(driver.ResultNoRows) mc.db.ExpectExec("INSERT.*abi_entries").WillReturnResult(driver.ResultNoRows) - mc.db.ExpectBegin() mc.db.ExpectExec("INSERT.*transactions").WillReturnError(fmt.Errorf("pop")) mc.db.ExpectRollback() - - mc.keyManager.On("ResolveEthAddressBatchNewDatabaseTX", mock.Anything, []string{"sender1"}). - Return([]*tktypes.EthAddress{tktypes.RandAddress()}, nil) - - }, mockPublicSubmitTxRollback(t)) + mockResolveKeyOKThenFail(t, mc, "sender1", tktypes.RandAddress()) + mc.publicTxMgr.On("ValidateTransaction", mock.Anything, mock.Anything, mock.Anything).Return(nil) + }) defer done() exampleABI := abi.ABI{{Type: abi.Function, Name: "doIt"}} @@ -583,18 +568,8 @@ func TestInsertTransactionFail(t *testing.T) { func TestInsertTransactionPublicTxPrepareFail(t *testing.T) { ctx, txm, done := newTestTransactionManager(t, false, mockInsertABI, func(conf *pldconf.TxManagerConfig, mc *mockComponents) { - mockSubmissionBatch := componentmocks.NewPublicTxBatch(t) - rejectedSubmission := componentmocks.NewPublicTxRejected(t) - rejectedSubmission.On("RejectedError").Return(fmt.Errorf("pop")) - mockSubmissionBatch.On("Rejected").Return([]components.PublicTxRejected{ - rejectedSubmission, - }) - mockSubmissionBatch.On("Completed", mock.Anything, false).Return(nil) - mc.publicTxMgr.On("PrepareSubmissionBatch", mock.Anything, mock.Anything).Return(mockSubmissionBatch, nil) - - mc.keyManager.On("ResolveEthAddressBatchNewDatabaseTX", mock.Anything, []string{"sender1"}). - Return([]*tktypes.EthAddress{tktypes.RandAddress()}, nil) - + mockResolveKeyOKThenFail(t, mc, "sender1", tktypes.RandAddress()) + mc.publicTxMgr.On("ValidateTransaction", mock.Anything, mock.Anything, mock.Anything).Return(fmt.Errorf("pop")) }) defer done() @@ -615,10 +590,10 @@ func TestInsertTransactionPublicTxPrepareFail(t *testing.T) { func TestInsertTransactionPublicTxPrepareReject(t *testing.T) { ctx, txm, done := newTestTransactionManager(t, false, mockInsertABI, func(conf *pldconf.TxManagerConfig, mc *mockComponents) { - mc.keyManager.On("ResolveEthAddressBatchNewDatabaseTX", mock.Anything, []string{"sender1"}). - Return([]*tktypes.EthAddress{tktypes.RandAddress()}, nil) - - mc.publicTxMgr.On("PrepareSubmissionBatch", mock.Anything, mock.Anything).Return(nil, fmt.Errorf("pop")) + mockResolveKeyOKThenFail(t, mc, "sender1", tktypes.RandAddress()) + mc.publicTxMgr.On("ValidateTransaction", mock.Anything, mock.Anything, mock.Anything).Return(nil) + mc.db.ExpectExec("INSERT.*transactions").WillReturnResult(driver.ResultNoRows) + mc.publicTxMgr.On("WriteNewTransactions", mock.Anything, mock.Anything, mock.Anything).Return(nil, nil, fmt.Errorf("pop")) }) defer done() @@ -634,11 +609,9 @@ func TestInsertTransactionPublicTxPrepareReject(t *testing.T) { } func TestInsertTransactionOkDefaultConstructor(t *testing.T) { - ctx, txm, done := newTestTransactionManager(t, false, mockInsertABIAndTransactionOK, mockPublicSubmitWithDataTxOk(t), - func(conf *pldconf.TxManagerConfig, mc *mockComponents) { - mc.keyManager.On("ResolveEthAddressBatchNewDatabaseTX", mock.Anything, []string{"sender1"}). - Return([]*tktypes.EthAddress{tktypes.RandAddress()}, nil) - }) + ctx, txm, done := newTestTransactionManager(t, false, + mockInsertABIAndTransactionOK(true), + mockSubmitPublicTxOk(t, tktypes.RandAddress())) defer done() // Default public constructor invoke @@ -660,7 +633,7 @@ func TestCheckIdempotencyKeyNoOverrideErrIfFail(t *testing.T) { defer done() // Default public constructor invoke - err := txm.checkIdempotencyKeys(ctx, fmt.Errorf("pop"), false, []*pldapi.TransactionInput{{TransactionBase: pldapi.TransactionBase{ + err := txm.checkIdempotencyKeys(ctx, fmt.Errorf("pop"), []*pldapi.TransactionInput{{TransactionBase: pldapi.TransactionBase{ IdempotencyKey: "idem1", }}}) assert.Regexp(t, "pop", err) @@ -684,7 +657,7 @@ func TestCallTransactionNoFrom(t *testing.T) { ec := ethclient.NewUnconnectedRPCClient(context.Background(), &pldconf.EthClientConfig{}, 0) ctx, txm, done := newTestTransactionManager(t, false, - mockInsertABI, + mockInsertABINoBegin, func(conf *pldconf.TxManagerConfig, mc *mockComponents) { mc.ethClientFactory.On("HTTPClient").Return(ec) }) @@ -720,7 +693,7 @@ func TestCallTransactionWithFrom(t *testing.T) { ec := ethclient.NewUnconnectedRPCClient(context.Background(), &pldconf.EthClientConfig{}, 0) ctx, txm, done := newTestTransactionManager(t, false, - mockInsertABI, + mockInsertABINoBegin, func(conf *pldconf.TxManagerConfig, mc *mockComponents) { mc.keyManager.On("ResolveEthAddressNewDatabaseTX", mock.Anything, "red.one"). Return(tktypes.RandAddress(), nil) @@ -775,7 +748,7 @@ func TestCallTransactionPrivOk(t *testing.T) { }, } - ctx, txm, done := newTestTransactionManager(t, false, mockInsertABI, func(conf *pldconf.TxManagerConfig, mc *mockComponents) { + ctx, txm, done := newTestTransactionManager(t, false, mockInsertABINoBegin, func(conf *pldconf.TxManagerConfig, mc *mockComponents) { res, err := fnDef.Outputs.ParseJSON([]byte(`{"spins": 42}`)) require.NoError(t, err) @@ -804,7 +777,7 @@ func TestCallTransactionPrivOk(t *testing.T) { func TestCallTransactionPrivFail(t *testing.T) { fnDef := &abi.Entry{Name: "ohSnap", Type: abi.Function} - ctx, txm, done := newTestTransactionManager(t, false, mockInsertABI, func(conf *pldconf.TxManagerConfig, mc *mockComponents) { + ctx, txm, done := newTestTransactionManager(t, false, mockInsertABINoBegin, func(conf *pldconf.TxManagerConfig, mc *mockComponents) { mc.privateTxMgr.On("CallPrivateSmartContract", mock.Anything, mock.Anything). Return(nil, fmt.Errorf("snap")) }) @@ -825,7 +798,7 @@ func TestCallTransactionPrivFail(t *testing.T) { } func TestCallTransactionPrivMissingTo(t *testing.T) { - ctx, txm, done := newTestTransactionManager(t, false, mockInsertABI) + ctx, txm, done := newTestTransactionManager(t, false, mockInsertABINoBegin) defer done() err := txm.CallTransaction(ctx, nil, &pldapi.TransactionCall{ @@ -840,7 +813,7 @@ func TestCallTransactionPrivMissingTo(t *testing.T) { } func TestCallTransactionBadSerializer(t *testing.T) { - ctx, txm, done := newTestTransactionManager(t, false, mockInsertABI) + ctx, txm, done := newTestTransactionManager(t, false, mockInsertABINoBegin) defer done() err := txm.CallTransaction(ctx, nil, &pldapi.TransactionCall{ diff --git a/core/go/pkg/blockindexer/block_indexer.go b/core/go/pkg/blockindexer/block_indexer.go index 3e4deaf78..55dc53081 100644 --- a/core/go/pkg/blockindexer/block_indexer.go +++ b/core/go/pkg/blockindexer/block_indexer.go @@ -993,7 +993,7 @@ func (bi *blockIndexer) matchLog(ctx context.Context, abi abi.ABI, in *LogJSONRP } return true } else { - log.L(ctx).Debugf("Event %d/%d/%d does not match ABI event %s matchSource=%v (tx=%s,address=%s): %s", in.BlockNumber, in.TransactionIndex, in.LogIndex, abiEntry, source, in.TransactionHash, in.Address, err) + log.L(ctx).Tracef("Event %d/%d/%d does not match ABI event %s matchSource=%v (tx=%s,address=%s): %s", in.BlockNumber, in.TransactionIndex, in.LogIndex, abiEntry, source, in.TransactionHash, in.Address, err) } } return false diff --git a/core/go/pkg/ethclient/client.go b/core/go/pkg/ethclient/client.go index 434334247..625bcb54c 100644 --- a/core/go/pkg/ethclient/client.go +++ b/core/go/pkg/ethclient/client.go @@ -23,7 +23,6 @@ import ( "fmt" "math/big" "strings" - "time" "github.com/hyperledger/firefly-common/pkg/fftypes" "github.com/hyperledger/firefly-common/pkg/i18n" @@ -193,33 +192,13 @@ func (ec *ethClient) ChainID() int64 { return ec.chainID } -// setupChainID is a helper to get the chain ID from the node -// we assume the node could intermittently have issues, be starting up -// in paralle, etc. and therefore implement an expotential backoff mechanism -// in the event of initial failure. func (ec *ethClient) setupChainID(ctx context.Context) error { var chainID ethtypes.HexUint64 - var rpcErr error - - const maxRetries = 5 - const baseDelay = 1 * time.Second - for retries := int64(0); retries < maxRetries; retries++ { - rpcErr = ec.rpc.CallRPC(ctx, &chainID, "eth_chainId") - if rpcErr == nil { - break - } - log.L(ctx).Warnf("eth_chainId failed, retrying: %+v", rpcErr) - // Calculate exponential backoff delay - backoff := baseDelay * time.Duration(1<... + */ +public class VirtualMachineDiagnostics +{ + /** Object Name of DiagnosticCommandMBean. */ + public final static String DIAGNOSTIC_COMMAND_MBEAN_OBJECT_NAME = + "com.sun.management:type=DiagnosticCommand"; + + /** My MBean Server. */ + private final MBeanServer server = ManagementFactory.getPlatformMBeanServer(); + + /** Platform MBean Server. */ + private final ObjectName objectName; + + /** + * Create an instance of me with the provided object name. + * + * @param newObjectName ObjectName associated with + * DiagnosticCommand MBean. + */ + private VirtualMachineDiagnostics(final ObjectName newObjectName) + { + this.objectName = newObjectName; + } + + /** + * Only publicly available method for instantiating an instance + * of me. + * + * @return An instance of me. + */ + public static VirtualMachineDiagnostics newInstance() + { + try + { + final ObjectName objectName = new ObjectName(DIAGNOSTIC_COMMAND_MBEAN_OBJECT_NAME); + return new VirtualMachineDiagnostics(objectName); + } + catch (MalformedObjectNameException badObjectNameEx) + { + throw new RuntimeException( + "Unable to create an ObjectName and so unable to create instance of VirtualMachineDiagnostics"); + } + } + + /** + * Provide class statistics as single String. + * + * This is only supported when {@code -XX:+UnlockDiagnosticVMOptions} is enabled. + * + * @return Single string containing formatted class statistics. + */ + public String getClassStatistics() + { + return invokeNoStringArgumentsCommand("gcClassStats", "GC Class Statistics"); + } + + /** + * Provide class histogram as single String. + * + * @return Single string containing formatted class histogram. + */ + public String getHistogram() + { + return invokeNoStringArgumentsCommand("gcClassHistogram", "GC Class Histogram"); + } + + /** + * Provide thread dump as single String. + * + * @return Single string containing formatted thread dump. + */ + public String getThreadDump() + { + return invokeNoStringArgumentsCommand("threadPrint", "Thread Dump"); + } + + /** + * Provide virtual machine uptime as single String. + * + * @return Single string containing virtual machine uptime. + */ + public String getVirtualMachineUptime() + { + return invokeNoStringArgumentsCommand("vmUptime", "Virtual Machine Uptime"); + } + + /** + * Provide list of supported operations (help). + * + * @return Single string containing names of supported operations. + */ + public String getAvailableOperations() + { + return invokeNoStringArgumentsCommand("help", "Help (List Commands)"); + } + + /** + * Provide a String representing the Virtual Machine flags. + * + * @return String containing the virtual machine flags. + */ + public String getVirtualMachineFlags() + { + return invokeNoStringArgumentsCommand("vmFlags", "Determine VM flags"); + } + + /** + * Provide String representing active/current garbage collector. + * + * @return String representation of current garbage collector + * ("Parallel/Throughput", "Concurrent Mark Sweep (CMS)", + * "Garbage First", or "UNDETERMINED"). + */ + public String determineGarbageCollector() + { + String garbageCollector; + final String vmFlags = getVirtualMachineFlags(); + if (vmFlags.contains("+UseParallelGC") || vmFlags.contains("+UseParallelOldGC")) + { + garbageCollector = "Parallel/Throughput"; + } + else if (vmFlags.contains("+UseConcMarkSweepGC")) + { + garbageCollector = "Concurrent Mark Sweep (CMS)"; + } + else if (vmFlags.contains("+UseG1GC")) + { + garbageCollector = "Garbage First"; + } + else + { + garbageCollector = "UNDETERMINED"; + } + return garbageCollector; + } + + /** + * Invoke operation on the DiagnosticCommandMBean that accepts + * String array argument but does not require any String + * argument and returns a String. + * + * @param operationName Name of operation on DiagnosticCommandMBean. + * @param operationDescription Description of operation being invoked + * on the DiagnosticCommandMBean. + * @return String returned by DiagnosticCommandMBean operation. + */ + private String invokeNoStringArgumentsCommand( + final String operationName, final String operationDescription) + { + String result; + try + { + result = (String) server.invoke(objectName, operationName, new Object[] {null}, new String[]{String[].class.getName()}); + } + catch (InstanceNotFoundException | ReflectionException | MBeanException exception) + { + result = "ERROR: Unable to access '" + operationDescription + "' - " + exception; + } + return result; + } + + public static void main(final String[] arguments) + { + final VirtualMachineDiagnostics instance = VirtualMachineDiagnostics.newInstance(); + out.println("VM Flags:\n\t" + instance.getVirtualMachineFlags()); + out.println("Garbage Collector: " + instance.determineGarbageCollector()); + out.println("Histogram:\n" + instance.getHistogram()); + out.println("Thread Stack:\n" + instance.getThreadDump()); + out.println("VM Uptime: " + instance.getVirtualMachineUptime()); + out.println("Class Statistics: " + instance.getClassStatistics()); + out.println("Supported Operations:\n" + instance.getAvailableOperations()); + } +} diff --git a/core/java/src/main/java/io/kaleido/paladin/loader/PluginLoader.java b/core/java/src/main/java/io/kaleido/paladin/loader/PluginLoader.java index b4d90d935..d9a76d2eb 100644 --- a/core/java/src/main/java/io/kaleido/paladin/loader/PluginLoader.java +++ b/core/java/src/main/java/io/kaleido/paladin/loader/PluginLoader.java @@ -15,6 +15,7 @@ package io.kaleido.paladin.loader; +import io.kaleido.paladin.diagnostics.VirtualMachineDiagnostics; import io.kaleido.paladin.toolkit.PluginControllerGrpc; import io.kaleido.paladin.toolkit.Service; import io.kaleido.paladin.toolkit.Service.PluginLoad; @@ -133,8 +134,25 @@ private synchronized void scheduleConnect() { CompletableFuture.delayedExecutor(delay, TimeUnit.MILLISECONDS)); } + private void threadDump() { + try { + System.err.println(VirtualMachineDiagnostics.newInstance().getThreadDump()); + } catch(Throwable e) { + LOGGER.error("failed to initiate thread dump", e); + } + } + @Override public void onNext(PluginLoad loadInstruction) { + if (loadInstruction.hasSysCommand()) { + // we are processing a system command rather than an actual load + switch (loadInstruction.getSysCommand()) { + case THREAD_DUMP -> threadDump(); + default -> LOGGER.warn("unrecognized system command {}", loadInstruction.getSysCommand()); + } + return; + } + PluginInfo info = new PluginInfo(grpcTarget, loadInstruction.getPlugin().getPluginType().toString(), loadInstruction.getPlugin().getName(), loadInstruction.getPlugin().getId()); LOGGER.info("load instruction for {} {} [{}] libType={} location={} class={}", diff --git a/core/java/src/test/java/io/kaleido/paladin/TestStartTestbedWithNoopDomains.java b/core/java/src/test/java/io/kaleido/paladin/TestStartTestbedWithNoopDomains.java index 8662485b8..00022f74a 100644 --- a/core/java/src/test/java/io/kaleido/paladin/TestStartTestbedWithNoopDomains.java +++ b/core/java/src/test/java/io/kaleido/paladin/TestStartTestbedWithNoopDomains.java @@ -15,6 +15,7 @@ package io.kaleido.paladin; +import io.kaleido.paladin.diagnostics.VirtualMachineDiagnostics; import io.kaleido.paladin.testbed.Testbed; import io.kaleido.paladin.toolkit.JsonHex; import org.junit.jupiter.api.Test; diff --git a/doc-site/docs/reference/types/publictx.md b/doc-site/docs/reference/types/publictx.md index 356c1b7c0..54b873785 100644 --- a/doc-site/docs/reference/types/publictx.md +++ b/doc-site/docs/reference/types/publictx.md @@ -8,7 +8,7 @@ title: PublicTx ```json { "from": "0x0000000000000000000000000000000000000000", - "nonce": "0x0", + "nonce": null, "created": 0, "transactionHash": null } @@ -18,6 +18,7 @@ title: PublicTx | Field Name | Description | Type | |------------|-------------|------| +| `localId` | A locally generated numeric ID for the public transaction. Unique within the node | `uint64` | | `to` | The target contract address (optional) | [`EthAddress`](simpletypes.md#ethaddress) | | `data` | The pre-encoded calldata (optional) | [`HexBytes`](simpletypes.md#hexbytes) | | `from` | The sender's Ethereum address | [`EthAddress`](simpletypes.md#ethaddress) | diff --git a/domains/integration-test/go.sum b/domains/integration-test/go.sum index c077d2844..91a9aa0d7 100644 --- a/domains/integration-test/go.sum +++ b/domains/integration-test/go.sum @@ -119,16 +119,21 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/hyperledger-labs/zeto/go-sdk v0.0.0-20241004174307-aa3c1fdf0966 h1:J5ZVvMRxRlM1GLtUdPTCNNRpB4QpJij/+ndud1QU8WY= +github.com/hyperledger-labs/zeto/go-sdk v0.0.0-20241004174307-aa3c1fdf0966/go.mod h1:WyUa1UIizlcBQnTEMK5tWWskFz2glSTm7S6HaJTarps= github.com/hyperledger/firefly-common v1.4.11 h1:WKv2hQuNpS7yP51THxzpzrqU3jkln23C9vq5iminzBk= github.com/hyperledger/firefly-common v1.4.11/go.mod h1:E7w/RxNtVnX52WXLQW9f2xVAgZnW70voZeE9sZrx/q0= github.com/hyperledger/firefly-signer v1.1.19-0.20241027192206-656dd986267e h1:iqIs0NPtE9h1Vy4WBm2dsS4XbBIdpZPMNprlwdmIC0U= +github.com/hyperledger/firefly-signer v1.1.19-0.20241027192206-656dd986267e/go.mod h1:HDaDdht94JypRTunRGrcPL5Pvxfh4yigjatTrie5JUI= github.com/iden3/go-iden3-crypto v0.0.17 h1:NdkceRLJo/pI4UpcjVah4lN/a3yzxRUGXqxbWcYh9mY= github.com/iden3/go-iden3-crypto v0.0.17/go.mod h1:dLpM4vEPJ3nDHzhWFXDjzkn1qHoBeOT/3UEhXsEsP3E= github.com/iden3/go-rapidsnark/prover v0.0.12 h1:IOqZrK+0J91zU7goB74qAzdwPg1ZRpZvsGjgpz4c+MQ= +github.com/iden3/go-rapidsnark/prover v0.0.12/go.mod h1:mUNLeDXYOW2igiPuhHZHD7kzSp/GjHSWND03aYyECvQ= github.com/iden3/go-rapidsnark/types v0.0.3 h1:f0s1Qdut1qHe1O67+m+xUVRBPwSXnq5j0xSrBi0jqM4= +github.com/iden3/go-rapidsnark/types v0.0.3/go.mod h1:ApgcaUxKIgSRA6fAeFxK7p+lgXXfG4oA2HN5DhFlfF4= github.com/iden3/go-rapidsnark/witness/v2 v2.0.0 h1:mkY6VDfwKVJc83QGKmwVXY2LYepidPrFAxskrjr8UCs= github.com/iden3/go-rapidsnark/witness/v2 v2.0.0/go.mod h1:3JRjqUfW1hgI9hzLDO0v8z/DUkR0ZUehhYLlnIfRxnA= github.com/iden3/go-rapidsnark/witness/wasmer v0.0.0-20240914111027-9588ce2d7e1b h1:7z3IPlhkPGx8bwJvZGC8uKDH0SUYOss2DYBzlLZZF2c= +github.com/iden3/go-rapidsnark/witness/wasmer v0.0.0-20240914111027-9588ce2d7e1b/go.mod h1:WUtPVKXrhfZHJXavwId2+8J/fKMHQ92N0MZDxt8sfEA= github.com/iden3/wasmer-go v0.0.1 h1:TZKh8Se8B/73PvWrcu+FTU9L1k5XYAmtFbioj7l0Uog= github.com/iden3/wasmer-go v0.0.1/go.mod h1:ZnZBAO012M7o+Q1INXLRIxKQgEcH2FuwL0Iga8A4ufg= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= @@ -246,6 +251,7 @@ github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIK github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w= +github.com/spf13/cast v1.7.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= @@ -289,6 +295,7 @@ golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDf golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= +golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= @@ -313,6 +320,7 @@ golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= +golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -345,6 +353,7 @@ golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -355,6 +364,7 @@ golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24= +golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -366,6 +376,7 @@ golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= +golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -380,7 +391,9 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/genproto/googleapis/rpc v0.0.0-20241007155032-5fefd90f89a9 h1:QCqS/PdaHTSWGvupk2F/ehwHtGc0/GYkT+3GAcR1CCc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241007155032-5fefd90f89a9/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI= google.golang.org/grpc v1.67.1 h1:zWnc1Vrcno+lHZCOofnIMvycFcc0QRGIzm9dhnDX68E= +google.golang.org/grpc v1.67.1/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -388,6 +401,7 @@ google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miE google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= +google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/domains/noto/go.mod b/domains/noto/go.mod index 25a485fdd..497a8edc8 100644 --- a/domains/noto/go.mod +++ b/domains/noto/go.mod @@ -4,6 +4,7 @@ go 1.22.5 require ( github.com/go-resty/resty/v2 v2.14.0 + github.com/google/uuid v1.6.0 github.com/hyperledger/firefly-common v1.4.11 github.com/hyperledger/firefly-signer v1.1.19-0.20241027192206-656dd986267e github.com/kaleido-io/paladin/core v0.0.0-00010101000000-000000000000 @@ -30,7 +31,6 @@ require ( github.com/go-openapi/jsonpointer v0.20.2 // indirect github.com/go-openapi/swag v0.22.7 // indirect github.com/golang-migrate/migrate/v4 v4.17.1 // indirect - github.com/google/uuid v1.6.0 // indirect github.com/gorilla/mux v1.8.1 // indirect github.com/gorilla/websocket v1.5.3 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect diff --git a/domains/noto/go.sum b/domains/noto/go.sum index 1a1c50c65..49a6ca6d5 100644 --- a/domains/noto/go.sum +++ b/domains/noto/go.sum @@ -119,6 +119,7 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO github.com/hyperledger/firefly-common v1.4.11 h1:WKv2hQuNpS7yP51THxzpzrqU3jkln23C9vq5iminzBk= github.com/hyperledger/firefly-common v1.4.11/go.mod h1:E7w/RxNtVnX52WXLQW9f2xVAgZnW70voZeE9sZrx/q0= github.com/hyperledger/firefly-signer v1.1.19-0.20241027192206-656dd986267e h1:iqIs0NPtE9h1Vy4WBm2dsS4XbBIdpZPMNprlwdmIC0U= +github.com/hyperledger/firefly-signer v1.1.19-0.20241027192206-656dd986267e/go.mod h1:HDaDdht94JypRTunRGrcPL5Pvxfh4yigjatTrie5JUI= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/invopop/yaml v0.2.0 h1:7zky/qH+O0DwAyoobXUqvVBwgBFRxKoQ/3FjcVpjTMY= @@ -176,6 +177,7 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY= +github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= @@ -231,6 +233,7 @@ github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIK github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w= +github.com/spf13/cast v1.7.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= @@ -274,6 +277,7 @@ golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDf golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= +golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= @@ -298,6 +302,7 @@ golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= +golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -330,6 +335,7 @@ golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -340,6 +346,7 @@ golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24= +golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -351,6 +358,7 @@ golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= +golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -365,7 +373,9 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/genproto/googleapis/rpc v0.0.0-20241007155032-5fefd90f89a9 h1:QCqS/PdaHTSWGvupk2F/ehwHtGc0/GYkT+3GAcR1CCc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241007155032-5fefd90f89a9/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI= google.golang.org/grpc v1.67.1 h1:zWnc1Vrcno+lHZCOofnIMvycFcc0QRGIzm9dhnDX68E= +google.golang.org/grpc v1.67.1/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -373,6 +383,7 @@ google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miE google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= +google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/domains/zeto/go.mod b/domains/zeto/go.mod index 03b29fb6d..03bebe962 100644 --- a/domains/zeto/go.mod +++ b/domains/zeto/go.mod @@ -14,6 +14,7 @@ require ( github.com/iden3/go-rapidsnark/witness/wasmer v0.0.0-20240914111027-9588ce2d7e1b github.com/kaleido-io/paladin/config v0.0.0-00010101000000-000000000000 github.com/kaleido-io/paladin/core v0.0.0-00010101000000-000000000000 + github.com/kaleido-io/paladin/domains/integration-test v0.0.0-20241115165222-77f5db364e58 github.com/kaleido-io/paladin/toolkit v0.0.0-00010101000000-000000000000 github.com/stretchr/testify v1.9.0 golang.org/x/text v0.19.0 @@ -56,6 +57,7 @@ require ( github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/josharian/intern v1.0.0 // indirect + github.com/kaleido-io/paladin/domains/noto v0.0.0-00010101000000-000000000000 // indirect github.com/lib/pq v1.10.9 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect @@ -94,7 +96,6 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.28.0 // indirect golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect - golang.org/x/mod v0.20.0 // indirect golang.org/x/net v0.30.0 // indirect golang.org/x/sync v0.8.0 // indirect golang.org/x/sys v0.26.0 // indirect @@ -116,3 +117,5 @@ replace github.com/kaleido-io/paladin/core => ../../core/go replace github.com/kaleido-io/paladin/toolkit => ../../toolkit/go replace github.com/kaleido-io/paladin/config => ../../config + +replace github.com/kaleido-io/paladin/domains/noto => ../noto diff --git a/domains/zeto/go.sum b/domains/zeto/go.sum index 195438785..c19a2ac6a 100644 --- a/domains/zeto/go.sum +++ b/domains/zeto/go.sum @@ -123,6 +123,7 @@ github.com/hyperledger-labs/zeto/go-sdk v0.0.0-20241004174307-aa3c1fdf0966/go.mo github.com/hyperledger/firefly-common v1.4.11 h1:WKv2hQuNpS7yP51THxzpzrqU3jkln23C9vq5iminzBk= github.com/hyperledger/firefly-common v1.4.11/go.mod h1:E7w/RxNtVnX52WXLQW9f2xVAgZnW70voZeE9sZrx/q0= github.com/hyperledger/firefly-signer v1.1.19-0.20241027192206-656dd986267e h1:iqIs0NPtE9h1Vy4WBm2dsS4XbBIdpZPMNprlwdmIC0U= +github.com/hyperledger/firefly-signer v1.1.19-0.20241027192206-656dd986267e/go.mod h1:HDaDdht94JypRTunRGrcPL5Pvxfh4yigjatTrie5JUI= github.com/iden3/go-iden3-crypto v0.0.17 h1:NdkceRLJo/pI4UpcjVah4lN/a3yzxRUGXqxbWcYh9mY= github.com/iden3/go-iden3-crypto v0.0.17/go.mod h1:dLpM4vEPJ3nDHzhWFXDjzkn1qHoBeOT/3UEhXsEsP3E= github.com/iden3/go-rapidsnark/prover v0.0.12 h1:IOqZrK+0J91zU7goB74qAzdwPg1ZRpZvsGjgpz4c+MQ= @@ -158,6 +159,8 @@ github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/ github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= +github.com/kaleido-io/paladin/domains/integration-test v0.0.0-20241115165222-77f5db364e58 h1:dXUIEWt2DVC9jC/RPc8IFjNBEgDbaMtrQAWVkWd0Q+g= +github.com/kaleido-io/paladin/domains/integration-test v0.0.0-20241115165222-77f5db364e58/go.mod h1:TTvrf+Ql/n/KaVeFS7bqN/1Wb7hf/aWv3E74vDt7Lqs= github.com/karlseguin/ccache v2.0.3+incompatible h1:j68C9tWOROiOLWTS/kCGg9IcJG+ACqn5+0+t8Oh83UU= github.com/karlseguin/ccache v2.0.3+incompatible/go.mod h1:CM9tNPzT6EdRh14+jiW8mEF9mkNZuuE51qmgGYUB93w= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= @@ -194,6 +197,7 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY= +github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= @@ -249,6 +253,7 @@ github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIK github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w= +github.com/spf13/cast v1.7.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= diff --git a/domains/zeto/integration-test/e2e_test.go b/domains/zeto/integration-test/e2e_test.go index 81ca12017..5fe41b667 100644 --- a/domains/zeto/integration-test/e2e_test.go +++ b/domains/zeto/integration-test/e2e_test.go @@ -61,6 +61,7 @@ type zetoDomainTestSuite struct { } func (s *zetoDomainTestSuite) SetupSuite() { + log.SetLevel("debug") s.hdWalletSeed = testbed.HDWalletSeedScopedToTest() domainContracts := DeployZetoContracts(s.T(), s.hdWalletSeed, "./config-for-deploy.yaml", controllerName) s.deployedContracts = domainContracts diff --git a/operator/go.mod b/operator/go.mod index 34cacba08..e317c390d 100644 --- a/operator/go.mod +++ b/operator/go.mod @@ -14,6 +14,7 @@ require ( github.com/onsi/ginkgo/v2 v2.17.2 github.com/onsi/gomega v1.33.1 github.com/pelletier/go-toml/v2 v2.2.3 + github.com/sirupsen/logrus v1.9.3 github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.9.0 github.com/tyler-smith/go-bip39 v1.1.0 @@ -88,7 +89,6 @@ require ( github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 // indirect github.com/shopspring/decimal v1.4.0 // indirect - github.com/sirupsen/logrus v1.9.3 // indirect github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.7.0 // indirect diff --git a/operator/internal/controller/besu_controller.go b/operator/internal/controller/besu_controller.go index 554b9567e..b5f2e80e2 100644 --- a/operator/internal/controller/besu_controller.go +++ b/operator/internal/controller/besu_controller.go @@ -273,6 +273,7 @@ func (r *BesuReconciler) generateBesuConfigTOML(node *corev1alpha1.Besu) (string setIfUnset("logging", "DEBUG") setIfUnset("revert-reason-enabled", true) setIfUnset("tx-pool", "SEQUENCED") + setIfUnset("tx-pool-limit-by-account-percentage", 1.0) // To give a stable network through node restarts we use hostnames in static-nodes.json // https://besu.hyperledger.org/24.1.0/public-networks/concepts/node-keys#enode-url diff --git a/operator/internal/controller/paladin_controller.go b/operator/internal/controller/paladin_controller.go index e72de5653..25b1d7a4d 100644 --- a/operator/internal/controller/paladin_controller.go +++ b/operator/internal/controller/paladin_controller.go @@ -30,6 +30,7 @@ import ( "time" "github.com/Masterminds/sprig/v3" + "github.com/kaleido-io/paladin/config/pkg/confutil" "github.com/kaleido-io/paladin/config/pkg/pldconf" "github.com/tyler-smith/go-bip39" appsv1 "k8s.io/api/apps/v1" @@ -676,6 +677,14 @@ func (r *PaladinReconciler) generatePaladinConfig(ctx context.Context, node *cor } } + // Enable debug server by default on localhost:6060 + if pldConf.DebugServer.Enabled == nil { + pldConf.DebugServer.Enabled = confutil.P(true) + if pldConf.DebugServer.Port == nil { + pldConf.DebugServer.Port = confutil.P(6060) + } + } + // Node name can be overridden, but defaults to the CR name if pldConf.NodeName == "" { pldConf.NodeName = node.Name diff --git a/operator/test/e2e/e2e_bonds_test.go b/operator/test/e2e/e2e_bonds_test.go index cdb006a4b..13175c2b1 100644 --- a/operator/test/e2e/e2e_bonds_test.go +++ b/operator/test/e2e/e2e_bonds_test.go @@ -53,6 +53,7 @@ var _ = Describe("controller", Ordered, func() { defer GinkgoRecover() BeforeAll(func() { + // Skip("for now") }) AfterAll(func() { @@ -215,7 +216,7 @@ var _ = Describe("controller", Ordered, func() { var privacyGroups map[string]*privacyGroup - It("resovles participants", func() { + It("resolves participants", func() { resolveParticipant := func(identity string, node string) *participant { addr, err := rpc[node].PTX().ResolveVerifier(ctx, fmt.Sprintf("%s@%s", identity, node), algorithms.ECDSA_SECP256K1, verifiers.ETH_ADDRESS) if err != nil { diff --git a/operator/test/e2e/e2e_noto_pente_test.go b/operator/test/e2e/e2e_noto_pente_test.go index bf4a30c5b..b7623d7ff 100644 --- a/operator/test/e2e/e2e_noto_pente_test.go +++ b/operator/test/e2e/e2e_noto_pente_test.go @@ -91,8 +91,106 @@ func getJSONPropertyAs(jsonData tktypes.RawJSON, name string, toValue any) { } } +var pentePrivGroupComps = abi.ParameterArray{ + {Name: "salt", Type: "bytes32"}, + {Name: "members", Type: "string[]"}, +} + +var penteGroupABI = &abi.Parameter{ + Name: "group", Type: "tuple", Components: pentePrivGroupComps, +} + +var penteConstructorABI = &abi.Entry{ + Type: abi.Constructor, Inputs: abi.ParameterArray{ + penteGroupABI, + {Name: "evmVersion", Type: "string"}, + {Name: "endorsementType", Type: "string"}, + {Name: "externalCallsEnabled", Type: "bool"}, + }, +} + +type penteConstructorParams struct { + Group nototypes.PentePrivateGroup `json:"group"` + EVMVersion string `json:"evmVersion"` + EndorsementType string `json:"endorsementType"` + ExternalCallsEnabled bool `json:"externalCallsEnabled"` +} + +// This works for both ERC20Simple and NotoTrackerERC20 when invoked via Pente +var erc20PrivateABI = abi.ABI{ + { + Type: abi.Function, + Name: "deploy", + Inputs: abi.ParameterArray{ + penteGroupABI, + {Name: "bytecode", Type: "bytes"}, + {Name: "inputs", Type: "tuple", Components: abi.ParameterArray{ + {Name: "name", Type: "string"}, + {Name: "symbol", Type: "string"}, + }}, + }, + }, + { + Type: abi.Function, + Name: "mint", + Inputs: abi.ParameterArray{ + penteGroupABI, + {Name: "to", Type: "address"}, + {Name: "inputs", Type: "tuple", Components: abi.ParameterArray{ + {Name: "to", Type: "address"}, + {Name: "amount", Type: "uint256"}, + }}, + }, + }, + { + Type: abi.Function, + Name: "transfer", + Inputs: abi.ParameterArray{ + penteGroupABI, + {Name: "to", Type: "address"}, + {Name: "inputs", Type: "tuple", Components: abi.ParameterArray{ + {Name: "to", Type: "address"}, + {Name: "value", Type: "uint256"}, + }}, + }, + }, + { + Type: abi.Function, + Name: "balanceOf", + Inputs: abi.ParameterArray{ + penteGroupABI, + {Name: "to", Type: "address"}, + {Name: "inputs", Type: "tuple", Components: abi.ParameterArray{ + {Name: "account", Type: "address"}, + }}, + }, + Outputs: abi.ParameterArray{ + {Type: "uint256"}, + }, + }, +} + +type penteDeployParams struct { + Group nototypes.PentePrivateGroup `json:"group"` + Bytecode tktypes.HexBytes `json:"bytecode"` + Inputs any `json:"inputs"` +} + +type penteInvokeParams struct { + Group nototypes.PentePrivateGroup `json:"group"` + To tktypes.EthAddress `json:"to"` + Inputs any `json:"inputs"` +} + +type penteReceipt struct { + Receipt struct { + ContractAddress *tktypes.EthAddress `json:"contractAddress"` + } `json:"receipt"` +} + var _ = Describe("noto/pente - simple", Ordered, func() { BeforeAll(func() { + // Skip("for now") }) AfterAll(func() { @@ -267,102 +365,6 @@ var _ = Describe("noto/pente - simple", Ordered, func() { testLog("done testing noto in isolation") }) - pentePrivGroupComps := abi.ParameterArray{ - {Name: "salt", Type: "bytes32"}, - {Name: "members", Type: "string[]"}, - } - penteGroupABI := &abi.Parameter{ - Name: "group", Type: "tuple", Components: pentePrivGroupComps, - } - - penteConstructorABI := &abi.Entry{ - Type: abi.Constructor, Inputs: abi.ParameterArray{ - penteGroupABI, - {Name: "evmVersion", Type: "string"}, - {Name: "endorsementType", Type: "string"}, - {Name: "externalCallsEnabled", Type: "bool"}, - }, - } - - type penteConstructorParams struct { - Group nototypes.PentePrivateGroup `json:"group"` - EVMVersion string `json:"evmVersion"` - EndorsementType string `json:"endorsementType"` - ExternalCallsEnabled bool `json:"externalCallsEnabled"` - } - - // This works for both ERC20Simple and NotoTrackerERC20 when invoked via Pente - erc20PrivateABI := abi.ABI{ - { - Type: abi.Function, - Name: "deploy", - Inputs: abi.ParameterArray{ - penteGroupABI, - {Name: "bytecode", Type: "bytes"}, - {Name: "inputs", Type: "tuple", Components: abi.ParameterArray{ - {Name: "name", Type: "string"}, - {Name: "symbol", Type: "string"}, - }}, - }, - }, - { - Type: abi.Function, - Name: "mint", - Inputs: abi.ParameterArray{ - penteGroupABI, - {Name: "to", Type: "address"}, - {Name: "inputs", Type: "tuple", Components: abi.ParameterArray{ - {Name: "to", Type: "address"}, - {Name: "amount", Type: "uint256"}, - }}, - }, - }, - { - Type: abi.Function, - Name: "transfer", - Inputs: abi.ParameterArray{ - penteGroupABI, - {Name: "to", Type: "address"}, - {Name: "inputs", Type: "tuple", Components: abi.ParameterArray{ - {Name: "to", Type: "address"}, - {Name: "value", Type: "uint256"}, - }}, - }, - }, - { - Type: abi.Function, - Name: "balanceOf", - Inputs: abi.ParameterArray{ - penteGroupABI, - {Name: "to", Type: "address"}, - {Name: "inputs", Type: "tuple", Components: abi.ParameterArray{ - {Name: "account", Type: "address"}, - }}, - }, - Outputs: abi.ParameterArray{ - {Type: "uint256"}, - }, - }, - } - - type penteDeployParams struct { - Group nototypes.PentePrivateGroup `json:"group"` - Bytecode tktypes.HexBytes `json:"bytecode"` - Inputs any `json:"inputs"` - } - - type penteInvokeParams struct { - Group nototypes.PentePrivateGroup `json:"group"` - To tktypes.EthAddress `json:"to"` - Inputs any `json:"inputs"` - } - - type penteReceipt struct { - Receipt struct { - ContractAddress *tktypes.EthAddress `json:"contractAddress"` - } `json:"receipt"` - } - penteGroupNodes1and2 := nototypes.PentePrivateGroup{ Salt: tktypes.Bytes32(tktypes.RandBytes(32)), // unique salt must be shared privately to retain anonymity Members: []string{"bob@node1", "sally@node2"}, // these will be salted to establish the endorsement key identifiers diff --git a/operator/test/e2e/e2e_pente_parallel_test.go b/operator/test/e2e/e2e_pente_parallel_test.go new file mode 100644 index 000000000..1222bddcf --- /dev/null +++ b/operator/test/e2e/e2e_pente_parallel_test.go @@ -0,0 +1,353 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "crypto/rand" + "encoding/json" + "fmt" + "math/big" + "time" + + _ "embed" + + "github.com/google/uuid" + "github.com/hyperledger/firefly-signer/pkg/abi" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/kaleido-io/paladin/config/pkg/pldconf" + nototypes "github.com/kaleido-io/paladin/domains/noto/pkg/types" + "github.com/kaleido-io/paladin/toolkit/pkg/algorithms" + "github.com/kaleido-io/paladin/toolkit/pkg/pldclient" + "github.com/kaleido-io/paladin/toolkit/pkg/solutils" + "github.com/kaleido-io/paladin/toolkit/pkg/tktypes" + "github.com/kaleido-io/paladin/toolkit/pkg/verifiers" +) + +var _ = Describe("pente - parallelism on a single contract", Ordered, func() { + BeforeAll(func() { + }) + + AfterAll(func() { + }) + + Context("Pente with an ERC-20 modifying multiple accounts concurrently contending for account state", func() { + + ctx := context.Background() + rpc := map[string]pldclient.PaladinClient{} + + connectNode := func(url, name string) { + Eventually(func() bool { + return withTimeout(func(ctx context.Context) bool { + pld, err := pldclient.New().HTTP(ctx, &pldconf.HTTPClientConfig{URL: url}) + if err == nil { + queriedName, err := pld.Transport().NodeName(ctx) + Expect(err).To(BeNil()) + Expect(queriedName).To(Equal(name)) + rpc[name] = pld + } + return err == nil + }) + }).Should(BeTrue()) + } + + It("waits to connect to all three nodes", func() { + connectNode(node1HttpURL, "node1") + connectNode(node2HttpURL, "node2") + connectNode(node3HttpURL, "node3") + }) + + It("checks nodes can talk to each other", func() { + for src := range rpc { + for dest := range rpc { + Eventually(func() bool { + return withTimeout(func(ctx context.Context) bool { + verifier, err := rpc[src].PTX().ResolveVerifier(ctx, fmt.Sprintf("test@%s", dest), + algorithms.ECDSA_SECP256K1, verifiers.ETH_ADDRESS) + if err == nil { + addr, err := tktypes.ParseEthAddress(verifier) + Expect(err).To(BeNil()) + Expect(addr).ToNot(BeNil()) + } + return err == nil + }) + }).Should(BeTrue()) + } + } + }) + + penteGroupStars := nototypes.PentePrivateGroup{ + Salt: tktypes.Bytes32(tktypes.RandBytes(32)), // unique salt must be shared privately to retain anonymity + Members: []string{"tara@node1", "hoshi@node2", "seren@node3"}, // these will be salted to establish the endorsement key identifiers + } + + var penteContract *tktypes.EthAddress + It("deploys a pente privacy group across all three nodes", func() { + + const ENDORSEMENT_TYPE__GROUP_SCOPED_IDENTITIES = "group_scoped_identities" + + deploy := rpc["node1"].ForABI(ctx, abi.ABI{penteConstructorABI}). + Private(). + Domain("pente"). + Constructor(). + Inputs(&penteConstructorParams{ + Group: penteGroupStars, + EVMVersion: "shanghai", + EndorsementType: ENDORSEMENT_TYPE__GROUP_SCOPED_IDENTITIES, + ExternalCallsEnabled: true, + }). + From("random." + uuid.NewString()). // anyone can submit this by design + Send(). + Wait(5 * time.Second) + Expect(deploy.Error()).To(BeNil()) + Expect(deploy.Receipt().ContractAddress).ToNot(BeNil()) + penteContract = deploy.Receipt().ContractAddress + testLog("Pente privacy group %s (salt=%s) deployed by TX %s", penteContract, penteGroupStars.Salt, deploy.ID()) + }) + + erc20Simple := solutils.MustLoadBuild(ERC20SimpleBuildJSON) + var erc20DeployID uuid.UUID + It("deploys a vanilla ERC-20 into the the privacy group with a minter/owner", func() { + + deploy := rpc["node1"].ForABI(ctx, erc20PrivateABI). + Private(). + Domain("pente"). + To(penteContract). + Function("deploy"). + Inputs(&penteDeployParams{ + Group: penteGroupStars, + Bytecode: erc20Simple.Bytecode, + Inputs: map[string]any{ + "name": "Stars", + "symbol": "STAR", + }, + }). + From("tara@node1"). + Send(). + Wait(5 * time.Second) + testLog("Deployed SimpleERC20 contract into privacy group in transaction %s", deploy.ID()) + Expect(deploy.Error()).To(BeNil()) + erc20DeployID = deploy.ID() + }) + + var erc20StarsAddr *tktypes.EthAddress + It("requests the receipt from pente to get the contract address", func() { + + domainReceiptJSON, err := rpc["node1"].PTX().GetDomainReceipt(ctx, "pente", erc20DeployID) + Expect(err).To(BeNil()) + var pr penteReceipt + err = json.Unmarshal(domainReceiptJSON, &pr) + Expect(err).To(BeNil()) + erc20StarsAddr = pr.Receipt.ContractAddress + testLog("SimpleERC20 contractAddress (within privacy group): %s", erc20StarsAddr) + + }) + + getEthAddress := func(identity, node string) tktypes.EthAddress { + addr, err := rpc[node].PTX().ResolveVerifier(ctx, fmt.Sprintf("%s@%s", identity, node), algorithms.ECDSA_SECP256K1, verifiers.ETH_ADDRESS) + Expect(err).To(BeNil()) + return *tktypes.MustEthAddress(addr) + } + getERC20Balance := func(identity, node string) *tktypes.HexUint256 { + addr := getEthAddress(identity, node) + type ercBalanceOf struct { + Param0 *tktypes.HexUint256 `json:"0"` + } + var result ercBalanceOf + err := rpc[node].ForABI(ctx, erc20PrivateABI). + Private(). + Domain("pente"). + To(penteContract). + Function("balanceOf"). + Inputs(&penteInvokeParams{ + Group: penteGroupStars, + To: *erc20StarsAddr, + Inputs: map[string]any{ + "account": addr.String(), + }, + }). + Outputs(&result). + From(fmt.Sprintf("%s@%s", identity, node)). + Call() + Expect(err).To(BeNil()) + Expect(result).ToNot(BeNil()) + return result.Param0 + } + + users := [][]string{ + {"tara", "node1"}, + {"hoshi", "node2"}, + {"seren", "node3"}, + } + + It("mints some ERC-20 inside the the privacy group", func() { + + for _, user := range users { + + invoke := rpc["node1"].ForABI(ctx, erc20PrivateABI). + Private(). + Domain("pente"). + To(penteContract). + Function("mint"). + Inputs(&penteInvokeParams{ + Group: penteGroupStars, + To: *erc20StarsAddr, + Inputs: map[string]any{ + "to": getEthAddress(user[0], user[1]), + "amount": with18Decimals(1000), + }, + }). + From("tara@node1"). // operator + Send(). + Wait(5 * time.Second) + testLog("SimpleERC20 mint transaction %s", invoke.ID()) + Expect(invoke.Error()).To(BeNil()) + } + + }) + + startingBalance := int64(1000) + It("check ERC-20 balance of each", func() { + + for _, user := range users { + userBalance := getERC20Balance(user[0], user[1]) + testLog("SimpleERC20 balance after mint to %s@%s: %s", user[0], user[1], userBalance.Int()) + Expect(userBalance.String()).To(Equal(with18Decimals(startingBalance).String())) + } + + }) + + It("runs three parallel sets of transfers, with each parallel set being synchronous", func() { + + results := make(chan error) + for _iUser, _user := range users { + iUser := _iUser + user := _user + go func() { + var err error + defer func() { + results <- err + }() + + const count = 10 + toUser := users[(iUser+1)%len(users)] + for i := 0; i < count && err == nil; i++ { + bigAmount, _ := rand.Int(rand.Reader, big.NewInt(9)) + amount := bigAmount.Int64() + 1 + invoke := rpc[user[1]].ForABI(ctx, erc20PrivateABI). + Private(). + Domain("pente"). + To(penteContract). + Function("transfer"). + Inputs(&penteInvokeParams{ + Group: penteGroupStars, + To: *erc20StarsAddr, + Inputs: map[string]any{ + "to": getEthAddress(toUser[0], toUser[1]), + "value": with18Decimals(amount), + }, + }). + From(fmt.Sprintf("%s@%s", user[0], user[1])). + Send(). + // We submit the transactions one-at-a-time within each go-routine in this test + // (but have three concurrent go routines running) + Wait(5 * time.Second) + testLog("[%d]:%.3d/%.3d SimpleERC20 mint %d from %s@%s to %s@%s transaction %s", + iUser, i, count, amount, user[0], user[1], toUser[0], toUser[1], invoke.ID()) + err = invoke.Error() + } + }() + } + // Wait for the three go routines to complete + for i := 0; i < len(users); i++ { + Expect(<-results).To(BeNil()) + } + }) + + It("check ERC-20 balances add up to the correct total", func() { + + totalBalance := new(big.Int) + for _, user := range users { + userBalance := getERC20Balance(user[0], user[1]) + testLog("SimpleERC20 balance %s@%s after transfers: %s", user[0], user[1], userBalance.Int()) + totalBalance = totalBalance.Add(totalBalance, userBalance.Int()) + } + Expect(totalBalance.String()).To(Equal(with18Decimals(startingBalance * int64(len(users))).Int().String())) + + }) + + It("runs three parallel sets of transfers, all submitted as a stream and checked at the end", func() { + + results := make(chan []pldclient.SentTransaction) + for _iUser, _user := range users { + iUser := _iUser + user := _user + go func() { + const count = 10 + transfers := make([]pldclient.SentTransaction, 0, count) + toUser := users[(iUser+1)%len(users)] + for i := 0; i < count; i++ { + bigAmount, _ := rand.Int(rand.Reader, big.NewInt(9)) + amount := bigAmount.Int64() + 1 + invoke := rpc[user[1]].ForABI(ctx, erc20PrivateABI). + Private(). + Domain("pente"). + To(penteContract). + Function("transfer"). + Inputs(&penteInvokeParams{ + Group: penteGroupStars, + To: *erc20StarsAddr, + Inputs: map[string]any{ + "to": getEthAddress(toUser[0], toUser[1]), + "value": with18Decimals(amount), + }, + }). + From(fmt.Sprintf("%s@%s", user[0], user[1])). + Send() + testLog("[%d]:%.3d/%.3d SimpleERC20 mint %d from %s@%s to %s@%s transaction %s", + iUser, i, count, amount, user[0], user[1], toUser[0], toUser[1], invoke.ID()) + transfers = append(transfers, invoke) + } + results <- transfers + }() + } + // Wait for the three go routines to complete + for i := 0; i < len(users); i++ { + transfers := <-results + for _, transfer := range transfers { + testLog("SimpleERC20 wait for completion of transfer %s", transfer.ID()) + Expect(transfer.Wait(10 * time.Second).Error()).To(BeNil()) + } + } + }) + + It("check ERC-20 balances add up to the correct total", func() { + + totalBalance := new(big.Int) + for _, user := range users { + userBalance := getERC20Balance(user[0], user[1]) + testLog("SimpleERC20 balance %s@%s after transfers: %s", user[0], user[1], userBalance.Int()) + totalBalance = totalBalance.Add(totalBalance, userBalance.Int()) + } + Expect(totalBalance.String()).To(Equal(with18Decimals(startingBalance * int64(len(users))).Int().String())) + + }) + + }) +}) diff --git a/operator/test/e2e/e2e_suite_test.go b/operator/test/e2e/e2e_suite_test.go index a1c7192c0..9ace952f4 100644 --- a/operator/test/e2e/e2e_suite_test.go +++ b/operator/test/e2e/e2e_suite_test.go @@ -32,6 +32,6 @@ func TestE2E(t *testing.T) { logrus.SetLevel(logrus.WarnLevel) RegisterFailHandler(Fail) - fmt.Fprintf(GinkgoWriter, "Starting operator-go suite\n") + _, _ = fmt.Fprintf(GinkgoWriter, "Starting operator-go suite\n") RunSpecs(t, "e2e suite") } diff --git a/operator/test/e2e/e2e_zeto_test.go b/operator/test/e2e/e2e_zeto_test.go index 26edae157..1ecb5f316 100644 --- a/operator/test/e2e/e2e_zeto_test.go +++ b/operator/test/e2e/e2e_zeto_test.go @@ -45,6 +45,7 @@ const isNullifier = false var _ = Describe(fmt.Sprintf("zeto - %s", tokenType), Ordered, func() { BeforeAll(func() { + // Skip("for now") }) AfterAll(func() { diff --git a/testinfra/docker-compose-test.yml b/testinfra/docker-compose-test.yml index 213f995b8..999e40c6a 100755 --- a/testinfra/docker-compose-test.yml +++ b/testinfra/docker-compose-test.yml @@ -14,7 +14,7 @@ services: command: - /var/besu besu: - image: hyperledger/besu:latest + image: hyperledger/besu:24.10.0 user: 0:0 volumes: - besu_data:/var/besu:rw diff --git a/toolkit/go/pkg/httpserver/debugserver.go b/toolkit/go/pkg/httpserver/debugserver.go new file mode 100644 index 000000000..e50644261 --- /dev/null +++ b/toolkit/go/pkg/httpserver/debugserver.go @@ -0,0 +1,55 @@ +// Copyright © 2023 Kaleido, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package httpserver + +import ( + "context" + "net/http/pprof" + + "github.com/gorilla/mux" + "github.com/kaleido-io/paladin/config/pkg/pldconf" + "github.com/kaleido-io/paladin/toolkit/pkg/log" +) + +type DebugServer interface { + Server + Router() *mux.Router +} + +type debugServer struct { + Server + r *mux.Router +} + +func (ds *debugServer) Router() *mux.Router { + return ds.r +} + +func NewDebugServer(ctx context.Context, debugServerConf *pldconf.HTTPServerConfig) (_ DebugServer, err error) { + r := mux.NewRouter() + r.PathPrefix("/debug/pprof/cmdline").HandlerFunc(pprof.Cmdline) + r.PathPrefix("/debug/pprof/profile").HandlerFunc(pprof.Profile) + r.PathPrefix("/debug/pprof/symbol").HandlerFunc(pprof.Symbol) + r.PathPrefix("/debug/pprof/trace").HandlerFunc(pprof.Trace) + r.PathPrefix("/debug/pprof/").HandlerFunc(pprof.Index) + server, err := NewServer(ctx, "debug", debugServerConf, r) + if err != nil { + return nil, err + } + log.L(ctx).Infof("Debug server running on %s", server.Addr()) + return &debugServer{Server: server, r: r}, nil +} diff --git a/toolkit/go/pkg/httpserver/debugserver_test.go b/toolkit/go/pkg/httpserver/debugserver_test.go new file mode 100644 index 000000000..b3cfd95ae --- /dev/null +++ b/toolkit/go/pkg/httpserver/debugserver_test.go @@ -0,0 +1,66 @@ +// Copyright © 2022 Kaleido, Inc. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package httpserver + +import ( + "context" + "fmt" + "io" + "net/http" + "testing" + + "github.com/kaleido-io/paladin/config/pkg/confutil" + "github.com/kaleido-io/paladin/config/pkg/pldconf" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func newTestDebugServer(t *testing.T, conf *pldconf.HTTPServerConfig) (string, *debugServer, func()) { + conf.Address = confutil.P("127.0.0.1") + conf.Port = confutil.P(0) + s, err := NewDebugServer(context.Background(), conf) + require.NoError(t, err) + ds := s.(*debugServer) + err = s.Start() + require.NoError(t, err) + + return fmt.Sprintf("http://%s", s.Addr()), ds, s.Stop +} + +func TestDebugServerStackTrace(t *testing.T) { + + url, ds, done := newTestDebugServer(t, &pldconf.HTTPServerConfig{}) + defer done() + + resp, err := http.Get(fmt.Sprintf("%s/debug/pprof/goroutine?debug=2", url)) + require.NoError(t, err) + + b, err := io.ReadAll(resp.Body) + require.NoError(t, err) + + assert.Regexp(t, "debugserver_test.go", string(b)) + + require.NotNil(t, ds.Router()) + +} + +func TestDebugServerFail(t *testing.T) { + + _, err := NewDebugServer(context.Background(), &pldconf.HTTPServerConfig{}) + assert.Regexp(t, "PD020601", err) + +} diff --git a/toolkit/go/pkg/log/log.go b/toolkit/go/pkg/log/log.go index acf1a78fb..bf9431932 100644 --- a/toolkit/go/pkg/log/log.go +++ b/toolkit/go/pkg/log/log.go @@ -53,10 +53,12 @@ func InitConfig(conf *pldconf.LogConfig) { output := confutil.StringNotEmpty(conf.Output, *pldconf.LogDefaults.Output) switch output { case "file": + filename := confutil.StringNotEmpty(conf.File.Filename, *pldconf.LogDefaults.File.Filename) + rootLogger.Infof("Logs diverted to %s", filename) maxSizeBytes := confutil.ByteSize(conf.File.MaxSize, 0, *pldconf.LogDefaults.File.MaxSize) maxAgeDuration := confutil.DurationMin(conf.File.MaxAge, 0, *pldconf.LogDefaults.File.MaxAge) lumberjack := &lumberjack.Logger{ - Filename: confutil.StringNotEmpty(conf.File.Filename, *pldconf.LogDefaults.File.Filename), + Filename: filename, MaxSize: int(math.Ceil(float64(maxSizeBytes) / 1024 / 1024)), /* round up in megabytes */ MaxBackups: confutil.IntMin(conf.File.MaxBackups, 0, *pldconf.LogDefaults.File.MaxBackups), MaxAge: int(math.Ceil(float64(maxAgeDuration) / float64(time.Hour) / 24)), /* round up in days */ diff --git a/toolkit/go/pkg/pldapi/public_tx.go b/toolkit/go/pkg/pldapi/public_tx.go index 4946b2ef6..25629ac99 100644 --- a/toolkit/go/pkg/pldapi/public_tx.go +++ b/toolkit/go/pkg/pldapi/public_tx.go @@ -60,10 +60,11 @@ type PublicTxSubmissionData struct { } type PublicTx struct { + LocalID *uint64 `docstruct:"PublicTx" json:"localId,omitempty"` // only a local DB identifier for the public transaction. Not directly related to nonce order To *tktypes.EthAddress `docstruct:"PublicTx" json:"to,omitempty"` Data tktypes.HexBytes `docstruct:"PublicTx" json:"data,omitempty"` From tktypes.EthAddress `docstruct:"PublicTx" json:"from"` - Nonce tktypes.HexUint64 `docstruct:"PublicTx" json:"nonce"` + Nonce *tktypes.HexUint64 `docstruct:"PublicTx" json:"nonce"` Created tktypes.Timestamp `docstruct:"PublicTx" json:"created"` CompletedAt *tktypes.Timestamp `docstruct:"PublicTx" json:"completedAt,omitempty"` // only once confirmed TransactionHash *tktypes.Bytes32 `docstruct:"PublicTx" json:"transactionHash"` // only once confirmed diff --git a/toolkit/go/pkg/pldapi/transaction.go b/toolkit/go/pkg/pldapi/transaction.go index 3c953dfad..cbc567598 100644 --- a/toolkit/go/pkg/pldapi/transaction.go +++ b/toolkit/go/pkg/pldapi/transaction.go @@ -80,7 +80,7 @@ type TransactionBase struct { // TODO: PublicTransactions string list } -// The full transaction you query, with input an doutput +// The full transaction you query, with input and output type Transaction struct { ID *uuid.UUID `docstruct:"Transaction" json:"id,omitempty"` // server generated UUID for this transaction (query only) Created tktypes.Timestamp `docstruct:"Transaction" json:"created,omitempty"` // server generated creation timestamp for this transaction (query only) diff --git a/toolkit/go/pkg/pldclient/txbuilder.go b/toolkit/go/pkg/pldclient/txbuilder.go index 9f97dc7cd..dfecf1236 100644 --- a/toolkit/go/pkg/pldclient/txbuilder.go +++ b/toolkit/go/pkg/pldclient/txbuilder.go @@ -736,7 +736,7 @@ func txPoller[R any](ch *chainable, timeout time.Duration, txID uuid.UUID, doGet // Check we didn't timeout waitTime := time.Since(startTime) if waitTime > timeout { - ch.deferError(i18n.WrapError(ch.ctx, lastErr, tkmsgs.MsgPaladinClientPollTimedOut, attempt, waitTime)) + ch.deferError(i18n.WrapError(ch.ctx, lastErr, tkmsgs.MsgPaladinClientPollTxTimedOut, attempt, waitTime, txID)) return result } // Wait before polling diff --git a/toolkit/go/pkg/tkmsgs/en_descriptions.go b/toolkit/go/pkg/tkmsgs/en_descriptions.go index 0212e4ba6..88ae59b70 100644 --- a/toolkit/go/pkg/tkmsgs/en_descriptions.go +++ b/toolkit/go/pkg/tkmsgs/en_descriptions.go @@ -84,6 +84,7 @@ var ( PublicTxSubmissionNonce = ffm("PublicTxSubmission.nonce", "The transaction nonce") PublicTxSubmissionDataTime = ffm("PublicTxSubmissionData.time", "The submission time") PublicTxSubmissionDataTransactionHash = ffm("PublicTxSubmissionData.transactionHash", "The transaction hash") + PublicTxLocalID = ffm("PublicTx.localId", "A locally generated numeric ID for the public transaction. Unique within the node") PublicTxTo = ffm("PublicTx.to", "The target contract address (optional)") PublicTxData = ffm("PublicTx.data", "The pre-encoded calldata (optional)") PublicTxFrom = ffm("PublicTx.from", "The sender's Ethereum address") diff --git a/toolkit/go/pkg/tkmsgs/en_errors.go b/toolkit/go/pkg/tkmsgs/en_errors.go index 4314ab9bc..5cbb3baf7 100644 --- a/toolkit/go/pkg/tkmsgs/en_errors.go +++ b/toolkit/go/pkg/tkmsgs/en_errors.go @@ -72,7 +72,7 @@ var ( MsgPaladinClientNoABISupplied = ffe("PD020213", "No ABI supplied") MsgPaladinClientNoDomain = ffe("PD020214", "No domain specified for private transaction") MsgPaladinClientNoFunction = ffe("PD020215", "No function specified") - MsgPaladinClientPollTimedOut = ffe("PD020216", "Polling timed out after %d attempts in %s") + MsgPaladinClientPollTxTimedOut = ffe("PD020216", "Polling timed out after %d attempts in %s for transaction %s") // Plugin PD0203XX MsgPluginUnsupportedRequest = ffe("PD020300", "Unsupported request %T") diff --git a/toolkit/java/src/main/java/io/kaleido/paladin/toolkit/PluginInstance.java b/toolkit/java/src/main/java/io/kaleido/paladin/toolkit/PluginInstance.java index 50da8e558..90fcb6734 100644 --- a/toolkit/java/src/main/java/io/kaleido/paladin/toolkit/PluginInstance.java +++ b/toolkit/java/src/main/java/io/kaleido/paladin/toolkit/PluginInstance.java @@ -128,7 +128,7 @@ final Service.Header getReplyHeader(MSG req) { build(); } - final CompletableFuture requestReply(MSG requestMessage) { + final synchronized CompletableFuture requestReply(MSG requestMessage) { CompletableFuture inflight = inflightRequests.addRequest(UUID.fromString(getHeader(requestMessage).getMessageId())); sendStream.onNext(requestMessage); @@ -163,12 +163,12 @@ private UUID getCorrelationUUID(Service.Header header) { } } - private Void send(MSG msg) { + private synchronized Void send(MSG msg) { sendStream.onNext(msg); return null; } - private Void sendErrorReply(Service.Header reqHeader, Throwable t) { + private synchronized Void sendErrorReply(Service.Header reqHeader, Throwable t) { Service.Header resHeader = Service.Header.newBuilder(). setPluginId(pluginId). setMessageId(UUID.randomUUID().toString()). diff --git a/toolkit/proto/protos/service.proto b/toolkit/proto/protos/service.proto index f8beaaab0..28909604c 100644 --- a/toolkit/proto/protos/service.proto +++ b/toolkit/proto/protos/service.proto @@ -182,6 +182,10 @@ message PluginLoad { LibType lib_type = 2; // The binary type of the plugin string lib_location = 3; // The location of the plugin (such as a Java Jar file or C library load spec) optional string class = 4; // For JAR type we need to specify a class inside the Jar as well + enum SysCommand { + THREAD_DUMP = 0; + } + optional SysCommand sys_command = 5; } service PluginController {