Merge branch 'rampup' into disable-zkapp-changes-implementation

.gitmodules

@@ -14,6 +14,3 @@
 [submodule "src/external/c-reference-signer"]
 	path = src/external/c-reference-signer
 	url = https://github.com/MinaProtocol/c-reference-signer.git
-[submodule "src/lib/snarky_js_bindings/snarkyjs"]
-	path = src/lib/snarkyjs
-	url = https://github.com/o1-labs/snarkyjs.git

CODEOWNERS

@@ -51,7 +51,6 @@
 /src/lib/snarky_curves/ @MinaProtocol/crypto-eng-reviewers
 /src/lib/snarky_field_extensions/ @MinaProtocol/crypto-eng-reviewers
 /src/lib/snarky_group_map/ @MinaProtocol/crypto-eng-reviewers
-/src/lib/snarkyjs @MinaProtocol/product-eng-reviewers
 /src/lib/snarky_log/ @MinaProtocol/crypto-eng-reviewers
 /src/lib/unsigned_extended/ @MinaProtocol/crypto-eng-reviewers
 

Makefile

@@ -115,33 +115,6 @@ build_intgtest: ocaml_checks
 	dune build --profile=$(DUNE_PROFILE) src/app/test_executive/test_executive.exe src/app/logproc/logproc.exe
 	$(info Build complete)
 
-client_sdk: ocaml_checks
-	$(info Starting Build)
-	ulimit -s 65532 && (ulimit -n 10240 || true) && dune build src/app/client_sdk/client_sdk.bc.js
-	$(info Build complete)
-
-client_sdk_test_sigs: ocaml_checks
-	$(info Starting Build)
-	ulimit -s 65532 && (ulimit -n 10240 || true) && dune build src/app/client_sdk/tests/test_signatures.exe --profile=mainnet
-	$(info Build complete)
-
-client_sdk_test_sigs_nonconsensus: ocaml_checks
-	$(info Starting Build)
-	ulimit -s 65532 && (ulimit -n 10240 || true) && dune build src/app/client_sdk/tests/test_signatures_nonconsensus.exe --profile=nonconsensus_mainnet
-	$(info Build complete)
-
-snarkyjs: ocaml_checks
-	$(info Starting Build)
-	(ulimit -s 65532) && (ulimit -n 10240 || true) \
-	&& bash ./src/lib/snarkyjs/src/bindings/scripts/build-snarkyjs-node.sh
-	$(info Build complete)
-
-snarkyjs_no_types: ocaml_checks
-	$(info Starting Build)
-	((ulimit -s 65532) || true) && (ulimit -n 10240 || true) \
-	&& bash ./src/lib/snarkyjs/src/bindings/scripts/build-snarkyjs-node-artifacts.sh
-	$(info Build complete)
-
 rosetta_lib_encodings: ocaml_checks
 	$(info Starting Build)
 	ulimit -s 65532 && (ulimit -n 10240 || true) && dune build src/lib/rosetta_lib/test/test_encodings.exe --profile=mainnet

buildkite/scripts/rosetta-integration-tests-fast.sh

@@ -0,0 +1 @@
+buildkite/scripts/rosetta-integration-tests.sh --mode=minimal

buildkite/scripts/rosetta-integration-tests-full.sh

@@ -0,0 +1 @@
+buildkite/scripts/rosetta-integration-tests.sh --mode=full

buildkite/scripts/rosetta-integration-tests.sh

@@ -1,21 +1,22 @@
 #!/bin/bash
-
-# Deploy a sandboxed Mina daemon with an archive and a Rosetta instance.
-# Deploy 2 zkApps from https://github.com/MinaProtocol/rosetta-integration-test-zkapps
-# to the network, interact with them and add some regular transactions.
-# Then run full rosetta-cli tests against the Rosetta instance.
-
-# NPM and NodeJS are installed through NVM, versions are stored in environment
-# variables below. Zkapp-cli is installed globally through NPM, however, to
-# ensure compatibility with the daemon, we use o1js pinned in the Mina repo.
-# The repo is mounted into the container at /workdir, so we can build o1js from
-# that source. It is important to make sure that the zkapp-cli version installed
-# is compatible with o1js version used.
+set -eox pipefail
 
 # These tests use the mina-dev binary, as rosetta-cli assumes we use a testnet.
 # See https://github.com/coinbase/rosetta-sdk-go/blob/master/keys/signer_pallas.go#L222
 
-set -eo pipefail
+# Defines scope of test. Currently supported are:
+# - minimal -> only quick checks (~5 mins)
+# - full -> all checks
+MODE="minimal"
+
+while [ $# -gt 0 ]; do
+  case "$1" in
+  --mode=*)
+    MODE="${1#*=}"
+    ;;
+  esac
+  shift
+done
 
 export MINA_NETWORK=${MINA_NETWORK:=sandbox}
 export LOG_LEVEL="${LOG_LEVEL:=Info}"
@@ -47,43 +48,6 @@ export MINA_CONFIG_FILE=$HOME/${MINA_NETWORK}.json
 export MINA_CONFIG_DIR="${MINA_CONFIG_DIR:=$HOME/.mina-config}"
 export MINA_GRAPHQL_PORT=${MINA_GRAPHQL_PORT:=3085}
 
-# Test variables
-export ROSETTA_INT_TEST_ZKAPPS_VERSION=${ROSETTA_INT_TEST_ZKAPPS_VERSION:=rosetta-ci-tests}
-
-# We need a version which is compatible with o1js pinned to the Mina repo.
-# Should be set to 'latest' most of the time, but occasionally we might need
-# an older one.
-export ZKAPP_CLI_VERSION=0.11.0
-
-# Nodejs variables
-export NVM_VERSION=0.39.3
-export NODE_VERSION=20.6.1
-
-# zkApps variables
-export ZKAPP_PATH=$HOME/zkapps
-
-echo "=========================== INSTALLING NPM ==========================="
-curl -so- https://raw.githubusercontent.com/nvm-sh/nvm/v${NVM_VERSION}/install.sh | bash &>/dev/null
-export NVM_DIR="$HOME/.nvm"
-[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"                   # This loads nvm
-[ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion" # This loads nvm bash_completion
-
-nvm install $NODE_VERSION
-nvm use --delete-prefix $NODE_VERSION
-
-mkdir ~/.npm-global
-npm config set prefix '~/.npm-global'
-export PATH=~/.npm-global/bin:$PATH
-
-# Install zkapp-cli and Typescript compiler.
-npm install --no-progress --global "zkapp-cli@${ZKAPP_CLI_VERSION}" "typescript@latest"
-
-# Build o1js so that we can use it later.
-pushd /workdir/src/lib/snarkyjs
-npm ci
-npm run build
-popd
-
 # Rosetta CLI variables
 # Files from ROSETTA_CLI_CONFIG_FILES will be read from
 # ROSETTA_CONFIGURATION_INPUT_DIR and some placeholders will be
@@ -95,10 +59,6 @@ ROSETTA_CLI_MAIN_CONFIG_FILE=${ROSETTA_CLI_MAIN_CONFIG_FILE:="config.json"}
 # Frequency (in seconds) at which payment operations will be sent
 TRANSACTION_FREQUENCY=60
 
-# Fetch zkApps
-curl -Ls https://github.com/MinaProtocol/rosetta-integration-test-zkapps/tarball/$ROSETTA_INT_TEST_ZKAPPS_VERSION | tar xz -C /tmp
-mv /tmp/MinaProtocol-rosetta-integration-test-zkapps-* $ZKAPP_PATH
-
 # Libp2p Keypair
 echo "=========================== GENERATING KEYPAIR IN ${MINA_LIBP2P_KEYPAIR_PATH} ==========================="
 mina-dev libp2p generate-keypair -privkey-path $MINA_LIBP2P_KEYPAIR_PATH
@@ -108,9 +68,18 @@ echo "=========================== GENERATING GENESIS LEDGER FOR ${MINA_NETWORK}
 mkdir -p $MINA_KEYS_PATH
 mina-dev advanced generate-keypair --privkey-path $MINA_KEYS_PATH/block-producer.key
 mina-dev advanced generate-keypair --privkey-path $MINA_KEYS_PATH/snark-producer.key
+mina-dev advanced generate-keypair --privkey-path $MINA_KEYS_PATH/zkapp-fee-payer.key
+mina-dev advanced generate-keypair --privkey-path $MINA_KEYS_PATH/zkapp-sender.key
+mina-dev advanced generate-keypair --privkey-path $MINA_KEYS_PATH/zkapp-account.key
 chmod -R 0700 $MINA_KEYS_PATH
 BLOCK_PRODUCER_PK=$(cat $MINA_KEYS_PATH/block-producer.key.pub)
 SNARK_PRODUCER_PK=$(cat $MINA_KEYS_PATH/snark-producer.key.pub)
+ZKAPP_FEE_PAYER_KEY=$MINA_KEYS_PATH/zkapp-fee-payer.key
+ZKAPP_FEE_PAYER_PUB_KEY=$(cat ${ZKAPP_FEE_PAYER_KEY}.pub)
+ZKAPP_SENDER_KEY=$MINA_KEYS_PATH/zkapp-sender.key
+ZKAPP_SENDER_PUB_KEY=$(cat ${ZKAPP_SENDER_KEY}.pub)
+ZKAPP_ACCOUNT_KEY=$MINA_KEYS_PATH/zkapp-account.key
+ZKAPP_ACCOUNT_PUB_KEY=$(cat ${ZKAPP_ACCOUNT_KEY}.pub)
 
 mkdir -p $MINA_CONFIG_DIR/wallets/store
 cp $MINA_KEYS_PATH/block-producer.key $MINA_CONFIG_DIR/wallets/store/$BLOCK_PRODUCER_PK
@@ -122,25 +91,15 @@ cat <<EOF >"$MINA_CONFIG_FILE"
   "ledger": {
     "name": "${MINA_NETWORK}",
     "accounts": [
-      { "pk": "${BLOCK_PRODUCER_PK}", "balance": "1000", "delegate": null, "sk": null },
-      { "pk": "${SNARK_PRODUCER_PK}", "balance": "2000", "delegate": "${BLOCK_PRODUCER_PK}", "sk": null }
+      { "pk": "${BLOCK_PRODUCER_PK}", "balance": "101550000.000000000", "delegate": null, "sk": null },
+      { "pk": "${SNARK_PRODUCER_PK}", "balance": "605500.000000000", "delegate": "${BLOCK_PRODUCER_PK}", "sk": null },
+      { "pk": "${ZKAPP_FEE_PAYER_PUB_KEY}", "balance": "10055.000000000", "delegate": null, "sk": null },
+      { "pk": "${ZKAPP_SENDER_PUB_KEY}", "balance": "10055.000000000", "delegate": null, "sk": null },
+      { "pk": "${ZKAPP_ACCOUNT_PUB_KEY}", "balance": "10055.000000000", "delegate": null, "sk": null }
     ]
   }
 }
 EOF
-for zkapp_path in ${ZKAPP_PATH}/*/; do
-  zkapp_path=${zkapp_path%/}
-  zkapp=$(basename $zkapp_path)
-  # Generate zkApp account keypair
-  mina-dev advanced generate-keypair --privkey-path ${MINA_KEYS_PATH}/zkapp-${zkapp}-account.key
-  # Generate zkApp fee payer keypair
-  mina-dev advanced generate-keypair --privkey-path ${MINA_KEYS_PATH}/zkapp-${zkapp}-fee-payer.key
-  zkapp_fee_payer_pk=$(cat $MINA_KEYS_PATH/zkapp-${zkapp}-fee-payer.key.pub)
-  line="[{ \"pk\": \"${zkapp_fee_payer_pk}\", \"balance\": \"10000\", \"delegate\": null, \"sk\": null }]"
-  jq ".ledger.accounts |= . + ${line}" $MINA_CONFIG_FILE >${MINA_CONFIG_FILE}.tmp
-  mv ${MINA_CONFIG_FILE}.tmp $MINA_CONFIG_FILE
-done
-cat $MINA_CONFIG_FILE | jq .
 
 # Substitute placeholders in rosetta-cli configuration
 ROSETTA_CONFIGURATION_OUTPUT_DIR=/tmp/rosetta-cli-config
@@ -160,11 +119,6 @@ done
 echo "==================== IMPORTING GENESIS ACCOUNTS ======================"
 mina-dev accounts import --privkey-path $MINA_KEYS_PATH/block-producer.key --config-directory $MINA_CONFIG_DIR
 mina-dev accounts import --privkey-path $MINA_KEYS_PATH/snark-producer.key --config-directory $MINA_CONFIG_DIR
-for zkapp_path in ${ZKAPP_PATH}/*/; do
-  zkapp_path=${zkapp_path%/}
-  zkapp=$(basename $zkapp_path)
-  mina-dev accounts import --privkey-path $MINA_KEYS_PATH/zkapp-${zkapp}-fee-payer.key --config-directory $MINA_CONFIG_DIR
-done
 
 # Postgres
 echo "========================= INITIALIZING POSTGRESQL ==========================="
@@ -222,100 +176,56 @@ until [ $daemon_status == "Synced" ]; do
   echo "Daemon Status: ${daemon_status}"
 done
 
+send_zkapp_txn() {
+  local GRAPHQL_REQUEST="$1"
+  local ESCAPED_GRAPHQL_REQUEST="${GRAPHQL_REQUEST//\"/\\\"}"
+  local ENDPOINT="http://127.0.0.1:${MINA_GRAPHQL_PORT}/graphql"
+
+  curl -X POST \
+    -H "Content-Type: application/json" \
+    --data "{\"query\":\"$ESCAPED_GRAPHQL_REQUEST\"}" \
+    "$ENDPOINT"
+}
+
+echo "========================= ZKAPP ACCOUNT SETTING UP ==========================="
+ZKAPP_TXN_QUERY=$(zkapp_test_transaction create-zkapp-account --fee-payer-key ${ZKAPP_FEE_PAYER_KEY} --nonce 0 --sender-key ${ZKAPP_SENDER_KEY} --sender-nonce 0 --receiver-amount 1000 --zkapp-account-key ${ZKAPP_ACCOUNT_KEY} --fee 5 | sed 1,7d)
+send_zkapp_txn "${ZKAPP_TXN_QUERY}"
+
 # Unlock Genesis Accounts
 echo "==================== UNLOCKING GENESIS ACCOUNTS ======================"
 mina-dev accounts unlock --public-key $BLOCK_PRODUCER_PK
 mina-dev accounts unlock --public-key $SNARK_PRODUCER_PK
 
-# Start sending payments
-send_payments() {
+# Start sending value transfer transactions
+send_value_transfer_txns() {
   mina-dev client send-payment -rest-server http://127.0.0.1:${MINA_GRAPHQL_PORT}/graphql -amount 1 -nonce 0 -receiver $BLOCK_PRODUCER_PK -sender $BLOCK_PRODUCER_PK
   while true; do
     sleep $TRANSACTION_FREQUENCY
     mina-dev client send-payment -rest-server http://127.0.0.1:${MINA_GRAPHQL_PORT}/graphql -amount 1 -receiver $BLOCK_PRODUCER_PK -sender $BLOCK_PRODUCER_PK
   done
 }
-send_payments &
-
-# Fee payer cache creation
-echo "==================== PREPARE FEE PAYER CACHE ======================"
-zkapp_fee_payer_pk=$(cat ${MINA_KEYS_PATH}/zkapp-${zkapp}-fee-payer.key.pub)
-zkapp_fee_payer_privkey=$(mina-dev advanced dump-keypair --privkey-path "${MINA_KEYS_PATH}/zkapp-${zkapp}-fee-payer.key" | sed -ne "s/Private key: //p")
-
-mkdir -p /root/.cache/zkapp-cli/keys
-echo -e "{\n    \"privateKey\": \"${zkapp_fee_payer_privkey}\",\n    \"publicKey\": \"${zkapp_fee_payer_pk}\"\n}" >/root/.cache/zkapp-cli/keys/sandbox.json
-
-# Deploy zkApps
-echo "==================== DEPLOYING ZKAPPS ======================"
-echo "If this fails, it's likely due to incompatibility between the
-o1js and zkapp-cli versions in use."
-echo "NOTE: At the moment the daemon still has an old version of
-      snarkyjs pinned to it, so we cannot use the latest version of
-      zkapp-cli, which requires o1js. Once the pinned snarkyjs version
-      gets updated, this build will most likely fail and we will then
-      need to update the zkapp-cli version used here. This is
-      unfortunate, but necessary. Please delete this warning once it's
-      done." > /dev/stderr
-
-deploy_txs=()
-for zkapp_path in ${ZKAPP_PATH}/*/; do
-  zkapp_path=${zkapp_path%/}
-  zkapp=$(basename $zkapp_path)
-  echo "Deploying ${zkapp}..."
-
-  zkapp_account_pk=$(cat ${MINA_KEYS_PATH}/zkapp-${zkapp}-account.key.pub)
-  zkapp_account_privkey=$(mina-dev advanced dump-keypair --privkey-path "${MINA_KEYS_PATH}/zkapp-${zkapp}-account.key" | sed -ne "s/Private key: //p")
-
-  mkdir -p ${zkapp_path}/keys
-  echo -e "{\n    \"privateKey\": \"${zkapp_account_privkey}\",\n    \"publicKey\": \"${zkapp_account_pk}\"\n}" >"${zkapp_path}/keys/sandbox.json"
-
-  cat <<EOF >"${zkapp_path}/config.json"
-{
-  "version": 1,
-  "networks": {
-    "sandbox": {
-      "url": "http://127.0.0.1:${MINA_GRAPHQL_PORT}/graphql",
-      "keyPath": "keys/sandbox.json",
-      "feepayerKeyPath": "/root/.cache/zkapp-cli/keys/sandbox.json",
-      "feepayerAlias": "sandbox",
-      "fee": "1"
-    }
-  }
-}
-EOF
-  cd "$zkapp_path"
-  npm ci
-  npm run build
-  txn=$(zk deploy sandbox -y | sed -ne "s/https:\/\/berkeley.minaexplorer.com\/transaction\///p")
-  deploy_txs+=txn
-  cd -
-  echo "Done."
-done
-
-# TODO: wait until all zkApps deploy txns are included in a block
+send_value_transfer_txns &
 
-next_block_time=$(mina-dev client status --json | jq '.next_block_production.timing[1].time' | tr -d '"')
-curr_time=$(date +%s%N | cut -b1-13)
-sleep_time=$((($next_block_time - $curr_time) / 1000))
-echo "Sleeping for ${sleep_time}s until next block is created..."
-sleep ${sleep_time}
-
-# Start calling zkApp methods
-echo "==================== INTERACTING WITH ZKAPPS ======================"
-RECEIVER_PK=$BLOCK_PRODUCER_PK
-for zkapp_path in ${ZKAPP_PATH}/*/; do
-  zkapp_path=${zkapp_path%/}
-  zkapp=$(basename $zkapp_path)
-  echo "Interacting with ${zkapp}..."
-
-  cd "$zkapp_path"
-  ./interact.sh sandbox
-  cd -
-  echo "Done."
-done
+# Start sending zkapp transactions
+ZKAPP_FEE_PAYER_NONCE=1
+ZKAPP_SENDER_NONCE=1
+ZKAPP_STATE=0
+send_zkapp_transactions() {
+  while true; do
+    ZKAPP_TXN_QUERY=$(zkapp_test_transaction transfer-funds-one-receiver --fee-payer-key ${ZKAPP_FEE_PAYER_KEY} --nonce ${ZKAPP_FEE_PAYER_NONCE} --sender-key ${ZKAPP_SENDER_KEY} --sender-nonce ${ZKAPP_SENDER_NONCE} --receiver-amount 1 --fee 5 --receiver ${ZKAPP_ACCOUNT_PUB_KEY} | sed 1,5d)
+    send_zkapp_txn "${ZKAPP_TXN_QUERY}"
+    let ZKAPP_FEE_PAYER_NONCE++
+    let ZKAPP_SENDER_NONCE++
+
+    ZKAPP_TXN_QUERY=$(zkapp_test_transaction update-state --fee-payer-key ${ZKAPP_FEE_PAYER_KEY} --nonce ${ZKAPP_FEE_PAYER_NONCE} --zkapp-account-key ${ZKAPP_SENDER_KEY} --zkapp-state ${ZKAPP_STATE} --fee 5 | sed 1,5d)
+    send_zkapp_txn "${ZKAPP_TXN_QUERY}"
+    let ZKAPP_FEE_PAYER_NONCE++
+    let ZKAPP_STATE++
+  done
+}
+send_zkapp_transactions &
 
-next_block_time=$(mina-dev client status --json | jq '.next_block_production.timing[1].time' | tr -d '"')
-curr_time=$(date +%s%N | cut -b1-13)
+next_block_time=$(mina-dev client status --json | jq '.next_block_production.timing[1].time' | tr -d '"') curr_time=$(date +%s%N | cut -b1-13)
 sleep_time=$((($next_block_time - $curr_time) / 1000))
 echo "Sleeping for ${sleep_time}s until next block is created..."
 sleep ${sleep_time}
@@ -327,11 +237,15 @@ rosetta-cli configuration:validate ${ROSETTA_CONFIGURATION_FILE}
 echo "========================= ROSETTA CLI: CHECK:SPEC ==========================="
 rosetta-cli check:spec --all --configuration-file ${ROSETTA_CONFIGURATION_FILE}
 
-echo "========================= ROSETTA CLI: CHECK:CONSTRUCTION ==========================="
-rosetta-cli check:construction --configuration-file ${ROSETTA_CONFIGURATION_FILE}
+if [[ $MODE == "full" ]]; then
+
+  echo "========================= ROSETTA CLI: CHECK:CONSTRUCTION ==========================="
+  rosetta-cli check:construction --configuration-file ${ROSETTA_CONFIGURATION_FILE}
+
+  echo "========================= ROSETTA CLI: CHECK:DATA ==========================="
+  rosetta-cli check:data --configuration-file ${ROSETTA_CONFIGURATION_FILE}
 
-echo "========================= ROSETTA CLI: CHECK:DATA ==========================="
-rosetta-cli check:data --configuration-file ${ROSETTA_CONFIGURATION_FILE}
+  echo "========================= ROSETTA CLI: CHECK:PERF ==========================="
+  echo "rosetta-cli check:perf" # Will run this command when tests are fully implemented
 
-echo "========================= ROSETTA CLI: CHECK:PERF ==========================="
-echo "rosetta-cli check:perf" # Will run this command when tests are fully implemented
+fi