Merge branch 'rampup' into fix/keep-helper-stderr

buildkite/scripts/export-git-env-vars.sh

@@ -29,13 +29,17 @@ export MINA_DEB_CODENAME=${MINA_DEB_CODENAME:=bullseye}
 
 
 if [ "${BUILDKITE_REPO}" != "${MINA_REPO}" ]; then 
+  # Abort if `BUILDKITE_REPO` doesn't have the expected format
+  echo ${BUILDKITE_REPO} | grep -P '^.*github.com[:\/](.*)\.git$' > /dev/null || \
+      (echo "BUILDKITE_REPO does not have the expected format" && false)
+
   # We don't want to allow some operations on fork repository which should be done on main repo only. 
   # Publish to docker hub or publish to unstable debian channel should be exclusive to main repo as it can override 
   # packages from main repo (by using the same commit and the same branch from forked repository)
 
   # We don't want to use tags (as this can replace our dockers/debian packages). Instead we are using repo name
   # For example: for given repo 'https://github.com/dkijania/mina.git' we convert it to 'dkijania_mina' 
-  export GITTAG=1.0.0$(echo ${BUILDKITE_REPO} | sed -e "s/https:\/\/github.com\///g" | sed -e "s/.git//g" | sed -e "s/\//-/g")
+  export GITTAG=1.0.0$(echo ${BUILDKITE_REPO} | sed -e 's/^.*github.com[:\/]\(.*\)\.git$/\1/' -e 's/\//-/')
   export THIS_COMMIT_TAG=""
   RELEASE=unstable
 

scripts/archive/zkapp_duplicated_precondition/add_constraint.sql

@@ -0,0 +1,2 @@
+ALTER TABLE zkapp_account_precondition
+ADD CONSTRAINT zkapp_account_precondition_unique UNIQUE(balance_id, receipt_chain_hash, delegate_id, state_id, action_state_id, proved_state, is_new, nonce_id);

scripts/archive/zkapp_duplicated_precondition/remove_duplicated_zkapp_account_precondition_rows.sh

@@ -0,0 +1,65 @@
+#!/bin/bash
+
+set -e
+
+if [ $# -ne 1 ]; then
+    echo "Usage $0 archive-db"
+    exit 0
+fi
+
+ARCHIVE_DB=$1
+BODY_DATA_BCK=zkapp_update_body_data_table.bck
+PRECONDITION_DATA_BCK=zkapp_precondition_table.bck
+PRECONDITION_DATA_TMP=zkapp_precondition_table.tmp
+PRECONDITION_UPDATE_SCRIPT=zkapp_precondition_script.sql
+BODY_UPDATE_SCRIPT=zkapp_update_body_script.sql
+
+echo "This Shell script will produce two SQL scripts that will remove duplicated zkApp account preconditions rows in the archive DB" "$ARCHIVE_DB"
+
+rm -f $BODY_DATA_BCK
+rm -f $PRECONDITION_DATA_BCK
+rm -f $PRECONDITION_DATA_TMP
+rm -f $PRECONDITION_UPDATE_SCRIPT
+rm -f $BODY_UPDATE_SCRIPT
+
+ALL_TABLES="balance_id,receipt_chain_hash,delegate_id,state_id,action_state_id,proved_state, is_new, nonce_id"
+
+echo "Creating backup of zkapp_account_preconditions table " "$PRECONDITION_DATA_BCK" 
+
+echo "SELECT * FROM zkapp_account_precondition" | \
+    psql -U postgres --csv -q -t "$ARCHIVE_DB" > $PRECONDITION_DATA_BCK
+
+echo "Creating backup of zkapp_update_body table " "$BODY_DATA_BCK"
+
+echo "SELECT * FROM zkapp_account_update_body" | \
+    psql -U postgres --csv -q -t "$ARCHIVE_DB" > $BODY_DATA_BCK
+
+
+echo "Creating temporary file of zkapp_account_preconditions table with duplicated rows" "$PRECONDITION_DATA_TMP"
+
+echo "SELECT  MIN(id) as id, $ALL_TABLES, MAX(id) as duplicated_id  \
+		FROM zkapp_account_precondition \
+		GROUP BY $ALL_TABLES \
+		HAVING COUNT(*) > 1" | \
+    psql -U postgres --csv -q -t "$ARCHIVE_DB" > $PRECONDITION_DATA_TMP
+
+
+if [[ $(wc -l < $PRECONDITION_DATA_TMP) -gt 0 ]]; then
+
+    echo "Creating SQL scripts for removing duplications" "$PRECONDITION_UPDATE_SCRIPT" "and updating references" "$BODY_UPDATE_SCRIPT"
+
+    cat $PRECONDITION_DATA_TMP | while IFS= read -r line 
+    do  
+        ID_TO_REPLACE=$(echo "$line" | awk -F , '{print $1}');
+        ID_TO_REMOVE=$(echo "$line" | awk -F , '{print $10}');
+        echo -n .
+        echo "$ID_TO_REPLACE $ID_TO_REMOVE" | awk '{print "UPDATE zkapp_account_update_body SET zkapp_account_precondition_id = " $1 " WHERE zkapp_account_precondition_id =" $2 ";"}' >> $BODY_UPDATE_SCRIPT
+        echo "$ID_TO_REMOVE" | awk '{print "DELETE FROM zkapp_account_precondition WHERE id=" $1 ";"}' >> $PRECONDITION_UPDATE_SCRIPT
+    done
+
+    echo
+    echo "SQL scripts are ready: $PRECONDITION_UPDATE_SCRIPT, $BODY_UPDATE_SCRIPT !"
+else
+    echo
+    echo "No duplicated rows found. Database state is correct !"
+fi

src/app/archive/add_indexes.sql

@@ -0,0 +1,3 @@
+CREATE INDEX idx_zkapp_field_array_element_ids ON zkapp_field_array(element_ids);
+
+CREATE INDEX idx_zkapp_events_element_ids ON zkapp_events(element_ids);

src/app/archive/dune-project

@@ -1,3 +1,3 @@
 (lang dune 2.8)
 (name archive_lib)
-(implicit_transitive_deps false)
+(implicit_transitive_deps true)

src/app/archive/lib/dune-project

@@ -1,3 +1,3 @@
 (lang dune 2.8)
 (name graphql_query)
-(implicit_transitive_deps false)
+(implicit_transitive_deps true)

src/app/archive/zkapp_tables.sql

@@ -33,6 +33,8 @@ CREATE TABLE zkapp_field_array
 , element_ids              int[]   NOT NULL
 );
 
+CREATE INDEX idx_zkapp_field_array_element_ids ON zkapp_field_array(element_ids);
+
 /* Fixed-width arrays of algebraic fields, given as id's from
    zkapp_field
 
@@ -81,6 +83,8 @@ CREATE TABLE zkapp_events
 , element_ids              int[]            NOT NULL
 );
 
+CREATE INDEX idx_zkapp_events_element_ids ON zkapp_events(element_ids);
+
 /* field elements derived from verification keys */
 CREATE TABLE zkapp_verification_key_hashes
 ( id                                    serial          PRIMARY KEY
@@ -162,6 +166,7 @@ CREATE TABLE zkapp_account_precondition
 , action_state_id          int                     REFERENCES zkapp_field(id)
 , proved_state             boolean
 , is_new                   boolean
+, UNIQUE(balance_id, receipt_chain_hash, delegate_id, state_id, action_state_id, proved_state, is_new, nonce_id)
 );
 
 CREATE TABLE zkapp_accounts

src/app/benchmarks/dune-project

@@ -1,3 +1,3 @@
 (lang dune 3.3)
 (name benchmarks)
-(implicit_transitive_deps false)
+(implicit_transitive_deps true)

src/app/cli/src/dune-project

@@ -1,3 +1,3 @@
 (lang dune 3.3)
 (name graphql)
-(implicit_transitive_deps false)
+(implicit_transitive_deps true)

src/app/dump_blocks/dune-project

@@ -1,3 +1,3 @@
 (lang dune 2.8)
 (name dump_blocks)
-(implicit_transitive_deps false)
+(implicit_transitive_deps true)

src/app/logproc/dune-project

@@ -1,3 +1,3 @@
 (lang dune 2.8)
 (name logproc)
-(implicit_transitive_deps false)
+(implicit_transitive_deps true)

src/app/reformat/dune-project

@@ -1,3 +1,3 @@
 (lang dune 3.2)
 (name reformat)
-(implicit_transitive_deps false)
+(implicit_transitive_deps true)

src/app/rosetta/dune-project

@@ -1,3 +1,3 @@
 (lang dune 3.3)
 (name rosetta)
-(implicit_transitive_deps false)
+(implicit_transitive_deps true)

src/app/runtime_genesis_ledger/dune-project

@@ -1,3 +1,3 @@
 (lang dune 2.8)
 (name runtime_genesis_ledger)
-(implicit_transitive_deps false)
+(implicit_transitive_deps true)

src/app/zkapp_test_transaction/dune-project

@@ -1,3 +1,3 @@
 (lang dune 2.8)
 (name zkapp_test_transaction)
-(implicit_transitive_deps false)
+(implicit_transitive_deps true)

src/config/debug.mlh

@@ -33,3 +33,4 @@
 [%%import "/src/config/features/dev.mlh"]
 [%%undef compaction_interval]
 [%%define vrf_poll_interval 0]
+[%%undef zkapp_cmd_limit]

src/config/dev.mlh

@@ -35,3 +35,4 @@
 [%%import "/src/config/features/dev.mlh"]
 [%%undef compaction_interval]
 [%%define vrf_poll_interval 0]
+[%%undef zkapp_cmd_limit]

src/config/dev_medium_curves.mlh

@@ -33,3 +33,4 @@
 [%%import "/src/config/features/dev.mlh"]
 [%%undef compaction_interval]
 [%%define vrf_poll_interval 0]
+[%%undef zkapp_cmd_limit]

src/config/dev_snark.mlh

@@ -33,3 +33,4 @@
 [%%import "/src/config/features/dev.mlh"]
 [%%undef compaction_interval]
 [%%define vrf_poll_interval 0]
+[%%undef zkapp_cmd_limit]

src/config/devnet.mlh

@@ -46,3 +46,4 @@
 (* 2*block_window_duration *)
 [%%define compaction_interval 360000]
 [%%define vrf_poll_interval 5000]
+[%%undef zkapp_cmd_limit]

src/config/fake_hash.mlh

@@ -33,3 +33,4 @@
 [%%import "/src/config/features/dev.mlh"]
 [%%undef compaction_interval]
 [%%define vrf_poll_interval 0]
+[%%undef zkapp_cmd_limit]

src/config/fork.mlh

@@ -1,3 +1,4 @@
 [%%undef fork_previous_length]
 [%%undef fork_previous_state_hash]
 [%%undef fork_genesis_slot]
+[%%undef zkapp_cmd_limit]

src/config/fork_at_3757.mlh

@@ -1,3 +1,4 @@
 [%%define fork_previous_length 3757]
 [%%define fork_previous_state_hash "3NKR3QYJ7qwxiGgX39umahgdT8BH5yXBQwQtpYZdvodCXcsndK7f"]
-[%%define fork_genesis_slot 12796]
+[%%define fork_genesis_slot 12796]
+[%%undef zkapp_cmd_limit]

src/config/fuzz_medium.mlh

@@ -32,3 +32,4 @@
 [%%import "/src/config/features/dev.mlh"]
 [%%undef compaction_interval]
 [%%define vrf_poll_interval 0]
+[%%undef zkapp_cmd_limit]

src/config/fuzz_small.mlh

@@ -32,3 +32,4 @@
 [%%import "/src/config/features/dev.mlh"]
 [%%undef compaction_interval]
 [%%define vrf_poll_interval 0]
+[%%undef zkapp_cmd_limit]

src/config/integration_tests.mlh

@@ -5,3 +5,4 @@
 (* This profile is only used for the test executive binary, so we don't need
    snark keys, a valid genesis proof, etc. *)
 [%%import "/src/config/proof_level/none.mlh"]
+[%%undef zkapp_cmd_limit]

src/config/lightnet.mlh

@@ -46,3 +46,4 @@
 (* 2*block_window_duration *)
 [%%define compaction_interval 360000]
 [%%define vrf_poll_interval 5000]
+[%%undef zkapp_cmd_limit]

src/config/mainnet.mlh

@@ -46,3 +46,4 @@
 (* 2*block_window_duration *)
 [%%define compaction_interval 360000]
 [%%define vrf_poll_interval 5000]
+[%%undef zkapp_cmd_limit]

src/config/nonconsensus_mainnet.mlh

@@ -2,3 +2,4 @@
 
 [%%undef consensus_mechanism]
 [%%undef compaction_interval]
+[%%undef zkapp_cmd_limit]

src/config/nonconsensus_medium_curves.mlh

@@ -32,3 +32,4 @@
 [%%import "/src/config/features/dev.mlh"]
 [%%undef compaction_interval]
 [%%define vrf_poll_interval 5000]
+[%%undef zkapp_cmd_limit]

src/config/print_versioned_types.mlh

@@ -33,3 +33,4 @@
 [%%import "/src/config/features/dev.mlh"]
 [%%undef compaction_interval]
 [%%define vrf_poll_interval 0]
+[%%undef zkapp_cmd_limit]

src/config/test_archive_processor.mlh

@@ -33,3 +33,4 @@
 [%%import "/src/config/features/dev.mlh"]
 [%%undef compaction_interval]
 [%%define vrf_poll_interval 0]
+[%%undef zkapp_cmd_limit]

src/config/test_postake.mlh

@@ -33,3 +33,4 @@
 [%%import "/src/config/features/dev.mlh"]
 [%%undef compaction_interval]
 [%%define vrf_poll_interval 5000]
+[%%undef zkapp_cmd_limit]

src/config/test_postake_catchup.mlh

@@ -33,3 +33,4 @@
 [%%import "/src/config/features/dev.mlh"]
 [%%undef compaction_interval]
 [%%define vrf_poll_interval 0]
+[%%undef zkapp_cmd_limit]

src/config/test_postake_five_even_txns.mlh

@@ -33,3 +33,4 @@
 [%%import "/src/config/features/dev.mlh"]
 [%%undef compaction_interval]
 [%%define vrf_poll_interval 0]
+[%%undef zkapp_cmd_limit]

src/config/test_postake_full_epoch.mlh

@@ -33,3 +33,4 @@
 [%%import "/src/config/features/dev.mlh"]
 [%%undef compaction_interval]
 [%%define vrf_poll_interval 0]
+[%%undef zkapp_cmd_limit]

src/config/test_postake_holy_grail.mlh

@@ -32,3 +32,4 @@
 [%%import "/src/config/fork.mlh"]
 [%%import "/src/config/features/dev.mlh"]
 [%%undef compaction_interval]
+[%%undef zkapp_cmd_limit]

src/config/test_postake_medium_curves.mlh

@@ -32,3 +32,4 @@
 [%%import "/src/config/fork.mlh"]
 [%%import "/src/config/features/dev.mlh"]
 [%%undef compaction_interval]
+[%%undef zkapp_cmd_limit]

src/config/test_postake_snarkless.mlh

@@ -33,3 +33,4 @@
 [%%import "/src/config/features/dev.mlh"]
 [%%define compaction_interval 360000]
 [%%define vrf_poll_interval 0]
+[%%undef zkapp_cmd_limit]

src/config/test_postake_snarkless_medium_curves.mlh

@@ -33,3 +33,4 @@
 [%%import "/src/config/features/dev.mlh"]
 [%%undef compaction_interval]
 [%%define vrf_poll_interval 0]
+[%%undef zkapp_cmd_limit]

src/config/test_postake_split.mlh

@@ -33,3 +33,4 @@
 [%%import "/src/config/features/dev.mlh"]
 [%%undef compaction_interval]
 [%%define vrf_poll_interval 0]
+[%%undef zkapp_cmd_limit]

src/config/test_postake_split_medium_curves.mlh

@@ -33,3 +33,4 @@
 [%%import "/src/config/features/dev.mlh"]
 [%%undef compaction_interval]
 [%%define vrf_poll_interval 0]
+[%%undef zkapp_cmd_limit]

src/config/test_postake_three_producers.mlh

@@ -34,3 +34,4 @@
 [%%import "/src/config/features/dev.mlh"]
 [%%undef compaction_interval]
 [%%define vrf_poll_interval 0]
+[%%undef zkapp_cmd_limit]

src/config/testnet_postake.mlh

@@ -34,3 +34,4 @@
 [%%import "/src/config/features/dev.mlh"]
 [%%undef compaction_interval]
 [%%define vrf_poll_interval 5000]
+[%%undef zkapp_cmd_limit]

src/config/testnet_postake_many_producers.mlh

@@ -32,3 +32,4 @@
 [%%import "/src/config/features/dev.mlh"]
 [%%undef compaction_interval]
 [%%define vrf_poll_interval 5000]
+[%%undef zkapp_cmd_limit]

src/config/testnet_postake_many_producers_medium_curves.mlh

@@ -34,3 +34,4 @@
 [%%import "/src/config/features/dev.mlh"]
 [%%undef compaction_interval]
 [%%define vrf_poll_interval 5000]
+[%%undef zkapp_cmd_limit]

src/config/testnet_postake_medium_curves.mlh

@@ -46,3 +46,4 @@
 (* 2*block_window_duration *)
 [%%define compaction_interval 360000]
 [%%define vrf_poll_interval 5000]
+[%%undef zkapp_cmd_limit]

src/config/testnet_postake_snarkless.mlh

@@ -34,3 +34,4 @@
 [%%import "/src/config/features/dev.mlh"]
 [%%undef compaction_interval]
 [%%define vrf_poll_interval 0]
+[%%undef zkapp_cmd_limit]

src/config/testnet_postake_snarkless_fake_hash.mlh

@@ -35,3 +35,4 @@
 [%%import "/src/config/features/dev.mlh"]
 [%%undef compaction_interval]
 [%%define vrf_poll_interval 0]
+[%%undef zkapp_cmd_limit]

src/config/testnet_public.mlh

@@ -34,3 +34,4 @@
 [%%import "/src/config/features/dev.mlh"]
 [%%undef compaction_interval]
 [%%define vrf_poll_interval 5000]
+[%%undef zkapp_cmd_limit]

src/config/txpool_size.mlh

@@ -1,3 +1,4 @@
 (* Note this value needs to be consistent across nodes to prevent spurious bans.
    see comment in transaction_pool.ml for more details. *)
 [%%define pool_max_size 3000]
+[%%undef zkapp_cmd_limit]

src/dune-project

@@ -1,5 +1,5 @@
 (lang dune 3.3)
-(implicit_transitive_deps false)
+(implicit_transitive_deps true)
 
 (package (name allocation_functor))
 (package (name archive_blocks))

src/lib/block_producer/block_producer.ml

@@ -71,6 +71,8 @@ let lift_sync f =
            [%log' error (Logger.create ())] "Ivar.fill bug is here!" ;
          Ivar.fill ivar (f ()) ) )
 
+let zkapp_cmd_limit = ref Mina_compile_config.zkapp_cmd_limit
+
 module Singleton_scheduler : sig
   type t
 
@@ -160,10 +162,10 @@ let report_transaction_inclusion_failures ~logger failed_txns =
       let leftover_bytes = available_bytes - base_error_size - 2 in
       wrap_error (`List (generate_errors failed_txns leftover_bytes)) )
 
-let generate_next_state ~constraint_constants ~previous_protocol_state
-    ~time_controller ~staged_ledger ~transactions ~get_completed_work ~logger
-    ~(block_data : Consensus.Data.Block_data.t) ~winner_pk ~scheduled_time
-    ~log_block_creation ~block_reward_threshold =
+let generate_next_state ~zkapp_cmd_limit ~constraint_constants
+    ~previous_protocol_state ~time_controller ~staged_ledger ~transactions
+    ~get_completed_work ~logger ~(block_data : Consensus.Data.Block_data.t)
+    ~winner_pk ~scheduled_time ~log_block_creation ~block_reward_threshold =
   let open Interruptible.Let_syntax in
   let previous_protocol_state_body_hash =
     Protocol_state.body previous_protocol_state |> Protocol_state.Body.hash
@@ -200,7 +202,7 @@ let generate_next_state ~constraint_constants ~previous_protocol_state
                 staged_ledger ~coinbase_receiver ~logger
                 ~current_state_view:previous_state_view
                 ~transactions_by_fee:transactions ~get_completed_work
-                ~log_block_creation ~supercharge_coinbase
+                ~log_block_creation ~supercharge_coinbase ~zkapp_cmd_limit
               |> Result.map ~f:(fun (diff, failed_txns) ->
                      if not (List.is_empty failed_txns) then
                        don't_wait_for
@@ -752,6 +754,7 @@ let run ~context:(module Context : CONTEXT) ~vrf_evaluator ~prover ~verifier
                 ~staged_ledger:(Breadcrumb.staged_ledger crumb)
                 ~transactions ~get_completed_work ~logger ~log_block_creation
                 ~winner_pk:winner_pubkey ~block_reward_threshold
+                ~zkapp_cmd_limit:!zkapp_cmd_limit
             in
             [%log internal] "Generate_next_state_done" ;
             match next_state_opt with