Merge branch 'rampup' into fix/wrap-circuit-features

buildkite/scripts/build-artifact.sh

@@ -28,6 +28,8 @@ dune build "--profile=${DUNE_PROFILE}" \
   src/app/replayer/replayer.exe \
   src/app/extract_blocks/extract_blocks.exe \
   src/app/archive_blocks/archive_blocks.exe \
+  src/app/last_vrf_output_to_b64/last_vrf_output_to_b64.exe \
+  src/app/receipt_chain_hash_to_b58/receipt_chain_hash_to_b58.exe \
   src/app/batch_txn_tool/batch_txn_tool.exe \
   src/app/missing_blocks_auditor/missing_blocks_auditor.exe \
   src/app/swap_bad_balances/swap_bad_balances.exe \

buildkite/scripts/connect-to-berkeley.sh

@@ -16,16 +16,17 @@ export DEBIAN_FRONTEND=noninteractive
 apt-get update
 apt-get install -y git apt-transport-https ca-certificates tzdata curl
 
-TESTNET_NAME="berkeley"
+TESTNET_VERSION_NAME="berkeley"
+TESTNET_NAME="testworld-2-0"
 
 git config --global --add safe.directory /workdir
 
 source buildkite/scripts/export-git-env-vars.sh
 
-echo "Installing mina daemon package: mina-${TESTNET_NAME}=${MINA_DEB_VERSION}"
+echo "Installing mina daemon package: mina-${TESTNET_VERSION_NAME}=${MINA_DEB_VERSION}"
 echo "deb [trusted=yes] http://packages.o1test.net $MINA_DEB_CODENAME $MINA_DEB_RELEASE" | tee /etc/apt/sources.list.d/mina.list
 apt-get update
-apt-get install --allow-downgrades -y "mina-${TESTNET_NAME}=${MINA_DEB_VERSION}"
+apt-get install --allow-downgrades -y "mina-${TESTNET_VERSION_NAME}=${MINA_DEB_VERSION}"
 
 # Remove lockfile if present
 rm ~/.mina-config/.mina-lock ||:
@@ -43,10 +44,10 @@ mina daemon \
   --libp2p-keypair "/root/libp2p-keys/key" \
 & # -background
 
-# Attempt to connect to the GraphQL client every 10s for up to 4 minutes
+# Attempt to connect to the GraphQL client every 30s for up to 12 minutes
 num_status_retries=24
 for ((i=1;i<=$num_status_retries;i++)); do
-  sleep 10s
+  sleep 30s
   set +e
   mina client status
   status_exit_code=$?

buildkite/scripts/connect-to-mainnet-on-compatible.sh

@@ -37,10 +37,10 @@ mina daemon \
 & # -background
 
 
-# Attempt to connect to the GraphQL client every 10s for up to 4 minutes
+# Attempt to connect to the GraphQL client every 30s for up to 12 minutes
 num_status_retries=24
 for ((i=1;i<=$num_status_retries;i++)); do
-  sleep 10s
+  sleep 30s
   set +e
   mina client status
   status_exit_code=$?

buildkite/scripts/connect-to-testnet.sh

@@ -0,0 +1,78 @@
+#!/bin/bash
+
+set -eo pipefail
+
+if [[ $# -ne 3 ]]; then
+    echo "Usage: $0 '<testnet-name>' '<wait-between-polling-graphql>''<wait-after-final-check>'"
+    exit 1
+fi
+
+TESTNET_VERSION_NAME="berkeley"
+TESTNET_NAME=$1
+WAIT_BETWEEN_POLLING_GRAPHQL=$2
+WAIT_AFTER_FINAL_CHECK=$3
+
+case "$BUILDKITE_PULL_REQUEST_BASE_BRANCH" in
+  rampup|berkeley|release/2.0.0|develop)
+  ;;
+  *)
+    echo "Not pulling against rampup, not running the connect test"
+    exit 0 ;;
+esac
+
+# Don't prompt for answers during apt-get install
+export DEBIAN_FRONTEND=noninteractive
+
+apt-get update
+apt-get install -y git apt-transport-https ca-certificates tzdata curl
+
+git config --global --add safe.directory /workdir
+
+source buildkite/scripts/export-git-env-vars.sh
+
+echo "Installing mina daemon package: mina-${TESTNET_VERSION_NAME}=${MINA_DEB_VERSION}"
+echo "deb [trusted=yes] http://packages.o1test.net $MINA_DEB_CODENAME $MINA_DEB_RELEASE" | tee /etc/apt/sources.list.d/mina.list
+apt-get update
+apt-get install --allow-downgrades -y "mina-${TESTNET_VERSION_NAME}=${MINA_DEB_VERSION}"
+
+# Remove lockfile if present
+rm ~/.mina-config/.mina-lock ||:
+
+mkdir -p /root/libp2p-keys/
+# Pre-generated random password for this quick test
+export MINA_LIBP2P_PASS=eithohShieshichoh8uaJ5iefo1reiRudaekohG7AeCeib4XuneDet2uGhu7lahf
+mina libp2p generate-keypair --privkey-path /root/libp2p-keys/key
+# Set permissions on the keypair so the daemon doesn't complain
+chmod -R 0700 /root/libp2p-keys/
+
+# Restart in the background
+mina daemon \
+  --peer-list-url "https://storage.googleapis.com/seed-lists/${TESTNET_NAME}_seeds.txt" \
+  --libp2p-keypair "/root/libp2p-keys/key" \
+& # -background
+
+# Attempt to connect to the GraphQL client every 10s for up to 8 minutes
+num_status_retries=24
+for ((i=1;i<=$num_status_retries;i++)); do
+  sleep $WAIT_BETWEEN_POLLING_GRAPHQL
+  set +e
+  mina client status
+  status_exit_code=$?
+  set -e
+  if [ $status_exit_code -eq 0 ]; then
+    break
+  elif [ $i -eq $num_status_retries ]; then
+    exit $status_exit_code
+  fi
+done
+
+# Check that the daemon has connected to peers and is still up after 2 mins
+sleep $WAIT_AFTER_FINAL_CHECK
+mina client status
+if [ $(mina advanced get-peers | wc -l) -gt 0 ]; then
+    echo "Found some peers"
+else
+    echo "No peers found"
+    exit 1
+fi
+

buildkite/scripts/export-git-env-vars.sh

@@ -3,6 +3,8 @@ set -euo pipefail
 
 echo "Exporting Variables: "
 
+export MINA_REPO="https://github.com/MinaProtocol/mina.git"
+
 function find_most_recent_numeric_tag() {
     TAG=$(git describe --always --abbrev=0 $1 | sed 's!/!-!g; s!_!-!g')
     if [[ $TAG != [0-9]* ]]; then
@@ -13,8 +15,7 @@ function find_most_recent_numeric_tag() {
 
 export GITHASH=$(git rev-parse --short=7 HEAD)
 export GITBRANCH=$(git rev-parse --symbolic-full-name --abbrev-ref HEAD |  sed 's!/!-!g; s!_!-!g' )
-# GITTAG is the closest tagged commit to this commit, while THIS_COMMIT_TAG only has a value when the current commit is tagged
-export GITTAG=$(find_most_recent_numeric_tag HEAD)
+
 export THIS_COMMIT_TAG=$(git tag --points-at HEAD)
 export PROJECT="mina"
 
@@ -24,8 +25,47 @@ export BUILD_URL=${BUILDKITE_BUILD_URL}
 set -u
 
 export MINA_DEB_CODENAME=${MINA_DEB_CODENAME:=bullseye}
-
 [[ -n "$BUILDKITE_BRANCH" ]] && export GITBRANCH=$(echo "$BUILDKITE_BRANCH" | sed 's!/!-!g; s!_!-!g')
+
+
+if [ "${BUILDKITE_REPO}" != "${MINA_REPO}" ]; then 
+  # We don't want to allow some operations on fork repository which should be done on main repo only. 
+  # Publish to docker hub or publish to unstable debian channel should be exclusive to main repo as it can override 
+  # packages from main repo (by using the same commit and the same branch from forked repository)
+
+  # We don't want to use tags (as this can replace our dockers/debian packages). Instead we are using repo name
+  # For example: for given repo 'https://github.com/dkijania/mina.git' we convert it to 'dkijania_mina' 
+  export GITTAG=1.0.0$(echo ${BUILDKITE_REPO} | sed -e "s/https:\/\/github.com\///g" | sed -e "s/.git//g" | sed -e "s/\//-/g")
+  export THIS_COMMIT_TAG=""
+  RELEASE=unstable
+
+else
+  # GITTAG is the closest tagged commit to this commit, while THIS_COMMIT_TAG only has a value when the current commit is tagged
+  export GITTAG=$(find_most_recent_numeric_tag HEAD)
+
+  # Determine deb repo to use
+  case $GITBRANCH in
+      berkeley|rampup|compatible|master|release*) # whitelist of branches that can be tagged
+          case "${THIS_COMMIT_TAG}" in
+            *alpha*) # any tag including the string `alpha`
+              RELEASE=alpha ;;
+            *beta*) # any tag including the string `beta`
+              RELEASE=beta ;;
+            *rampup*) # any tag including the string `rampup`
+              RELEASE=rampup ;;
+            ?*) # Any other non-empty tag. ? matches a single character and * matches 0 or more characters.
+              RELEASE=stable ;;
+            "") # No tag
+              RELEASE=unstable ;;
+            *) # The above set of cases should be exhaustive, if they're not then still set RELEASE=unstable
+              RELEASE=unstable
+              echo "git tag --points-at HEAD may have failed, falling back to unstable. Value: \"$(git tag --points-at HEAD)\""
+              ;;
+          esac ;;
+      *)
+          RELEASE=unstable ;;
+  esac
+fi
 
 if [[ -n "${THIS_COMMIT_TAG}" ]]; then # If the commit is tagged
     export MINA_DEB_VERSION="${GITTAG}-${GITHASH}"
@@ -35,30 +75,6 @@ else
     export MINA_DOCKER_TAG="$(echo "${MINA_DEB_VERSION}-${MINA_DEB_CODENAME}" | sed 's!/!-!g; s!_!-!g')"
 fi
 
-
-# Determine deb repo to use
-case $GITBRANCH in
-    berkeley|rampup|compatible|master|release*) # whitelist of branches that can be tagged
-        case "${THIS_COMMIT_TAG}" in
-          *alpha*) # any tag including the string `alpha`
-            RELEASE=alpha ;;
-          *beta*) # any tag including the string `beta`
-            RELEASE=beta ;;
-          *rampup*) # any tag including the string `rampup`
-            RELEASE=rampup ;;
-          ?*) # Any other non-empty tag. ? matches a single character and * matches 0 or more characters.
-            RELEASE=stable ;;
-          "") # No tag
-            RELEASE=unstable ;;
-          *) # The above set of cases should be exhaustive, if they're not then still set RELEASE=unstable
-            RELEASE=unstable
-            echo "git tag --points-at HEAD may have failed, falling back to unstable. Value: \"$(git tag --points-at HEAD)\""
-            ;;
-        esac ;;
-    *)
-        RELEASE=unstable ;;
-esac
-
 # Determine the packages to build (mainnet y/N)
 case $GITBRANCH in
     compatible|master|release/1*) # whitelist of branches that are "mainnet-like"

buildkite/scripts/generate-diff.sh

@@ -2,6 +2,8 @@
 
 TAG=$(git tag --points-at HEAD)
 
+source buildkite/scripts/handle-fork.sh
+
 # If this is not a PR build, or the HEAD is tagged, the entire build is dirty
 if [ -z "${BUILDKITE_PULL_REQUEST_BASE_BRANCH}" ]; then
   echo "This is not a PR build; considering all files dirty" >&2
@@ -11,7 +13,7 @@ elif [ -n "${TAG}" ]; then
   git ls-files
 else
   COMMIT=$(git log -1 --pretty=format:%H)
-  BASE_COMMIT=$(git log "origin/${BUILDKITE_PULL_REQUEST_BASE_BRANCH}" -1 --pretty=format:%H)
+  BASE_COMMIT=$(git log "${REMOTE}/${BUILDKITE_PULL_REQUEST_BASE_BRANCH}" -1 --pretty=format:%H)
   echo "Diffing current commit: ${COMMIT} against branch: ${BUILDKITE_PULL_REQUEST_BASE_BRANCH} (${BASE_COMMIT})" >&2 
-  git diff "origin/${BUILDKITE_PULL_REQUEST_BASE_BRANCH}" --name-only
+  git diff "${REMOTE}/${BUILDKITE_PULL_REQUEST_BASE_BRANCH}" --name-only
 fi

buildkite/scripts/handle-fork.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+export MINA_REPO="https://github.com/MinaProtocol/mina.git"
+
+if [ "${BUILDKITE_REPO}" ==  ${MINA_REPO} ]; then
+    echo "This is not a Forked repo, skipping..."
+    export REMOTE="origin"
+    export FORK=0
+else
+    git remote add mina ${MINA_REPO} || true
+    git fetch mina
+    export REMOTE="mina"
+    export FORK=1
+fi

buildkite/scripts/merges-cleanly.sh

@@ -4,21 +4,23 @@ BRANCH=$1
 CURRENT=$(git branch --show-current)
 echo 'Testing for conflicts between the current branch `'"${CURRENT}"'` and `'"${BRANCH}"'`...'
 
+
 # Adapted from this stackoverflow answer: https://stackoverflow.com/a/10856937
 # The git merge-tree command shows the content of a 3-way merge without
 # touching the index, which we can then search for conflict markers.
 
 # Tell git where to find ssl certs
 git config --global http.sslCAInfo /etc/ssl/certs/ca-bundle.crt
-# Fetch a fresh copy of the repo
-git fetch origin
+
+source buildkite/scripts/handle-fork.sh
+
 git config --global user.email "[email protected]"
 git config --global user.name "It's me, CI"
 # Check mergeability. We use flags so that
 # * `--no-commit` stops us from updating the index with a merge commit,
 # * `--no-ff` stops us from updating the index to the HEAD, if the merge is a
 #   straightforward fast-forward
-git merge --no-commit --no-ff origin/$BRANCH
+git merge --no-commit --no-ff ${REMOTE}/$BRANCH
 
 RET=$?
 

buildkite/scripts/refresh_code.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+source ./buildkite/scripts/handle-fork.sh
+git fetch ${REMOTE}

buildkite/scripts/run-snark-transaction-profiler.sh

@@ -8,13 +8,7 @@ export DEBIAN_FRONTEND=noninteractive
 apt-get update
 apt-get install -y git apt-transport-https ca-certificates tzdata curl python3
 
-case "$BUILDKITE_PULL_REQUEST_BASE_BRANCH" in
-  rampup|berkeley|release/2.0.0|develop)
-    TESTNET_NAME="berkeley"
-  ;;
-  *)
-    TESTNET_NAME="mainnet"
-esac
+TESTNET_NAME="berkeley"
 
 git config --global --add safe.directory /workdir
 
@@ -29,5 +23,5 @@ K=1
 MAX_NUM_UPDATES=4
 MIN_NUM_UPDATES=2   
 
-echo "--- Run Snark Transaction Profiler with parameters: --zkapps --k ${K} --max-num-updates ${MAX_NUM_UPDATES} --min-num-updates ${MIN_NUM_UPDATES}"
-python3 ./scripts/snark_transaction_profiler.py ${K} ${MAX_NUM_UPDATES} ${MIN_NUM_UPDATES}
+echo "-- Run Snark Transaction Profiler with parameters: --zkapps --k ${K} --max-num-updates ${MAX_NUM_UPDATES} --min-num-updates ${MIN_NUM_UPDATES}"
+python3 ./scripts/snark_transaction_profiler.py ${K} ${MAX_NUM_UPDATES} ${MIN_NUM_UPDATES}

buildkite/scripts/version-linter.sh

@@ -15,13 +15,15 @@ apt-get install -y git apt-transport-https ca-certificates tzdata curl python3 p
 
 git config --global --add safe.directory /workdir
 
+
+source buildkite/scripts/handle-fork.sh
 source buildkite/scripts/export-git-env-vars.sh
 
-pip3 install sexpdata
+pip3 install sexpdata==1.0.0
 
-base_branch=origin/${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
+base_branch=${REMOTE}/${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
 pr_branch=origin/${BUILDKITE_BRANCH}
-release_branch=origin/$1
+release_branch=${REMOTE}/$1
 
 echo "--- Run Python version linter with branches: ${pr_branch} ${base_branch} ${release_branch}"
 ./scripts/version-linter.py ${pr_branch} ${base_branch} ${release_branch}

buildkite/src/Command/ConnectToTestnet.dhall

@@ -9,20 +9,24 @@ let B/SoftFail = B.definitions/commandStep/properties/soft_fail/Type
 
 let Cmd = ../Lib/Cmds.dhall in
 
-{ step = \(dependsOn : List Command.TaggedKey.Type) ->
-    Command.build
-      Command.Config::{
-        commands = [
-          Cmd.runInDocker
-            Cmd.Docker::{
-              image = (../Constants/ContainerImages.dhall).ubuntu2004
-            }
-            "./buildkite/scripts/connect-to-berkeley.sh"
-        ],
-        label = "Connect to Berkeley",
-        soft_fail = Some (B/SoftFail.Boolean True),
-        key = "connect-to-berkeley",
-        target = Size.Large,
-        depends_on = dependsOn
-      }
+{ step = \(dependsOn : List Command.TaggedKey.Type) 
+    -> \(testnet : Text) 
+    -> \(wait_between_graphql_poll : Text) 
+    -> \(wait_before_final_check: Text )
+    -> \(soft_fail: B/SoftFail)
+    -> Command.build
+        Command.Config::{
+          commands = [
+            Cmd.runInDocker
+              Cmd.Docker::{
+                image = (../Constants/ContainerImages.dhall).ubuntu2004
+              }
+              "./buildkite/scripts/connect-to-testnet.sh ${testnet} ${wait_between_graphql_poll} ${wait_before_final_check}"
+          ],
+          label = "Connect to ${testnet}",
+          soft_fail = Some soft_fail,
+          key = "connect-to-${testnet}",
+          target = Size.Large,
+          depends_on = dependsOn
+        }
 }