From 806d4073d0293119d446414c34119058f35bc7ef Mon Sep 17 00:00:00 2001 From: Paul Steckler Date: Mon, 27 Feb 2023 17:42:19 -0800 Subject: [PATCH 1/7] New version linter --- buildkite/scripts/build-artifact.sh | 7 + buildkite/scripts/version-linter.sh | 20 +++ buildkite/src/Jobs/Test/VersionLint.dhall | 47 +++++ buildkite/src/Lib/Cmds.dhall | 1 - scripts/version-linter.py | 195 +++++++++++++++++++++ src/lib/mina_networking/mina_networking.ml | 6 +- 6 files changed, 272 insertions(+), 4 deletions(-) create mode 100755 buildkite/scripts/version-linter.sh create mode 100644 buildkite/src/Jobs/Test/VersionLint.dhall create mode 100755 scripts/version-linter.py diff --git a/buildkite/scripts/build-artifact.sh b/buildkite/scripts/build-artifact.sh index adb98a79ff8..2b020cd8ad5 100755 --- a/buildkite/scripts/build-artifact.sh +++ b/buildkite/scripts/build-artifact.sh @@ -35,6 +35,13 @@ dune build "--profile=${DUNE_PROFILE}" \ src/app/rosetta/rosetta_testnet_signatures.exe \ src/app/test_executive/test_executive.exe # 2>&1 | tee /tmp/buildocaml.log +TYPE_SHAPE_FILE=${MINA_COMMIT_SHA1:0:7}-type_shape.txt +MAX_DEPTH=12 + +echo "--- Create type shapes git note for commit: ${MINA_COMMIT_SHA1:0:7}" +./_build/default/src/app/cli/src/mina_testnet_signatures.exe \ + internal dump-type-shapes --max-depth ${MAX_DEPTH} > ${TYPE_SHAPE_FILE} + echo "--- Bundle all packages for Debian ${MINA_DEB_CODENAME}" echo " Includes mina daemon, archive-node, rosetta, generate keypair for berkeley" [[ ${MINA_BUILD_MAINNET} ]] && echo " MINA_BUILD_MAINNET is true so this includes the mainnet and devnet packages for mina-daemon as well" diff --git a/buildkite/scripts/version-linter.sh b/buildkite/scripts/version-linter.sh new file mode 100755 index 00000000000..4907d25128c --- /dev/null +++ b/buildkite/scripts/version-linter.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +set -eo pipefail + +if [[ $# -ne 1 ]]; then + echo "Usage: $0 " + exit 1 +fi + +source ~/.profile + +echo "--- Make build" +export LIBP2P_NIXLESS=1 PATH=/usr/lib/go/bin:$PATH GO=/usr/lib/go/bin/go +time make build + +base_branch=${BUILDKITE_PULL_REQUEST_BASE_BRANCH} +release_branch=$1 + +echo "--- Run Python version linter" +./scripts/version-linter.py ${base_branch} ${release_branch} diff --git a/buildkite/src/Jobs/Test/VersionLint.dhall b/buildkite/src/Jobs/Test/VersionLint.dhall new file mode 100644 index 00000000000..91ba63afb5b --- /dev/null +++ b/buildkite/src/Jobs/Test/VersionLint.dhall @@ -0,0 +1,47 @@ +let Prelude = ../../External/Prelude.dhall + +let Cmd = ../../Lib/Cmds.dhall +let S = ../../Lib/SelectFiles.dhall +let D = S.PathPattern + +let Pipeline = ../../Pipeline/Dsl.dhall +let JobSpec = ../../Pipeline/JobSpec.dhall + +let Command = ../../Command/Base.dhall +let RunInToolchain = ../../Command/RunInToolchain.dhall +let Docker = ../../Command/Docker/Type.dhall +let Size = ../../Command/Size.dhall + +let buildTestCmd : Text -> Size -> Command.Type = \(release_branch : Text) -> \(cmd_target : Size) -> + Command.build + Command.Config::{ + commands = [ Cmd.run "./buildkite/scripts/version-linter.sh ${release_branch}" ], + label = "Versioned type linter", + key = "version-linter", + target = cmd_target, + docker = None Docker.Type, + artifact_paths = [ S.contains "core_dumps/*" ] + } + +in + +Pipeline.build + Pipeline.Config::{ + spec = + let lintDirtyWhen = [ + S.strictlyStart (S.contains "src/lib"), + S.exactly "buildkite/src/Jobs/Test/VersionLint" "dhall", + S.exactly "buildkite/scripts/version_linter" "sh" + ] + + in + + JobSpec::{ + dirtyWhen = lintDirtyWhen, + path = "Test", + name = "VersionLint" + }, + steps = [ + buildTestCmd "develop" Size.Small + ] + } diff --git a/buildkite/src/Lib/Cmds.dhall b/buildkite/src/Lib/Cmds.dhall index f4acda8c00d..6919469fe08 100644 --- a/buildkite/src/Lib/Cmds.dhall +++ b/buildkite/src/Lib/Cmds.dhall @@ -143,4 +143,3 @@ let tests = in module ../Constants/ContainerEnvVars.dhall - diff --git a/scripts/version-linter.py b/scripts/version-linter.py new file mode 100755 index 00000000000..e9f72f72f56 --- /dev/null +++ b/scripts/version-linter.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 + +# version-linter.py -- makes sure serializations of versioned types don't change + +import subprocess +import os +import sys +import re + +exit_code=0 + +def set_error(): + global exit_code + exit_code=1 + +def branch_commit(branch): + result=subprocess.run(['git','log','-n','1','--format="%h"','--abbrev=7','--no-merges',f'{branch}'], + capture_output=True) + return result.stdout.decode('ascii').replace('"','').replace('\n','') + +def download_s3_type_shapes(role,branch,sha1) : + # Andrew : TODO download file + print ('Downloading type shape file for',role,'branch',branch,'at commit',sha1) + +def type_shape_file(sha1) : + # created by buildkite build-artifact script + # loaded to S3 bucket + sha1 + '-type-shapes.txt' + +def make_type_shape_dict(type_shape_file): + shape_dict=dict() + with open(type_shape_file) as file : + for entry in file : + if entry=='': + break + fields=entry.split(', ') + path=fields[0] + digest=fields[1] + shape=fields[2] + type_=fields[3] + shape_dict[path]=[digest,shape,type_] + return shape_dict + +def type_name_from_path(path): + components=path.split('.') + return components[len(components)-1] + +def module_path_from_path(path): + # filename.ml:module_path + components=path.split(':') + mod_path=components[1].split('.') + # modules ... Stable.Vn.t + return mod_path + +def is_signed_command_module_path(modpath): + return (modpath[0]=='Mina_base__Signed_command' and + modpath[1]=='Make_str' and + modpath[2]=='Stable' and + re.match(r'V[0-9]+',modpath[3]) and + modpath[4]=='t') + +def is_zkapp_command_module_path(modpath): + return (modpath[0]=='Mina_base__Zkapp_command' and + modpath[1]=='T' and + modpath[2]=='Stable' and + re.match(r'V[0-9]+',modpath[3]) and + modpath[4]=='t') + +def check_rpc_types(pr_type_dict,release_branch,release_type_dict) : + # every query, response type in release still present + for path in release_type_dict: + type_name=type_name_from_path(path) + if type_name=='query' or type_name=='response': + release_shape=release_type_dict[path] + if path in pr_type_dict: + pr_shape=pr_type_dict[path] + else: + pr_shape=None + if pr_shape is None: + print('RPC type at path',path,f'in release branch ({release_branch}) is missing from PR branch') + set_error() + +def check_command_types(pr_type_dict,release_branch,release_type_dict) : + # every signed command, zkapp command type in release still present + for path in release_type_dict: + module_path=module_path_from_path(path) + if is_signed_command_module_path(module_path) or is_zkapp_command_module_path(module_path): + release_shape=release_type_dict[path] + if path in pr_type_dict: + pr_shape=pr_type_dict[path] + else: + pr_shape=None + if pr_shape is None: + print('Command type at path',path,f'in release branch ({release_branch}) is missing from PR branch') + set_error() + +def check_type_shapes(pr_branch_dict,base_branch,base_type_dict,release_branch,release_type_dict) : + for path in pr_branch_dict: + if path in release_type_dict: + release_info=release_type_dict[path] + else: + release_info=None + if path in base_type_dict: + base_info=base_type_dict[path] + else: + base_info=None + pr_info=pr_branch_dict[path] + # an error if the shape has changed from both the release and base branches + # see RFC 0047 + if not release_info is None and not base_info is None: + # shape digests may differ, even if depth-limited shapes are the same + pr_digest=pr_info[0] + pr_shape=pr_info[1] + pr_type=pr_info[2] + release_digest=release_info[0] + release_shape=release_info[1] + release_type=release_info[2] + base_digest=base_info[0] + base_shape=base_info[1] + base_type=base_info[2] + if not pr_shape==release_shape and not pr_shape==base_shape: + print(f'At path {path}:') + print(f' Type shape in PR differs from shapes in release branch \'{release_branch}\' and base branch \'{base_branch}\'') + print (' In release branch:') + print (' Digest:',release_digest) + print (' Shape :',release_shape) + print (' Type :',release_type) + print (' In base branch:') + print (' Digest:',base_digest) + print (' Shape :',base_shape) + print (' Type :',base_type) + print (' In PR branch:') + print (' Digest:',pr_digest) + print (' Shape :',pr_shape) + print (' Type :',pr_type) + set_error() + # an error if the type was deleted in the base branch, added back in PR branch with different shape + # not mentioned in RFC 0047 + if not release_shape is None and base_shape is None: + if not pr_shape==release_shape: + print(f'At path {path}:') + print(f' Type shape in PR differs from shape in release branch \'{release_branch}\' (was deleted in base branch \'{base_branch}\')') + print (' In release branch:') + print (' Digest:',release_digest) + print (' Shape :',release_shape) + print (' Type :',release_type) + print (' In base branch:') + print (' Digest:',base_digest) + print (' Shape :',base_shape) + print (' Type :',base_type) + print (' In PR branch:') + print (' Digest:',pr_digest) + print (' Shape :',pr_shape) + print (' Type :',pr_type) + set_error() + # not an error if the type was introduced in the base branch, and the type changed in PR branch + +if __name__ == "__main__": + if len(sys.argv) != 4 : + print("Usage: %s pr-branch base-branch release-branch" % sys.argv[0], file=sys.stderr) + sys.exit(1) + + pr_branch=sys.argv[1] + base_branch=sys.argv[2] + release_branch=sys.argv[3] + + base_branch_commit=branch_commit(base_branch) + download_s3_type_shapes('base',base_branch,base_branch_commit) + + print('') + + release_branch_commit=branch_commit(release_branch) + download_s3_type_shapes('release',release_branch,release_branch_commit) + + print('') + + pr_branch_commit=branch_commit(pr_branch) + download_s3_type_shapes('pr',pr_branch,pr_branch_commit) + + print('') + + base_type_shape_file=type_shape_file(base_branch_commit) + base_type_dict=make_type_shape_dict(base_type_shape_file) + + release_type_shape_file=type_shape_file(release_branch_commit) + release_type_dict=make_type_shape_dict(release_type_shape_file) + + pr_type_shape_file=type_shape_file(pr_branch_commit) + pr_type_dict=make_type_shape_dict(pr_type_shape_file) + + check_rpc_types(pr_type_dict,release_branch,release_type_dict) + check_command_types(pr_type_dict,release_branch,release_type_dict) + check_type_shapes(pr_type_dict,base_branch,base_type_dict,release_branch,release_type_dict) + + sys.exit(exit_code) diff --git a/src/lib/mina_networking/mina_networking.ml b/src/lib/mina_networking/mina_networking.ml index 0a9a49cd33f..de07a9b1571 100644 --- a/src/lib/mina_networking/mina_networking.ml +++ b/src/lib/mina_networking/mina_networking.ml @@ -732,7 +732,7 @@ module Rpcs = struct ; k_block_hashes_and_timestamps : (State_hash.Stable.V1.t * string) list ; git_commit : string - ; uptime_minutes : int + ; uptime_minutes : int64 } [@@deriving to_yojson, of_yojson] @@ -747,7 +747,7 @@ module Rpcs = struct ; k_block_hashes_and_timestamps = status.k_block_hashes_and_timestamps ; git_commit = status.git_commit - ; uptime_minutes = status.uptime_minutes + ; uptime_minutes = 0 ; block_height_opt = None } end @@ -846,7 +846,7 @@ module Rpcs = struct ; k_block_hashes_and_timestamps = status.k_block_hashes_and_timestamps ; git_commit = status.git_commit - ; uptime_minutes = status.uptime_minutes + ; uptime_minutes = Int64.zero } let caller_model_of_response = function From dacfa0c24af65c686124be4fc1a6bc84f854ddb7 Mon Sep 17 00:00:00 2001 From: Andrew Trainor Date: Thu, 8 Jun 2023 13:34:51 -0400 Subject: [PATCH 2/7] feature/new-version-linter Use new mina-type-shapes bucket to handle type shape files --- buildkite/scripts/build-artifact.sh | 3 +++ scripts/version-linter.py | 15 ++++++++------- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/buildkite/scripts/build-artifact.sh b/buildkite/scripts/build-artifact.sh index 2b020cd8ad5..5fe033b7d87 100755 --- a/buildkite/scripts/build-artifact.sh +++ b/buildkite/scripts/build-artifact.sh @@ -42,6 +42,9 @@ echo "--- Create type shapes git note for commit: ${MINA_COMMIT_SHA1:0:7}" ./_build/default/src/app/cli/src/mina_testnet_signatures.exe \ internal dump-type-shapes --max-depth ${MAX_DEPTH} > ${TYPE_SHAPE_FILE} +echo "--- Uploading ${TYPE_SHAPE_FILE} to mina-type-shapes bucket for consumption by the version linter" +gcloud storage cp ${TYPE_SHAPE_FILE} gs://mina-type-shapes + echo "--- Bundle all packages for Debian ${MINA_DEB_CODENAME}" echo " Includes mina daemon, archive-node, rosetta, generate keypair for berkeley" [[ ${MINA_BUILD_MAINNET} ]] && echo " MINA_BUILD_MAINNET is true so this includes the mainnet and devnet packages for mina-daemon as well" diff --git a/scripts/version-linter.py b/scripts/version-linter.py index e9f72f72f56..1fec56363c6 100755 --- a/scripts/version-linter.py +++ b/scripts/version-linter.py @@ -18,13 +18,14 @@ def branch_commit(branch): capture_output=True) return result.stdout.decode('ascii').replace('"','').replace('\n','') -def download_s3_type_shapes(role,branch,sha1) : - # Andrew : TODO download file - print ('Downloading type shape file for',role,'branch',branch,'at commit',sha1) +def download_type_shapes(role,branch,sha1) : + file=type_shape_file(sha1) + result=subprocess.run(['wget' ,f'https://storage.googleapis.com/mina-type-shapes/{file}']) + print ('Downloading type shape file',file,'for',role,'branch',branch,'at commit',sha1) def type_shape_file(sha1) : # created by buildkite build-artifact script - # loaded to S3 bucket + # loaded to cloud bucket sha1 + '-type-shapes.txt' def make_type_shape_dict(type_shape_file): @@ -165,17 +166,17 @@ def check_type_shapes(pr_branch_dict,base_branch,base_type_dict,release_branch,r release_branch=sys.argv[3] base_branch_commit=branch_commit(base_branch) - download_s3_type_shapes('base',base_branch,base_branch_commit) + download_type_shapes('base',base_branch,base_branch_commit) print('') release_branch_commit=branch_commit(release_branch) - download_s3_type_shapes('release',release_branch,release_branch_commit) + download_type_shapes('release',release_branch,release_branch_commit) print('') pr_branch_commit=branch_commit(pr_branch) - download_s3_type_shapes('pr',pr_branch,pr_branch_commit) + download_type_shapes('pr',pr_branch,pr_branch_commit) print('') From bc54beccae26f05265bd804bdaf65df336255a4e Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 12 Sep 2023 22:03:22 +0200 Subject: [PATCH 3/7] added CI part for pushing shapes to gcloud fix python script with proper return and name for shape file --- buildkite/scripts/build-artifact.sh | 10 ------- buildkite/scripts/cache-through.sh | 10 ++++--- buildkite/scripts/dump-mina-type-shapes.sh | 26 ++++++++++++++++++ buildkite/scripts/version-linter.sh | 23 ++++++++++------ buildkite/src/Jobs/Test/VersionLint.dhall | 29 +++++++++++++++----- scripts/version-linter.py | 31 +++++++++++++++++++--- 6 files changed, 98 insertions(+), 31 deletions(-) create mode 100755 buildkite/scripts/dump-mina-type-shapes.sh diff --git a/buildkite/scripts/build-artifact.sh b/buildkite/scripts/build-artifact.sh index 5fe033b7d87..adb98a79ff8 100755 --- a/buildkite/scripts/build-artifact.sh +++ b/buildkite/scripts/build-artifact.sh @@ -35,16 +35,6 @@ dune build "--profile=${DUNE_PROFILE}" \ src/app/rosetta/rosetta_testnet_signatures.exe \ src/app/test_executive/test_executive.exe # 2>&1 | tee /tmp/buildocaml.log -TYPE_SHAPE_FILE=${MINA_COMMIT_SHA1:0:7}-type_shape.txt -MAX_DEPTH=12 - -echo "--- Create type shapes git note for commit: ${MINA_COMMIT_SHA1:0:7}" -./_build/default/src/app/cli/src/mina_testnet_signatures.exe \ - internal dump-type-shapes --max-depth ${MAX_DEPTH} > ${TYPE_SHAPE_FILE} - -echo "--- Uploading ${TYPE_SHAPE_FILE} to mina-type-shapes bucket for consumption by the version linter" -gcloud storage cp ${TYPE_SHAPE_FILE} gs://mina-type-shapes - echo "--- Bundle all packages for Debian ${MINA_DEB_CODENAME}" echo " Includes mina daemon, archive-node, rosetta, generate keypair for berkeley" [[ ${MINA_BUILD_MAINNET} ]] && echo " MINA_BUILD_MAINNET is true so this includes the mainnet and devnet packages for mina-daemon as well" diff --git a/buildkite/scripts/cache-through.sh b/buildkite/scripts/cache-through.sh index 79fea96b4fc..ffbd1c500eb 100755 --- a/buildkite/scripts/cache-through.sh +++ b/buildkite/scripts/cache-through.sh @@ -3,8 +3,8 @@ set -eou pipefail set +x -if [[ $# -ne 2 ]]; then - echo "Usage: $0 " +if [[ $# -lt 2 ]]; then + echo "Usage: $0 " exit 1 fi @@ -23,9 +23,13 @@ if [[ ! -f $UPLOAD_BIN ]]; then export GOOGLE_APPLICATION_CREDENTIALS=/tmp/gcp_creds.json && /usr/local/google-cloud-sdk/bin/gcloud auth activate-service-account bk-large@o1labs-192920.iam.gserviceaccount.com --key-file /tmp/gcp_creds.json fi -PREFIX=gs://buildkite_k8s/coda/shared FILE="$1" MISS_CMD="$2" +PREFIX="${3:-gs://buildkite_k8s/coda/shared}" + +echo ${PREFIX} + +$UPLOAD_BIN cp "${FILE}" "${PREFIX}${FILE}" set +e if [[ -f "${FILE}" ]] || $UPLOAD_BIN cp "${PREFIX}/${FILE}" .; then diff --git a/buildkite/scripts/dump-mina-type-shapes.sh b/buildkite/scripts/dump-mina-type-shapes.sh new file mode 100755 index 00000000000..c29c6f5e98b --- /dev/null +++ b/buildkite/scripts/dump-mina-type-shapes.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +set -eo pipefail + +# Don't prompt for answers during apt-get install +export DEBIAN_FRONTEND=noninteractive + +apt-get update +apt-get install -y git apt-transport-https ca-certificates tzdata curl + +TESTNET_NAME="berkeley" + +git config --global --add safe.directory /workdir +source buildkite/scripts/export-git-env-vars.sh + +echo "Installing mina daemon package: mina-${TESTNET_NAME}=${MINA_DEB_VERSION}" +echo "deb [trusted=yes] http://packages.o1test.net $MINA_DEB_CODENAME $MINA_DEB_RELEASE" | tee /etc/apt/sources.list.d/mina.list + +apt-get update +apt-get install --allow-downgrades -y "mina-${TESTNET_NAME}=${MINA_DEB_VERSION}" + +MINA_COMMIT_SHA1=$(git log -n 1 --format=%h --abbrev=7 --no-merges) +export TYPE_SHAPE_FILE=${MINA_COMMIT_SHA1}-type_shape.txt + +echo "--- Create type shapes git note for commit: ${MINA_COMMIT_SHA1}" +mina internal dump-type-shapes > ${TYPE_SHAPE_FILE} \ No newline at end of file diff --git a/buildkite/scripts/version-linter.sh b/buildkite/scripts/version-linter.sh index 4907d25128c..960f4c09d2d 100755 --- a/buildkite/scripts/version-linter.sh +++ b/buildkite/scripts/version-linter.sh @@ -7,14 +7,21 @@ if [[ $# -ne 1 ]]; then exit 1 fi -source ~/.profile +# Don't prompt for answers during apt-get install +export DEBIAN_FRONTEND=noninteractive -echo "--- Make build" -export LIBP2P_NIXLESS=1 PATH=/usr/lib/go/bin:$PATH GO=/usr/lib/go/bin/go -time make build +apt-get update +apt-get install -y git apt-transport-https ca-certificates tzdata curl python3 python3-pip wget -base_branch=${BUILDKITE_PULL_REQUEST_BASE_BRANCH} -release_branch=$1 +git config --global --add safe.directory /workdir -echo "--- Run Python version linter" -./scripts/version-linter.py ${base_branch} ${release_branch} +source buildkite/scripts/export-git-env-vars.sh + +pip3 install sexpdata + +base_branch=origin/${BUILDKITE_PULL_REQUEST_BASE_BRANCH} +pr_branch=origin/${BUILDKITE_BRANCH} +release_branch=origin/$1 + +echo "--- Run Python version linter with branches: ${pr_branch} ${base_branch} ${release_branch}" +./scripts/version-linter.py ${pr_branch} ${base_branch} ${release_branch} \ No newline at end of file diff --git a/buildkite/src/Jobs/Test/VersionLint.dhall b/buildkite/src/Jobs/Test/VersionLint.dhall index 91ba63afb5b..fcbdc34f21a 100644 --- a/buildkite/src/Jobs/Test/VersionLint.dhall +++ b/buildkite/src/Jobs/Test/VersionLint.dhall @@ -12,24 +12,41 @@ let RunInToolchain = ../../Command/RunInToolchain.dhall let Docker = ../../Command/Docker/Type.dhall let Size = ../../Command/Size.dhall -let buildTestCmd : Text -> Size -> Command.Type = \(release_branch : Text) -> \(cmd_target : Size) -> + +let dependsOn = [ + { name = "MinaArtifactBullseye", key = "build-deb-pkg" } +] + +in + +let buildTestCmd : Text -> Size -> List Command.TaggedKey.Type -> Command.Type = \(release_branch : Text) -> \(cmd_target : Size) -> \(dependsOn : List Command.TaggedKey.Type) -> Command.build Command.Config::{ - commands = [ Cmd.run "./buildkite/scripts/version-linter.sh ${release_branch}" ], + commands = [ + Cmd.runInDocker + Cmd.Docker::{ + image = (../../Constants/ContainerImages.dhall).ubuntu2004 + } "buildkite/scripts/dump-mina-type-shapes.sh", + Cmd.run "gsutil cp $(git log -n 1 --format=%h --abbrev=7 --no-merges)-type_shape.txt $MINA_TYPE_SHAPE gs://mina-type-shapes", + Cmd.runInDocker + Cmd.Docker::{ + image = (../../Constants/ContainerImages.dhall).ubuntu2004 + } "buildkite/scripts/version-linter.sh ${release_branch}" + ], label = "Versioned type linter", key = "version-linter", target = cmd_target, docker = None Docker.Type, + depends_on = dependsOn, artifact_paths = [ S.contains "core_dumps/*" ] } - in Pipeline.build Pipeline.Config::{ spec = let lintDirtyWhen = [ - S.strictlyStart (S.contains "src/lib"), + S.strictlyStart (S.contains "src"), S.exactly "buildkite/src/Jobs/Test/VersionLint" "dhall", S.exactly "buildkite/scripts/version_linter" "sh" ] @@ -42,6 +59,6 @@ Pipeline.build name = "VersionLint" }, steps = [ - buildTestCmd "develop" Size.Small + buildTestCmd "develop" Size.Small dependsOn ] - } + } \ No newline at end of file diff --git a/scripts/version-linter.py b/scripts/version-linter.py index 1fec56363c6..74c0e44a461 100755 --- a/scripts/version-linter.py +++ b/scripts/version-linter.py @@ -4,29 +4,52 @@ import subprocess import os +import io import sys import re +import sexpdata exit_code=0 +# type shape truncation depth +max_depth = 12 + def set_error(): global exit_code exit_code=1 def branch_commit(branch): + print ('Retrieving', branch, 'head commit...') result=subprocess.run(['git','log','-n','1','--format="%h"','--abbrev=7','--no-merges',f'{branch}'], capture_output=True) - return result.stdout.decode('ascii').replace('"','').replace('\n','') + output=result.stdout.decode('ascii') + print ('command stdout:', output) + print ('command stderr:', result.stderr.decode('ascii')) + return output.replace('"','').replace('\n','') def download_type_shapes(role,branch,sha1) : file=type_shape_file(sha1) - result=subprocess.run(['wget' ,f'https://storage.googleapis.com/mina-type-shapes/{file}']) print ('Downloading type shape file',file,'for',role,'branch',branch,'at commit',sha1) + result=subprocess.run(['wget' ,f'https://storage.googleapis.com/mina-type-shapes/{file}']) def type_shape_file(sha1) : # created by buildkite build-artifact script # loaded to cloud bucket - sha1 + '-type-shapes.txt' + return sha1 + '-type_shape.txt' + +# truncate type shapes to avoid false positives +def truncate_type_shape (sexp) : + def truncate_at_depth (sexp,curr_depth) : + if curr_depth >= max_depth : + return sexpdata.Symbol('.') + else : + if isinstance(sexp,list) : + return list(map(lambda item : truncate_at_depth (item,curr_depth+1),sexp)) + else : + return sexp + fp = io.StringIO() + sexpdata.dump(truncate_at_depth(sexp,0),fp) + return fp.getvalue () def make_type_shape_dict(type_shape_file): shape_dict=dict() @@ -37,7 +60,7 @@ def make_type_shape_dict(type_shape_file): fields=entry.split(', ') path=fields[0] digest=fields[1] - shape=fields[2] + shape=truncate_type_shape(fields[2]) type_=fields[3] shape_dict[path]=[digest,shape,type_] return shape_dict From 3dc8805e1691eea075d0f3affd43f8392ec17ac5 Mon Sep 17 00:00:00 2001 From: dkijania Date: Wed, 13 Sep 2023 09:22:13 +0200 Subject: [PATCH 4/7] Revert "Auxiliary commit to revert individual files from 4ce1c53c0701c9b9c650ad97c18c00ea1bf5bed1" This reverts commit f77d1eef445785cf15b1984948827761858a4805. --- src/lib/mina_networking/mina_networking.ml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lib/mina_networking/mina_networking.ml b/src/lib/mina_networking/mina_networking.ml index de07a9b1571..0a9a49cd33f 100644 --- a/src/lib/mina_networking/mina_networking.ml +++ b/src/lib/mina_networking/mina_networking.ml @@ -732,7 +732,7 @@ module Rpcs = struct ; k_block_hashes_and_timestamps : (State_hash.Stable.V1.t * string) list ; git_commit : string - ; uptime_minutes : int64 + ; uptime_minutes : int } [@@deriving to_yojson, of_yojson] @@ -747,7 +747,7 @@ module Rpcs = struct ; k_block_hashes_and_timestamps = status.k_block_hashes_and_timestamps ; git_commit = status.git_commit - ; uptime_minutes = 0 + ; uptime_minutes = status.uptime_minutes ; block_height_opt = None } end @@ -846,7 +846,7 @@ module Rpcs = struct ; k_block_hashes_and_timestamps = status.k_block_hashes_and_timestamps ; git_commit = status.git_commit - ; uptime_minutes = Int64.zero + ; uptime_minutes = status.uptime_minutes } let caller_model_of_response = function From e1e9925d6ea28fadbe5c4ae51e6e2c3bad94d660 Mon Sep 17 00:00:00 2001 From: dkijania Date: Wed, 13 Sep 2023 09:53:18 +0200 Subject: [PATCH 5/7] trigger build --- src/lib/mina_networking/mina_networking.ml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/mina_networking/mina_networking.ml b/src/lib/mina_networking/mina_networking.ml index 0a9a49cd33f..cd0b0c429bc 100644 --- a/src/lib/mina_networking/mina_networking.ml +++ b/src/lib/mina_networking/mina_networking.ml @@ -35,7 +35,7 @@ type Structured_log_events.t += * - add an entry to the Rpcs.rpc GADT definition for the new module (type ('query, 'response) rpc, below) * - add the new constructor for Rpcs.rpc to Rpcs.all_of_type_erased_rpc * - add a pattern matching case to Rpcs.implementation_of_rpc mapping the - * new constructor to the new module for your RPC + * new constructor to the new module for your RPC * - add a match case to `match_handler`, below *) module Rpcs = struct From de1a02be38c0ff762843af5460e3fbee7ab861b6 Mon Sep 17 00:00:00 2001 From: dkijania Date: Fri, 15 Sep 2023 15:53:28 +0200 Subject: [PATCH 6/7] Revert "Auxiliary commit to revert individual files from 4a62376fd09e511cd1aa670f5090cddc0552099c" This reverts commit 092dec031e23aacb5957ddec3a22bdcfb35a8ba6. --- buildkite/scripts/cache-through.sh | 10 +++------- src/lib/mina_networking/mina_networking.ml | 2 +- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/buildkite/scripts/cache-through.sh b/buildkite/scripts/cache-through.sh index ffbd1c500eb..79fea96b4fc 100755 --- a/buildkite/scripts/cache-through.sh +++ b/buildkite/scripts/cache-through.sh @@ -3,8 +3,8 @@ set -eou pipefail set +x -if [[ $# -lt 2 ]]; then - echo "Usage: $0 " +if [[ $# -ne 2 ]]; then + echo "Usage: $0 " exit 1 fi @@ -23,13 +23,9 @@ if [[ ! -f $UPLOAD_BIN ]]; then export GOOGLE_APPLICATION_CREDENTIALS=/tmp/gcp_creds.json && /usr/local/google-cloud-sdk/bin/gcloud auth activate-service-account bk-large@o1labs-192920.iam.gserviceaccount.com --key-file /tmp/gcp_creds.json fi +PREFIX=gs://buildkite_k8s/coda/shared FILE="$1" MISS_CMD="$2" -PREFIX="${3:-gs://buildkite_k8s/coda/shared}" - -echo ${PREFIX} - -$UPLOAD_BIN cp "${FILE}" "${PREFIX}${FILE}" set +e if [[ -f "${FILE}" ]] || $UPLOAD_BIN cp "${PREFIX}/${FILE}" .; then diff --git a/src/lib/mina_networking/mina_networking.ml b/src/lib/mina_networking/mina_networking.ml index cd0b0c429bc..0a9a49cd33f 100644 --- a/src/lib/mina_networking/mina_networking.ml +++ b/src/lib/mina_networking/mina_networking.ml @@ -35,7 +35,7 @@ type Structured_log_events.t += * - add an entry to the Rpcs.rpc GADT definition for the new module (type ('query, 'response) rpc, below) * - add the new constructor for Rpcs.rpc to Rpcs.all_of_type_erased_rpc * - add a pattern matching case to Rpcs.implementation_of_rpc mapping the - * new constructor to the new module for your RPC + * new constructor to the new module for your RPC * - add a match case to `match_handler`, below *) module Rpcs = struct From 1825063d81b0f03994c7592604b68a4b0d0f282c Mon Sep 17 00:00:00 2001 From: dkijania Date: Fri, 15 Sep 2023 15:54:45 +0200 Subject: [PATCH 7/7] Revert "Auxiliary commit to revert individual files from 73bb4c8a34fc35b824afe9fcf5e8c49138a1ead8" This reverts commit ea6b278fa8e0e1ca4a575839698ef8f0440d937e. --- buildkite/src/Lib/Cmds.dhall | 1 + 1 file changed, 1 insertion(+) diff --git a/buildkite/src/Lib/Cmds.dhall b/buildkite/src/Lib/Cmds.dhall index 6919469fe08..f4acda8c00d 100644 --- a/buildkite/src/Lib/Cmds.dhall +++ b/buildkite/src/Lib/Cmds.dhall @@ -143,3 +143,4 @@ let tests = in module ../Constants/ContainerEnvVars.dhall +