diff --git a/buildkite/scripts/dump-mina-type-shapes.sh b/buildkite/scripts/dump-mina-type-shapes.sh new file mode 100755 index 00000000000..c29c6f5e98b --- /dev/null +++ b/buildkite/scripts/dump-mina-type-shapes.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +set -eo pipefail + +# Don't prompt for answers during apt-get install +export DEBIAN_FRONTEND=noninteractive + +apt-get update +apt-get install -y git apt-transport-https ca-certificates tzdata curl + +TESTNET_NAME="berkeley" + +git config --global --add safe.directory /workdir +source buildkite/scripts/export-git-env-vars.sh + +echo "Installing mina daemon package: mina-${TESTNET_NAME}=${MINA_DEB_VERSION}" +echo "deb [trusted=yes] http://packages.o1test.net $MINA_DEB_CODENAME $MINA_DEB_RELEASE" | tee /etc/apt/sources.list.d/mina.list + +apt-get update +apt-get install --allow-downgrades -y "mina-${TESTNET_NAME}=${MINA_DEB_VERSION}" + +MINA_COMMIT_SHA1=$(git log -n 1 --format=%h --abbrev=7 --no-merges) +export TYPE_SHAPE_FILE=${MINA_COMMIT_SHA1}-type_shape.txt + +echo "--- Create type shapes git note for commit: ${MINA_COMMIT_SHA1}" +mina internal dump-type-shapes > ${TYPE_SHAPE_FILE} \ No newline at end of file diff --git a/buildkite/scripts/version-linter.sh b/buildkite/scripts/version-linter.sh new file mode 100755 index 00000000000..960f4c09d2d --- /dev/null +++ b/buildkite/scripts/version-linter.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +set -eo pipefail + +if [[ $# -ne 1 ]]; then + echo "Usage: $0 " + exit 1 +fi + +# Don't prompt for answers during apt-get install +export DEBIAN_FRONTEND=noninteractive + +apt-get update +apt-get install -y git apt-transport-https ca-certificates tzdata curl python3 python3-pip wget + +git config --global --add safe.directory /workdir + +source buildkite/scripts/export-git-env-vars.sh + +pip3 install sexpdata + +base_branch=origin/${BUILDKITE_PULL_REQUEST_BASE_BRANCH} +pr_branch=origin/${BUILDKITE_BRANCH} +release_branch=origin/$1 + +echo "--- Run Python version linter with branches: ${pr_branch} ${base_branch} ${release_branch}" +./scripts/version-linter.py ${pr_branch} ${base_branch} ${release_branch} \ No newline at end of file diff --git a/buildkite/src/Jobs/Test/VersionLint.dhall b/buildkite/src/Jobs/Test/VersionLint.dhall new file mode 100644 index 00000000000..fcbdc34f21a --- /dev/null +++ b/buildkite/src/Jobs/Test/VersionLint.dhall @@ -0,0 +1,64 @@ +let Prelude = ../../External/Prelude.dhall + +let Cmd = ../../Lib/Cmds.dhall +let S = ../../Lib/SelectFiles.dhall +let D = S.PathPattern + +let Pipeline = ../../Pipeline/Dsl.dhall +let JobSpec = ../../Pipeline/JobSpec.dhall + +let Command = ../../Command/Base.dhall +let RunInToolchain = ../../Command/RunInToolchain.dhall +let Docker = ../../Command/Docker/Type.dhall +let Size = ../../Command/Size.dhall + + +let dependsOn = [ + { name = "MinaArtifactBullseye", key = "build-deb-pkg" } +] + +in + +let buildTestCmd : Text -> Size -> List Command.TaggedKey.Type -> Command.Type = \(release_branch : Text) -> \(cmd_target : Size) -> \(dependsOn : List Command.TaggedKey.Type) -> + Command.build + Command.Config::{ + commands = [ + Cmd.runInDocker + Cmd.Docker::{ + image = (../../Constants/ContainerImages.dhall).ubuntu2004 + } "buildkite/scripts/dump-mina-type-shapes.sh", + Cmd.run "gsutil cp $(git log -n 1 --format=%h --abbrev=7 --no-merges)-type_shape.txt $MINA_TYPE_SHAPE gs://mina-type-shapes", + Cmd.runInDocker + Cmd.Docker::{ + image = (../../Constants/ContainerImages.dhall).ubuntu2004 + } "buildkite/scripts/version-linter.sh ${release_branch}" + ], + label = "Versioned type linter", + key = "version-linter", + target = cmd_target, + docker = None Docker.Type, + depends_on = dependsOn, + artifact_paths = [ S.contains "core_dumps/*" ] + } +in + +Pipeline.build + Pipeline.Config::{ + spec = + let lintDirtyWhen = [ + S.strictlyStart (S.contains "src"), + S.exactly "buildkite/src/Jobs/Test/VersionLint" "dhall", + S.exactly "buildkite/scripts/version_linter" "sh" + ] + + in + + JobSpec::{ + dirtyWhen = lintDirtyWhen, + path = "Test", + name = "VersionLint" + }, + steps = [ + buildTestCmd "develop" Size.Small dependsOn + ] + } \ No newline at end of file diff --git a/scripts/version-linter.py b/scripts/version-linter.py new file mode 100755 index 00000000000..74c0e44a461 --- /dev/null +++ b/scripts/version-linter.py @@ -0,0 +1,219 @@ +#!/usr/bin/env python3 + +# version-linter.py -- makes sure serializations of versioned types don't change + +import subprocess +import os +import io +import sys +import re +import sexpdata + +exit_code=0 + +# type shape truncation depth +max_depth = 12 + +def set_error(): + global exit_code + exit_code=1 + +def branch_commit(branch): + print ('Retrieving', branch, 'head commit...') + result=subprocess.run(['git','log','-n','1','--format="%h"','--abbrev=7','--no-merges',f'{branch}'], + capture_output=True) + output=result.stdout.decode('ascii') + print ('command stdout:', output) + print ('command stderr:', result.stderr.decode('ascii')) + return output.replace('"','').replace('\n','') + +def download_type_shapes(role,branch,sha1) : + file=type_shape_file(sha1) + print ('Downloading type shape file',file,'for',role,'branch',branch,'at commit',sha1) + result=subprocess.run(['wget' ,f'https://storage.googleapis.com/mina-type-shapes/{file}']) + +def type_shape_file(sha1) : + # created by buildkite build-artifact script + # loaded to cloud bucket + return sha1 + '-type_shape.txt' + +# truncate type shapes to avoid false positives +def truncate_type_shape (sexp) : + def truncate_at_depth (sexp,curr_depth) : + if curr_depth >= max_depth : + return sexpdata.Symbol('.') + else : + if isinstance(sexp,list) : + return list(map(lambda item : truncate_at_depth (item,curr_depth+1),sexp)) + else : + return sexp + fp = io.StringIO() + sexpdata.dump(truncate_at_depth(sexp,0),fp) + return fp.getvalue () + +def make_type_shape_dict(type_shape_file): + shape_dict=dict() + with open(type_shape_file) as file : + for entry in file : + if entry=='': + break + fields=entry.split(', ') + path=fields[0] + digest=fields[1] + shape=truncate_type_shape(fields[2]) + type_=fields[3] + shape_dict[path]=[digest,shape,type_] + return shape_dict + +def type_name_from_path(path): + components=path.split('.') + return components[len(components)-1] + +def module_path_from_path(path): + # filename.ml:module_path + components=path.split(':') + mod_path=components[1].split('.') + # modules ... Stable.Vn.t + return mod_path + +def is_signed_command_module_path(modpath): + return (modpath[0]=='Mina_base__Signed_command' and + modpath[1]=='Make_str' and + modpath[2]=='Stable' and + re.match(r'V[0-9]+',modpath[3]) and + modpath[4]=='t') + +def is_zkapp_command_module_path(modpath): + return (modpath[0]=='Mina_base__Zkapp_command' and + modpath[1]=='T' and + modpath[2]=='Stable' and + re.match(r'V[0-9]+',modpath[3]) and + modpath[4]=='t') + +def check_rpc_types(pr_type_dict,release_branch,release_type_dict) : + # every query, response type in release still present + for path in release_type_dict: + type_name=type_name_from_path(path) + if type_name=='query' or type_name=='response': + release_shape=release_type_dict[path] + if path in pr_type_dict: + pr_shape=pr_type_dict[path] + else: + pr_shape=None + if pr_shape is None: + print('RPC type at path',path,f'in release branch ({release_branch}) is missing from PR branch') + set_error() + +def check_command_types(pr_type_dict,release_branch,release_type_dict) : + # every signed command, zkapp command type in release still present + for path in release_type_dict: + module_path=module_path_from_path(path) + if is_signed_command_module_path(module_path) or is_zkapp_command_module_path(module_path): + release_shape=release_type_dict[path] + if path in pr_type_dict: + pr_shape=pr_type_dict[path] + else: + pr_shape=None + if pr_shape is None: + print('Command type at path',path,f'in release branch ({release_branch}) is missing from PR branch') + set_error() + +def check_type_shapes(pr_branch_dict,base_branch,base_type_dict,release_branch,release_type_dict) : + for path in pr_branch_dict: + if path in release_type_dict: + release_info=release_type_dict[path] + else: + release_info=None + if path in base_type_dict: + base_info=base_type_dict[path] + else: + base_info=None + pr_info=pr_branch_dict[path] + # an error if the shape has changed from both the release and base branches + # see RFC 0047 + if not release_info is None and not base_info is None: + # shape digests may differ, even if depth-limited shapes are the same + pr_digest=pr_info[0] + pr_shape=pr_info[1] + pr_type=pr_info[2] + release_digest=release_info[0] + release_shape=release_info[1] + release_type=release_info[2] + base_digest=base_info[0] + base_shape=base_info[1] + base_type=base_info[2] + if not pr_shape==release_shape and not pr_shape==base_shape: + print(f'At path {path}:') + print(f' Type shape in PR differs from shapes in release branch \'{release_branch}\' and base branch \'{base_branch}\'') + print (' In release branch:') + print (' Digest:',release_digest) + print (' Shape :',release_shape) + print (' Type :',release_type) + print (' In base branch:') + print (' Digest:',base_digest) + print (' Shape :',base_shape) + print (' Type :',base_type) + print (' In PR branch:') + print (' Digest:',pr_digest) + print (' Shape :',pr_shape) + print (' Type :',pr_type) + set_error() + # an error if the type was deleted in the base branch, added back in PR branch with different shape + # not mentioned in RFC 0047 + if not release_shape is None and base_shape is None: + if not pr_shape==release_shape: + print(f'At path {path}:') + print(f' Type shape in PR differs from shape in release branch \'{release_branch}\' (was deleted in base branch \'{base_branch}\')') + print (' In release branch:') + print (' Digest:',release_digest) + print (' Shape :',release_shape) + print (' Type :',release_type) + print (' In base branch:') + print (' Digest:',base_digest) + print (' Shape :',base_shape) + print (' Type :',base_type) + print (' In PR branch:') + print (' Digest:',pr_digest) + print (' Shape :',pr_shape) + print (' Type :',pr_type) + set_error() + # not an error if the type was introduced in the base branch, and the type changed in PR branch + +if __name__ == "__main__": + if len(sys.argv) != 4 : + print("Usage: %s pr-branch base-branch release-branch" % sys.argv[0], file=sys.stderr) + sys.exit(1) + + pr_branch=sys.argv[1] + base_branch=sys.argv[2] + release_branch=sys.argv[3] + + base_branch_commit=branch_commit(base_branch) + download_type_shapes('base',base_branch,base_branch_commit) + + print('') + + release_branch_commit=branch_commit(release_branch) + download_type_shapes('release',release_branch,release_branch_commit) + + print('') + + pr_branch_commit=branch_commit(pr_branch) + download_type_shapes('pr',pr_branch,pr_branch_commit) + + print('') + + base_type_shape_file=type_shape_file(base_branch_commit) + base_type_dict=make_type_shape_dict(base_type_shape_file) + + release_type_shape_file=type_shape_file(release_branch_commit) + release_type_dict=make_type_shape_dict(release_type_shape_file) + + pr_type_shape_file=type_shape_file(pr_branch_commit) + pr_type_dict=make_type_shape_dict(pr_type_shape_file) + + check_rpc_types(pr_type_dict,release_branch,release_type_dict) + check_command_types(pr_type_dict,release_branch,release_type_dict) + check_type_shapes(pr_type_dict,base_branch,base_type_dict,release_branch,release_type_dict) + + sys.exit(exit_code)