Skip to content

Commit

Permalink
introduce lightnet suffix for mina lighnet profile. Devnet
Browse files Browse the repository at this point in the history
  • Loading branch information
dkijania committed Sep 23, 2023
1 parent f611438 commit 10f63d5
Show file tree
Hide file tree
Showing 8 changed files with 109 additions and 31 deletions.
4 changes: 3 additions & 1 deletion buildkite/src/Command/DockerImage.dhall
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ let ReleaseSpec = {
deb_codename: Text,
deb_release: Text,
deb_version: Text,
deb_profile: Text,
extra_args: Text,
step_key: Text
},
Expand All @@ -30,6 +31,7 @@ let ReleaseSpec = {
service = "\\\${MINA_SERVICE}",
branch = "\\\${BUILDKITE_BRANCH}",
deb_codename = "bullseye",
deb_profile = "devnet",
deb_release = "\\\${MINA_DEB_RELEASE}",
deb_version = "\\\${MINA_DEB_VERSION}",
extra_args = "",
Expand All @@ -43,7 +45,7 @@ let generateStep = \(spec : ReleaseSpec.Type) ->
[
Cmd.run (
"export MINA_DEB_CODENAME=${spec.deb_codename} && source ./buildkite/scripts/export-git-env-vars.sh && ./scripts/release-docker.sh " ++
"--service ${spec.service} --version ${spec.version} --network ${spec.network} --branch ${spec.branch} --deb-codename ${spec.deb_codename} --deb-release ${spec.deb_release} --deb-version ${spec.deb_version} --extra-args \\\"${spec.extra_args}\\\""
"--service ${spec.service} --version ${spec.version} --network ${spec.network} --branch ${spec.branch} --deb-codename ${spec.deb_codename} --deb-release ${spec.deb_release} --deb-version ${spec.deb_version} --deb-profile ${spec.deb_profile} --extra-args \\\"${spec.extra_args}\\\""
)
]

Expand Down
46 changes: 25 additions & 21 deletions buildkite/src/Command/MinaArtifact.dhall
Original file line number Diff line number Diff line change
Expand Up @@ -13,31 +13,32 @@ let Size = ./Size.dhall
let Libp2p = ./Libp2pHelperBuild.dhall
let DockerImage = ./DockerImage.dhall
let DebianVersions = ../Constants/DebianVersions.dhall
let Profiles = ../Constants/Profiles.dhall

in

let pipeline : DebianVersions.DebVersion -> Pipeline.Config.Type = \(debVersion : DebianVersions.DebVersion) ->
let pipeline : DebianVersions.DebVersion -> Profiles.ProfileName -> Pipeline.Config.Type = \(debVersion : DebianVersions.DebVersion) ->
\(profile: Profiles.ProfileName) ->
Pipeline.Config::{
spec =
JobSpec::{
dirtyWhen = DebianVersions.dirtyWhen debVersion,
path = "Release",
name = "MinaArtifact${DebianVersions.capitalName debVersion}"
name = "MinaArtifact${DebianVersions.capitalName debVersion}${Profiles.capitalName profile}"
},
steps = [
Libp2p.step debVersion,
Command.build
Command.Config::{
commands =
DebianVersions.toolchainRunner debVersion [
"DUNE_PROFILE=devnet",
"AWS_ACCESS_KEY_ID",
"AWS_SECRET_ACCESS_KEY",
"MINA_BRANCH=$BUILDKITE_BRANCH",
"MINA_COMMIT_SHA1=$BUILDKITE_COMMIT",
"MINA_DEB_CODENAME=${DebianVersions.lowerName debVersion}"
] "./buildkite/scripts/build-artifact.sh",
label = "Build Mina for ${DebianVersions.capitalName debVersion}",
commands = DebianVersions.toolchainRunner debVersion [
"DUNE_PROFILE=${Profiles.lowerName profile}",
"AWS_ACCESS_KEY_ID",
"AWS_SECRET_ACCESS_KEY",
"MINA_BRANCH=$BUILDKITE_BRANCH",
"MINA_COMMIT_SHA1=$BUILDKITE_COMMIT",
"MINA_DEB_CODENAME=${DebianVersions.lowerName debVersion}"
] "./buildkite/scripts/build-artifact.sh",
label = "Build Mina for ${DebianVersions.capitalName debVersion} ${Profiles.capitalName profile}",
key = "build-deb-pkg",
target = Size.XLarge,
retries = [
Expand All @@ -53,7 +54,8 @@ let pipeline : DebianVersions.DebVersion -> Pipeline.Config.Type = \(debVersion
service="mina-daemon",
network="berkeley",
deb_codename="${DebianVersions.lowerName debVersion}",
step_key="daemon-berkeley-${DebianVersions.lowerName debVersion}-docker-image"
deb_profile="${Profiles.lowerName profile}",
step_key="daemon-berkeley-${DebianVersions.lowerName debVersion}-${Profiles.lowerName profile}-docker-image"
}

in
Expand All @@ -65,7 +67,7 @@ let pipeline : DebianVersions.DebVersion -> Pipeline.Config.Type = \(debVersion
deps=DebianVersions.dependsOn debVersion,
service="mina-test-executive",
deb_codename="${DebianVersions.lowerName debVersion}",
step_key="test-executive-${DebianVersions.lowerName debVersion}-docker-image"
step_key="test-executive-${DebianVersions.lowerName debVersion}-${Profiles.lowerName profile}-docker-image"
}
in
DockerImage.generateStep testExecutiveSpec,
Expand All @@ -76,7 +78,7 @@ let pipeline : DebianVersions.DebVersion -> Pipeline.Config.Type = \(debVersion
service="mina-batch-txn",
network="berkeley",
deb_codename="${DebianVersions.lowerName debVersion}",
step_key="batch-txn-${DebianVersions.lowerName debVersion}-docker-image"
step_key="batch-txn-${DebianVersions.lowerName debVersion}-${Profiles.lowerName profile}-docker-image"
}
in
DockerImage.generateStep batchTxnSpec,
Expand All @@ -86,7 +88,8 @@ let pipeline : DebianVersions.DebVersion -> Pipeline.Config.Type = \(debVersion
deps=DebianVersions.dependsOn debVersion,
service="mina-archive",
deb_codename="${DebianVersions.lowerName debVersion}",
step_key="archive-${DebianVersions.lowerName debVersion}-docker-image"
deb_profile="${Profiles.lowerName profile}",
step_key="archive-${DebianVersions.lowerName debVersion}-${Profiles.lowerName profile}-docker-image"
}
in
DockerImage.generateStep archiveSpec,
Expand All @@ -96,7 +99,7 @@ let pipeline : DebianVersions.DebVersion -> Pipeline.Config.Type = \(debVersion
service="mina-rosetta",
extra_args="--build-arg MINA_BRANCH=\\\${BUILDKITE_BRANCH} --cache-from ${DebianVersions.toolchainImage debVersion}",
deb_codename="${DebianVersions.lowerName debVersion}",
step_key="rosetta-${DebianVersions.lowerName debVersion}-docker-image"
step_key="rosetta-${DebianVersions.lowerName debVersion}-${Profiles.lowerName profile}-docker-image"
}
in

Expand All @@ -107,7 +110,7 @@ let pipeline : DebianVersions.DebVersion -> Pipeline.Config.Type = \(debVersion
deps=DebianVersions.dependsOn debVersion,
service="mina-zkapp-test-transaction",
deb_codename="${DebianVersions.lowerName debVersion}",
step_key="zkapp-test-transaction-${DebianVersions.lowerName debVersion}-docker-image"
step_key="zkapp-test-transaction-${DebianVersions.lowerName debVersion}-${Profiles.lowerName profile}-docker-image"
}

in
Expand All @@ -119,7 +122,8 @@ let pipeline : DebianVersions.DebVersion -> Pipeline.Config.Type = \(debVersion

in
{
bullseye = pipeline DebianVersions.DebVersion.Bullseye
, buster = pipeline DebianVersions.DebVersion.Buster
, focal = pipeline DebianVersions.DebVersion.Focal
bullseye = pipeline DebianVersions.DebVersion.Bullseye Profiles.ProfileName.Devnet
, bullseye-lightnet = pipeline DebianVersions.DebVersion.Bullseye Profiles.ProfileName.Lightnet
, buster = pipeline DebianVersions.DebVersion.Buster Profiles.ProfileName.Devnet
, focal = pipeline DebianVersions.DebVersion.Focal Profiles.ProfileName.Devnet
}
5 changes: 4 additions & 1 deletion buildkite/src/Constants/DebianVersions.dhall
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,10 @@ let toolchainImage = \(debVersion : DebVersion) ->
let dependsOn = \(debVersion : DebVersion) ->
merge {
Bookworm = [{ name = "MinaArtifactBookworm", key = "build-deb-pkg" }]
, Bullseye = [{ name = "MinaArtifactBullseye", key = "build-deb-pkg" }]
, Bullseye = [
{ name = "MinaArtifactBullseye", key = "build-deb-pkg" },
{ name = "MinaArtifactBullseyeLightnet", key = "build-deb-pkg" }
]
, Buster = [{ name = "MinaArtifactBuster", key = "build-deb-pkg" }]
, Jammy = [{ name = "MinaArtifactJammy", key = "build-deb-pkg" }]
, Focal = [{ name = "MinaArtifactFocal", key = "build-deb-pkg" }]
Expand Down
25 changes: 25 additions & 0 deletions buildkite/src/Constants/Profiles.dhall
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
let Prelude = ../External/Prelude.dhall

let ProfileName = < Devnet | Lightnet | Hardfork >

let capitalName = \(profileName : ProfileName) ->
merge {
Devnet = "Devnet"
, Lightnet = "Lightnet"
, Hardfork = "Hardfork"
} profileName

let lowerName = \(profileName : ProfileName) ->
merge {
Devnet = "devnet"
, Lightnet = "lightnet"
, Hardfork = "hardfork"
} profileName

in

{
ProfileName = ProfileName
, capitalName = capitalName
, lowerName = lowerName
}
6 changes: 6 additions & 0 deletions buildkite/src/Jobs/Release/MinaArtifactBullseyeLightnet.dhall
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
let ArtifactPipelines = ../../Command/MinaArtifact.dhall
let Pipeline = ../../Pipeline/Dsl.dhall

in

Pipeline.build ArtifactPipelines.bullseye-lightnet
16 changes: 15 additions & 1 deletion scripts/archive/build-release-archives.sh
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,21 @@ esac

echo "--- Building Mina archive package"

PROJECT="mina-archive"

# Determina suffix for mina name. To preserve backward compatibility devnet profile won't have any suffix.
# For others profiles we are adding suffix. For example '-lightnet'
case "${DUNE_PROFILE}" in
devnet)
PROJECT_SUFFIX=""
;;
*)
PROJECT_SUFFIX="-${DUNE_PROFILE}"
;;
esac



PROJECT="mina-archive${PROJECT_SUFFIX}"
BUILD_DIR="deb_build"

###### archiver deb
Expand Down
13 changes: 10 additions & 3 deletions scripts/rebuild-deb.sh
Original file line number Diff line number Diff line change
Expand Up @@ -45,10 +45,17 @@ case "${MINA_DEB_CODENAME}" in
;;
esac

case "${DUNE_PROFILE}" in
devnet)
MINA_DEB_NAME="mina-berkeley"
;;
*)
MINA_DEB_NAME="mina-berkeley-${DUNE_PROFILE}"
;;
esac

BUILDDIR="deb_build"


# Function to ease creation of Debian package control files
create_control_file() {

Expand Down Expand Up @@ -266,11 +273,11 @@ echo "------------------------------------------------------------"
echo "--- Building Mina Berkeley testnet signatures deb without keys:"

mkdir -p "${BUILDDIR}/DEBIAN"
create_control_file mina-berkeley "${SHARED_DEPS}${DAEMON_DEPS}" 'Mina Protocol Client and Daemon'
create_control_file "${MINA_DEB_NAME}" "${SHARED_DEPS}${DAEMON_DEPS}" 'Mina Protocol Client and Daemon'

copy_common_daemon_configs berkeley testnet 'seed-lists/berkeley_seeds.txt'

build_deb mina-berkeley
build_deb "${MINA_DEB_NAME}"

##################################### END BERKELEY PACKAGE #######################################

Expand Down
25 changes: 21 additions & 4 deletions scripts/release-docker.sh
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ function usage() {
echo " --deb-codename The debian codename (stretch or buster) to build the docker image from. Default=stretch"
echo " --deb-release The debian package release channel to pull from (unstable,alpha,beta,stable). Default=unstable"
echo " --deb-version The version string for the debian package to install"
echo " --deb-profile The profile string for the debian package to install"
echo ""
echo "Example: $0 --service faucet --version v0.1.0"
echo "Valid Services: ${VALID_SERVICES[*]}"
Expand All @@ -40,6 +41,7 @@ while [[ "$#" -gt 0 ]]; do case $1 in
--deb-codename) DEB_CODENAME="--build-arg deb_codename=$2"; shift;;
--deb-release) DEB_RELEASE="--build-arg deb_release=$2"; shift;;
--deb-version) DEB_VERSION="--build-arg deb_version=$2"; shift;;
--deb-profile) DEB_PROFILE="$2"; shift;;
--extra-args) EXTRA=${@:2}; shift $((${#}-1));;
*) echo "Unknown parameter passed: $1"; exit 1;;
esac; shift; done
Expand All @@ -55,9 +57,21 @@ case "${DEB_CODENAME##*=}" in
esac
IMAGE="--build-arg image=${IMAGE}"

# Determina suffix for mina name. To preserve backward compatibility devnet profile won't have any suffix.
# For others profiles we are adding suffix. For example '-lightnet'
case "${DEB_PROFILE}" in
devnet)
SERVICE_SUFFIX=""
;;
*)
SERVICE_SUFFIX="-${DEB_PROFILE}"
;;
esac


# Debug prints for visability
# Substring removal to cut the --build-arg arguments on the = so that the output is exactly the input flags https://wiki.bash-hackers.org/syntax/pe#substring_removal
echo "--service ${SERVICE} --version ${VERSION} --branch ${BRANCH##*=} --deb-version ${DEB_VERSION##*=} --deb-release ${DEB_RELEASE##*=} --deb-codename ${DEB_CODENAME##*=}"
echo "--service ${SERVICE} --version ${VERSION} --branch ${BRANCH##*=} --deb-version ${DEB_VERSION##*=} --deb-profile ${DEB_PROFILE##*=} --deb-release ${DEB_RELEASE##*=} --deb-codename ${DEB_CODENAME##*=}"
echo ${EXTRA}
echo "docker image: ${IMAGE}"

Expand All @@ -71,6 +85,7 @@ case "${SERVICE}" in
mina-archive)
DOCKERFILE_PATH="dockerfiles/Dockerfile-mina-archive"
DOCKER_CONTEXT="dockerfiles/"
SERVICE=${SERVICE}${SERVICE_SUFFIX}
;;
bot)
DOCKERFILE_PATH="frontend/bot/Dockerfile"
Expand All @@ -80,6 +95,7 @@ mina-daemon)
DOCKERFILE_PATH="dockerfiles/Dockerfile-mina-daemon"
DOCKER_CONTEXT="dockerfiles/"
VERSION="${VERSION}-${NETWORK##*=}"
SERVICE=${SERVICE}${SERVICE_SUFFIX}
;;
mina-toolchain)
DOCKERFILE_PATH="dockerfiles/stages/1-build-deps dockerfiles/stages/2-opam-deps dockerfiles/stages/3-toolchain"
Expand Down Expand Up @@ -119,7 +135,6 @@ itn-orchestrator)

esac


REPO="--build-arg MINA_REPO=${BUILDKITE_PULL_REQUEST_REPO}"
if [[ -z "${BUILDKITE_PULL_REQUEST_REPO}" ]]; then
REPO="--build-arg MINA_REPO=https://github.com/MinaProtocol/mina"
Expand All @@ -131,12 +146,14 @@ TAG="${DOCKER_REGISTRY}/${SERVICE}:${VERSION}"
GITHASH=$(git rev-parse --short=7 HEAD)
HASHTAG="${DOCKER_REGISTRY}/${SERVICE}:${GITHASH}-${DEB_CODENAME##*=}-${NETWORK##*=}"

DEB_PROFILE="--build-arg deb_profile=${DEB_PROFILE}"

# If DOCKER_CONTEXT is not specified, assume none and just pipe the dockerfile into docker build
extra_build_args=$(echo ${EXTRA} | tr -d '"')
if [[ -z "${DOCKER_CONTEXT}" ]]; then
cat $DOCKERFILE_PATH | docker build $CACHE $NETWORK $IMAGE $DEB_CODENAME $DEB_RELEASE $DEB_VERSION $BRANCH $REPO $extra_build_args -t "$TAG" -
cat $DOCKERFILE_PATH | docker build $CACHE $NETWORK $IMAGE $DEB_CODENAME $DEB_RELEASE $DEB_VERSION $DEB_PROFILE $BRANCH $REPO $extra_build_args -t "$TAG" -
else
docker build $CACHE $NETWORK $IMAGE $DEB_CODENAME $DEB_RELEASE $DEB_VERSION $BRANCH $REPO $extra_build_args $DOCKER_CONTEXT -t "$TAG" -f $DOCKERFILE_PATH
docker build $CACHE $NETWORK $IMAGE $DEB_CODENAME $DEB_RELEASE $DEB_VERSION $DEB_PROFILE $BRANCH $REPO $extra_build_args $DOCKER_CONTEXT -t "$TAG" -f $DOCKERFILE_PATH
fi

if [[ -z "$NOUPLOAD" ]] || [[ "$NOUPLOAD" -eq 0 ]]; then
Expand Down

0 comments on commit 10f63d5

Please sign in to comment.