extract profile settings and build docker based on its values

buildkite/scripts/run-test-executive.sh

@@ -2,8 +2,8 @@
 set -o pipefail -x
 
 TEST_NAME="$1"
-MINA_IMAGE="gcr.io/o1labs-192920/mina-daemon:$MINA_DOCKER_TAG-berkeley"
-ARCHIVE_IMAGE="gcr.io/o1labs-192920/mina-archive:$MINA_DOCKER_TAG"
+MINA_IMAGE="gcr.io/o1labs-192920/mina-daemon-lightnet:$MINA_DOCKER_TAG-berkeley"
+ARCHIVE_IMAGE="gcr.io/o1labs-192920/mina-archive-lightnet:$MINA_DOCKER_TAG"
 
 if [[ "${TEST_NAME:0:15}" == "block-prod-prio" ]] && [[ "$RUN_OPT_TESTS" == "" ]]; then
   echo "Skipping $TEST_NAME"

buildkite/src/Command/DockerImage.dhall

@@ -20,6 +20,7 @@ let ReleaseSpec = {
     deb_codename: Text,
     deb_release: Text,
     deb_version: Text,
+    deb_profile: Text,
     extra_args: Text,
     step_key: Text
   },
@@ -30,6 +31,7 @@ let ReleaseSpec = {
     service = "\\\${MINA_SERVICE}",
     branch = "\\\${BUILDKITE_BRANCH}",
     deb_codename = "bullseye",
+    deb_profile = "devnet",
     deb_release = "\\\${MINA_DEB_RELEASE}",
     deb_version = "\\\${MINA_DEB_VERSION}",
     extra_args = "",
@@ -43,7 +45,7 @@ let generateStep = \(spec : ReleaseSpec.Type) ->
     [
         Cmd.run (
           "export MINA_DEB_CODENAME=${spec.deb_codename} && source ./buildkite/scripts/export-git-env-vars.sh && ./scripts/release-docker.sh " ++
-              "--service ${spec.service} --version ${spec.version} --network ${spec.network} --branch ${spec.branch} --deb-codename ${spec.deb_codename} --deb-release ${spec.deb_release} --deb-version ${spec.deb_version} --extra-args \\\"${spec.extra_args}\\\""
+              "--service ${spec.service} --version ${spec.version} --network ${spec.network} --branch ${spec.branch} --deb-codename ${spec.deb_codename} --deb-release ${spec.deb_release} --deb-version ${spec.deb_version} --deb-profile ${spec.deb_profile} --extra-args \\\"${spec.extra_args}\\\""
         )
     ]
 

buildkite/src/Command/MinaArtifact.dhall

@@ -13,30 +13,32 @@ let Size = ./Size.dhall
 let Libp2p = ./Libp2pHelperBuild.dhall
 let DockerImage = ./DockerImage.dhall
 let DebianVersions = ../Constants/DebianVersions.dhall
+let Profiles = ../Constants/Profiles.dhall
 
 in
 
-let pipeline : DebianVersions.DebVersion -> Pipeline.Config.Type = \(debVersion : DebianVersions.DebVersion) ->
+let pipeline : DebianVersions.DebVersion -> Profiles.ProfileName -> Pipeline.Config.Type = \(debVersion : DebianVersions.DebVersion) ->
+  \(profile: Profiles.ProfileName) ->
     Pipeline.Config::{
       spec =
         JobSpec::{
           dirtyWhen = DebianVersions.dirtyWhen debVersion,
           path = "Release",
-          name = "MinaArtifact${DebianVersions.capitalName debVersion}"
+          name = "MinaArtifact${DebianVersions.capitalName debVersion}${Profiles.capitalName profile}"
         },
       steps = [
         Libp2p.step debVersion,
         Command.build
           Command.Config::{
             commands = DebianVersions.toolchainRunner debVersion [
-              "DUNE_PROFILE=devnet",
+              "DUNE_PROFILE=${Profiles.lowerName profile}",
               "AWS_ACCESS_KEY_ID",
               "AWS_SECRET_ACCESS_KEY",
               "MINA_BRANCH=$BUILDKITE_BRANCH",
               "MINA_COMMIT_SHA1=$BUILDKITE_COMMIT",
               "MINA_DEB_CODENAME=${DebianVersions.lowerName debVersion}"
             ] "./buildkite/scripts/build-artifact.sh",
-            label = "Build Mina for ${DebianVersions.capitalName debVersion}",
+            label = "Build Mina for ${DebianVersions.capitalName debVersion} ${Profiles.capitalName profile}",
             key = "build-deb-pkg",
             target = Size.XLarge,
             retries = [
@@ -52,7 +54,8 @@ let pipeline : DebianVersions.DebVersion -> Pipeline.Config.Type = \(debVersion
           service="mina-daemon",
           network="berkeley",
           deb_codename="${DebianVersions.lowerName debVersion}",
-          step_key="daemon-berkeley-${DebianVersions.lowerName debVersion}-docker-image"
+          deb_profile="${Profiles.lowerName profile}",
+          step_key="daemon-berkeley-${DebianVersions.lowerName debVersion}-${Profiles.lowerName profile}-docker-image"
         }
 
         in
@@ -64,7 +67,7 @@ let pipeline : DebianVersions.DebVersion -> Pipeline.Config.Type = \(debVersion
           deps=DebianVersions.dependsOn debVersion,
           service="mina-test-executive",
           deb_codename="${DebianVersions.lowerName debVersion}",
-          step_key="test-executive-${DebianVersions.lowerName debVersion}-docker-image"
+          step_key="test-executive-${DebianVersions.lowerName debVersion}-${Profiles.lowerName profile}-docker-image"
         }
         in
         DockerImage.generateStep testExecutiveSpec,
@@ -74,7 +77,8 @@ let pipeline : DebianVersions.DebVersion -> Pipeline.Config.Type = \(debVersion
           deps=DebianVersions.dependsOn debVersion,
           service="mina-archive",
           deb_codename="${DebianVersions.lowerName debVersion}",
-          step_key="archive-${DebianVersions.lowerName debVersion}-docker-image"
+          deb_profile="${Profiles.lowerName profile}",
+          step_key="archive-${DebianVersions.lowerName debVersion}-${Profiles.lowerName profile}-docker-image"
         }
         in
         DockerImage.generateStep archiveSpec,
@@ -84,7 +88,7 @@ let pipeline : DebianVersions.DebVersion -> Pipeline.Config.Type = \(debVersion
           service="mina-rosetta",
           extra_args="--build-arg MINA_BRANCH=\\\${BUILDKITE_BRANCH} --cache-from ${DebianVersions.toolchainImage debVersion}",
           deb_codename="${DebianVersions.lowerName debVersion}",
-          step_key="rosetta-${DebianVersions.lowerName debVersion}-docker-image"
+          step_key="rosetta-${DebianVersions.lowerName debVersion}-${Profiles.lowerName profile}-docker-image"
         }
         in
 
@@ -95,7 +99,7 @@ let pipeline : DebianVersions.DebVersion -> Pipeline.Config.Type = \(debVersion
           deps=DebianVersions.dependsOn debVersion,
           service="mina-zkapp-test-transaction",
           deb_codename="${DebianVersions.lowerName debVersion}",
-          step_key="zkapp-test-transaction-${DebianVersions.lowerName debVersion}-docker-image"
+          step_key="zkapp-test-transaction-${DebianVersions.lowerName debVersion}-${Profiles.lowerName profile}-docker-image"
         }
 
         in
@@ -107,7 +111,8 @@ let pipeline : DebianVersions.DebVersion -> Pipeline.Config.Type = \(debVersion
 
 in
 {
-  bullseye  = pipeline DebianVersions.DebVersion.Bullseye
-  , buster  = pipeline DebianVersions.DebVersion.Buster
-  , focal   = pipeline DebianVersions.DebVersion.Focal
+  bullseye  = pipeline DebianVersions.DebVersion.Bullseye Profiles.ProfileName.Devnet
+  , bullseye-lighnet  = pipeline DebianVersions.DebVersion.Bullseye Profiles.ProfileName.Lightnet
+  , buster  = pipeline DebianVersions.DebVersion.Buster Profiles.ProfileName.Devnet
+  , focal   = pipeline DebianVersions.DebVersion.Focal Profiles.ProfileName.Devnet
 }

buildkite/src/Constants/DebianVersions.dhall

@@ -48,11 +48,14 @@ let toolchainImage = \(debVersion : DebVersion) ->
 
 let dependsOn = \(debVersion : DebVersion) ->
   merge {
-    Bookworm = [{ name = "MinaArtifactBookworm", key = "build-deb-pkg" }]
-    , Bullseye = [{ name = "MinaArtifactBullseye", key = "build-deb-pkg" }]
-    , Buster = [{ name = "MinaArtifactBuster", key = "build-deb-pkg" }]
-    , Jammy = [{ name = "MinaArtifactJammy", key = "build-deb-pkg" }]
-    , Focal = [{ name = "MinaArtifactFocal", key = "build-deb-pkg" }]
+    Bookworm = [{ name = "MinaArtifactBookwormDevnet", key = "build-deb-pkg" }]
+    , Bullseye = [
+      { name = "MinaArtifactBullseyeDevnet", key = "build-deb-pkg" },
+      { name = "MinaArtifactBullseyeLightnet", key = "build-deb-pkg" }
+    ]
+    , Buster = [{ name = "MinaArtifactBusterDevnet", key = "build-deb-pkg" }]
+    , Jammy = [{ name = "MinaArtifactJammyDevnet", key = "build-deb-pkg" }]
+    , Focal = [{ name = "MinaArtifactFocalDevnet", key = "build-deb-pkg" }]
   } debVersion
 
 -- Most debian builds are only used for public releases

buildkite/src/Constants/Profiles.dhall

@@ -0,0 +1,25 @@
+let Prelude = ../External/Prelude.dhall
+
+let ProfileName = < Devnet | Lightnet | Hardfork >
+
+let capitalName = \(profileName : ProfileName) ->
+  merge {
+    Devnet = "Devnet"
+    , Lightnet = "Lightnet"
+    , Hardfork = "Hardfork"
+  } profileName
+
+let lowerName = \(profileName : ProfileName) ->
+  merge {
+    Devnet = "devnet"
+    , Lightnet = "lightnet"
+    , Hardfork = "hardfork"
+  } profileName
+
+in
+
+{
+  ProfileName = ProfileName
+  , capitalName = capitalName
+  , lowerName = lowerName
+}

buildkite/src/Jobs/Release/MinaArtifactBullseyeLightnet.dhall

@@ -0,0 +1,6 @@
+let ArtifactPipelines = ../../Command/MinaArtifact.dhall
+let Pipeline = ../../Pipeline/Dsl.dhall
+
+in
+
+Pipeline.build ArtifactPipelines.bullseye-lighnet

buildkite/src/Jobs/Test/CheckGraphQLSchema.dhall

@@ -6,7 +6,7 @@ let Pipeline = ../../Pipeline/Dsl.dhall
 let CheckGraphQLSchema = ../../Command/CheckGraphQLSchema.dhall
 
 let dependsOn = [
-    { name = "MinaArtifactBullseye", key = "build-deb-pkg" }
+    { name = "MinaArtifactBullseyeDevnet", key = "build-deb-pkg" }
 ]
 
 in Pipeline.build Pipeline.Config::{

buildkite/src/Jobs/Test/ConnectToBerkeley.dhall

@@ -6,7 +6,7 @@ let Pipeline = ../../Pipeline/Dsl.dhall
 let ConnectToTestnet = ../../Command/ConnectToTestnet.dhall
 
 let dependsOn = [
-  { name = "MinaArtifactBullseye", key = "daemon-berkeley-bullseye-docker-image" }
+  { name = "MinaArtifactBullseyeDevnet", key = "daemon-berkeley-bullseye-devnet-docker-image" }
 ]
 
 in Pipeline.build Pipeline.Config::{

buildkite/src/Jobs/Test/ReplayerTest.dhall.SUSPENDED

@@ -7,7 +7,7 @@ let JobSpec = ../../Pipeline/JobSpec.dhall
 let ReplayerTest = ../../Command/ReplayerTest.dhall
 
 let dependsOn =
-      [ { name = "MinaArtifactBullseye", key = "archive-bullseye-docker-image" } ]
+      [ { name = "MinaArtifactBullseyeDevnet", key = "archive-bullseye-devnet-docker-image" } ]
 
 in  Pipeline.build
       Pipeline.Config::{

buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall

@@ -42,7 +42,7 @@ Pipeline.build
           label = "Rosetta integration tests Bullseye"
           , key = "rosetta-integration-tests-bullseye"
           , target = Size.Small
-          , depends_on = [ { name = "MinaArtifactBullseye", key = "rosetta-bullseye-docker-image" } ]
+          , depends_on = [ { name = "MinaArtifactBullseyeDevnet", key = "rosetta-bullseye-devnet-docker-image" } ]
           , soft_fail = Some (B/SoftFail.Boolean True)
         }
     ]

buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall

@@ -7,14 +7,14 @@ let TestExecutive = ../../Command/TestExecutive.dhall
 
 let dependsOn = [
     { name = "TestnetIntegrationTests", key = "build-test-executive" },
-    { name = "MinaArtifactBullseye", key = "daemon-berkeley-bullseye-docker-image" },
-    { name = "MinaArtifactBullseye", key = "archive-bullseye-docker-image" }
+    { name = "MinaArtifactBullseyeLightnet", key = "daemon-berkeley-bullseye-lightnet-docker-image" },
+    { name = "MinaArtifactBullseyeLightnet", key = "archive-bullseye-lightnet-docker-image" }
 ]
 let dependsOnJs = [
     { name = "TestnetIntegrationTests", key = "build-test-executive" },
     { name = "TestnetIntegrationTests", key = "build-js-tests" },
-    { name = "MinaArtifactBullseye", key = "daemon-berkeley-bullseye-docker-image" },
-    { name = "MinaArtifactBullseye", key = "archive-bullseye-docker-image" }
+    { name = "MinaArtifactBullseyeLightnet", key = "daemon-berkeley-bullseye-lightnet-docker-image" },
+    { name = "MinaArtifactBullseyeLightnet", key = "archive-bullseye-lightnet-docker-image" }
 ]
 
 in Pipeline.build Pipeline.Config::{

dockerfiles/Dockerfile-mina-archive

@@ -5,6 +5,7 @@ FROM ${image}
 ARG deb_version
 ARG deb_codename=bullseye
 ARG deb_release=unstable
+ARG deb_profile=devnet
 
 ENV DEBIAN_FRONTEND noninteractive
 RUN echo "Building image with version $deb_codename $deb_release $deb_version"
@@ -38,7 +39,7 @@ RUN mkdir /healthcheck && curl https://raw.githubusercontent.com/MinaProtocol/mi
 # archive-node package
 RUN echo "deb [trusted=yes] http://packages.o1test.net $deb_codename $deb_release" > /etc/apt/sources.list.d/o1.list \
   && apt-get update --quiet --yes \
-  && apt-get install --quiet --yes "mina-archive=$deb_version" \
+  && apt-get install --quiet --yes "mina-archive-$deb_profile=$deb_version" \
   && rm -rf /var/lib/apt/lists/*
 
 ENTRYPOINT ["/usr/bin/dumb-init", "/entrypoint.sh"]

dockerfiles/Dockerfile-mina-daemon

@@ -4,6 +4,7 @@ FROM ${image}
 ARG deb_version
 ARG deb_release=unstable
 ARG deb_codename=bullseye
+ARG deb_profile=devnet
 ARG network=mainnet
 
 ENV DEBIAN_FRONTEND noninteractive
@@ -59,7 +60,7 @@ ENV USE_GKE_GCLOUD_AUTH_PLUGIN=True
 RUN echo "Building image with version $deb_version from repo $deb_release $deb_codename for network $network" \
   && echo "deb [trusted=yes] http://packages.o1test.net $deb_codename $deb_release" > /etc/apt/sources.list.d/o1.list \
   && apt-get update --quiet --yes \
-  && apt-get install --quiet --yes --allow-downgrades "mina-$network=$deb_version" \
+  && apt-get install --quiet --yes --allow-downgrades "mina-$network-$deb_profile=$deb_version" \
   && rm -rf /var/lib/apt/lists/*
 
 

dockerfiles/Dockerfile-mina-test-executive

@@ -4,6 +4,8 @@ FROM ${image}
 ARG deb_version
 ARG deb_release=unstable
 ARG deb_codename=bullseye
+ARG deb_profile=devnet
+ARG profile=devnet
 ARG MINA_BRANCH
 ARG MINA_REPO=https://github.com/MinaProtocol/mina
 

scripts/archive/build-release-archives.sh

@@ -29,7 +29,7 @@ esac
 
 echo "--- Building Mina archive package"
 
-PROJECT="mina-archive"
+PROJECT="mina-archive-${DUNE_PROFILE}"
 BUILD_DIR="deb_build"
 
 ###### archiver deb

scripts/rebuild-deb.sh

@@ -275,11 +275,11 @@ echo "------------------------------------------------------------"
 echo "--- Building Mina Berkeley testnet signatures deb without keys:"
 
 mkdir -p "${BUILDDIR}/DEBIAN"
-create_control_file mina-berkeley "${SHARED_DEPS}${DAEMON_DEPS}" 'Mina Protocol Client and Daemon'
+create_control_file "mina-berkeley-${DUNE_PROFILE}" "${SHARED_DEPS}${DAEMON_DEPS}" 'Mina Protocol Client and Daemon'
 
 copy_common_daemon_configs berkeley testnet 'seed-lists/berkeley_seeds.txt'
 
-build_deb mina-berkeley
+build_deb "mina-berkeley-${DUNE_PROFILE}"
 
 ##################################### END BERKELEY PACKAGE #######################################
 

scripts/release-docker.sh

@@ -24,6 +24,7 @@ function usage() {
   echo "      --deb-codename        The debian codename (stretch or buster) to build the docker image from. Default=stretch"
   echo "      --deb-release         The debian package release channel to pull from (unstable,alpha,beta,stable). Default=unstable"
   echo "      --deb-version         The version string for the debian package to install"
+  echo "      --deb-profile         The profile string for the debian package to install"
   echo ""
   echo "Example: $0 --service faucet --version v0.1.0"
   echo "Valid Services: ${VALID_SERVICES[*]}"
@@ -40,6 +41,7 @@ while [[ "$#" -gt 0 ]]; do case $1 in
   --deb-codename) DEB_CODENAME="--build-arg deb_codename=$2"; shift;;
   --deb-release) DEB_RELEASE="--build-arg deb_release=$2"; shift;;
   --deb-version) DEB_VERSION="--build-arg deb_version=$2"; shift;;
+  --deb-profile) DEB_PROFILE="$2"; shift;;
   --extra-args) EXTRA=${@:2}; shift $((${#}-1));;
   *) echo "Unknown parameter passed: $1"; exit 1;;
 esac; shift; done
@@ -57,7 +59,7 @@ IMAGE="--build-arg image=${IMAGE}"
 
 # Debug prints for visability
 # Substring removal to cut the --build-arg arguments on the = so that the output is exactly the input flags https://wiki.bash-hackers.org/syntax/pe#substring_removal
-echo "--service ${SERVICE} --version ${VERSION} --branch ${BRANCH##*=} --deb-version ${DEB_VERSION##*=} --deb-release ${DEB_RELEASE##*=} --deb-codename ${DEB_CODENAME##*=}"
+echo "--service ${SERVICE} --version ${VERSION} --branch ${BRANCH##*=} --deb-version ${DEB_VERSION##*=} --deb-profile ${DEB_PROFILE##*=} --deb-release ${DEB_RELEASE##*=} --deb-codename ${DEB_CODENAME##*=}"
 echo ${EXTRA}
 echo "docker image: ${IMAGE}"
 
@@ -71,6 +73,7 @@ case "${SERVICE}" in
 mina-archive)
   DOCKERFILE_PATH="dockerfiles/Dockerfile-mina-archive"
   DOCKER_CONTEXT="dockerfiles/"
+  SERVICE=${SERVICE}-${DEB_PROFILE}
   ;;
 bot)
   DOCKERFILE_PATH="frontend/bot/Dockerfile"
@@ -80,6 +83,7 @@ mina-daemon)
   DOCKERFILE_PATH="dockerfiles/Dockerfile-mina-daemon"
   DOCKER_CONTEXT="dockerfiles/"
   VERSION="${VERSION}-${NETWORK##*=}"
+  SERVICE=${SERVICE}-${DEB_PROFILE}
   ;;
 mina-toolchain)
   DOCKERFILE_PATH="dockerfiles/stages/1-build-deps dockerfiles/stages/2-opam-deps dockerfiles/stages/3-toolchain"
@@ -109,7 +113,6 @@ delegation-backend-toolchain)
   ;;
 esac
 
-
 REPO="--build-arg MINA_REPO=${BUILDKITE_PULL_REQUEST_REPO}"
 if [[ -z "${BUILDKITE_PULL_REQUEST_REPO}" ]]; then
   REPO="--build-arg MINA_REPO=https://github.com/MinaProtocol/mina"
@@ -121,12 +124,14 @@ TAG="${DOCKER_REGISTRY}/${SERVICE}:${VERSION}"
 GITHASH=$(git rev-parse --short=7 HEAD)
 HASHTAG="${DOCKER_REGISTRY}/${SERVICE}:${GITHASH}-${DEB_CODENAME##*=}-${NETWORK##*=}"
 
+DEB_PROFILE="--build-arg deb_profile=${DEB_PROFILE}"
+
 # If DOCKER_CONTEXT is not specified, assume none and just pipe the dockerfile into docker build
 extra_build_args=$(echo ${EXTRA} | tr -d '"')
 if [[ -z "${DOCKER_CONTEXT}" ]]; then
-  cat $DOCKERFILE_PATH | docker build $CACHE $NETWORK $IMAGE $DEB_CODENAME $DEB_RELEASE $DEB_VERSION $BRANCH $REPO $extra_build_args -t "$TAG" -
+  cat $DOCKERFILE_PATH | docker build $CACHE $NETWORK $IMAGE $DEB_CODENAME $DEB_RELEASE $DEB_VERSION $DEB_PROFILE $BRANCH $REPO $extra_build_args -t "$TAG" -
 else
-  docker build $CACHE $NETWORK $IMAGE $DEB_CODENAME $DEB_RELEASE $DEB_VERSION $BRANCH $REPO $extra_build_args $DOCKER_CONTEXT -t "$TAG" -f $DOCKERFILE_PATH
+  docker build $CACHE $NETWORK $IMAGE $DEB_CODENAME $DEB_RELEASE $DEB_VERSION $DEB_PROFILE $BRANCH $REPO $extra_build_args $DOCKER_CONTEXT -t "$TAG" -f $DOCKERFILE_PATH
 fi
 
 if [[ -z "$NOUPLOAD" ]] || [[ "$NOUPLOAD" -eq 0 ]]; then