aarlt · aarlt · Jul 13, 2020 · Jun 23, 2020 · Jul 14, 2020 · Jul 14, 2020
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -33,26 +33,11 @@ defaults:
 
   - run_build: &run_build
       name: Build
-      command: |
-        set -ex
-        if [ "$CIRCLE_BRANCH" = release -o -n "$CIRCLE_TAG" -o -n "$FORCE_RELEASE" ]; then echo -n > prerelease.txt; else date -u +"nightly.%Y.%-m.%-d" > prerelease.txt; fi
-        echo -n "$CIRCLE_SHA1" > commit_hash.txt
-        mkdir -p build
-        cd build
-        [ -n "$COVERAGE" -a "$CIRCLE_BRANCH" != release -a -z "$CIRCLE_TAG" ] && CMAKE_OPTIONS="$CMAKE_OPTIONS -DCOVERAGE=ON"
-        cmake .. -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:-Release} $CMAKE_OPTIONS -G "Unix Makefiles"
-        make -j4
+      command: scripts/ci/build.sh
 
   - run_build_ossfuzz: &run_build_ossfuzz
       name: Build_ossfuzz
-      command: |
-        mkdir -p build
-        cd build
-        protoc --proto_path=../test/tools/ossfuzz yulProto.proto --cpp_out=../test/tools/ossfuzz
-        protoc --proto_path=../test/tools/ossfuzz abiV2Proto.proto --cpp_out=../test/tools/ossfuzz
-        protoc --proto_path=../test/tools/ossfuzz solProto.proto --cpp_out=../test/tools/ossfuzz
-        cmake .. -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:-Release} $CMAKE_OPTIONS
-        make ossfuzz ossfuzz_proto ossfuzz_abiv2 -j4
+      command: scripts/ci/build_ossfuzz.sh
 
   - run_proofs: &run_proofs
       name: Correctness proofs for optimization rules

diff --git a/.github/workflows/buildpack-deps.yml b/.github/workflows/buildpack-deps.yml
@@ -0,0 +1,42 @@
+name: buildpack-deps
+
+on:
+  pull_request:
+    paths:
+      - 'scripts/docker/buildpack-deps/Dockerfile.emscripten'
+      - 'scripts/docker/buildpack-deps/Dockerfile.ubuntu1604.clang.ossfuzz'
+      - 'scripts/docker/buildpack-deps/Dockerfile.ubuntu1804'
+      - 'scripts/docker/buildpack-deps/Dockerfile.ubuntu2004.clang'
+      - 'scripts/docker/buildpack-deps/Dockerfile.ubuntu2004'
+
+jobs:
+  buildpack-deps:
+    env:
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      DOCKER_REPOSITORY: docker.pkg.github.com/${{ github.repository }}
+      MAKE_JOBS: 4
+      IMAGE_NAME: buildpack-deps
+
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        image_variant: [emscripten, ubuntu1604.clang.ossfuzz, ubuntu1804, ubuntu2004.clang, ubuntu2004]
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Upgrade ${{ env.IMAGE_NAME }}-${{ matrix.image_variant }}
+        run: |
+          echo "${GITHUB_TOKEN}" | docker login docker.pkg.github.com -u "${GITHUB_ACTOR}" --password-stdin
+          scripts/ci/docker_upgrade.sh ${{ env.IMAGE_NAME }} ${{ matrix.image_variant }} ${{ env.DOCKER_REPOSITORY }}
+          docker logout docker.pkg.github.com
+
+      - name: comment PR
+        if: "env.DOCKER_IMAGE"
+        uses: aarlt/[email protected]
+        with:
+          msg: "`${{ env.DOCKER_IMAGE }} ${{ env.DOCKER_REPO_DIGEST }}`."
diff --git a/.gitignore b/.gitignore
@@ -32,7 +32,7 @@ prerelease.txt
 
 # Build directory
 build/
-build*/
+/build*/
 emscripten_build/
 docs/_build
 __pycache__

diff --git a/scripts/ci/build.sh b/scripts/ci/build.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+set -ex
+
+ROOTDIR="$(dirname "$0")/../.."
+cd "${ROOTDIR}"
+
+# shellcheck disable=SC2166
+if [ "$CIRCLE_BRANCH" = release -o -n "$CIRCLE_TAG" -o -n "$FORCE_RELEASE" ]; then echo -n >prerelease.txt; else date -u +"nightly.%Y.%-m.%-d" >prerelease.txt; fi
+if [ -n "$CIRCLE_SHA1" ]
+then
+  echo -n "$CIRCLE_SHA1" >commit_hash.txt
+fi
+
+mkdir -p build
+cd build
+
+# shellcheck disable=SC2166
+[ -n "$COVERAGE" -a "$CIRCLE_BRANCH" != release -a -z "$CIRCLE_TAG" ] && CMAKE_OPTIONS="$CMAKE_OPTIONS -DCOVERAGE=ON"
+
+# shellcheck disable=SC2086
+cmake .. -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE:-Release}" $CMAKE_OPTIONS -G "Unix Makefiles"
+
+[[ "${MAKE_JOBS}" != "" ]] || MAKE_JOBS=4
+make -j "${MAKE_JOBS}"
diff --git a/scripts/ci/build_ossfuzz.sh b/scripts/ci/build_ossfuzz.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+set -e
+
+ROOTDIR="$(dirname "$0")/../.."
+BUILDDIR="${ROOTDIR}/build"
+
+mkdir -p "${BUILDDIR}"
+cd "${BUILDDIR}"
+
+protoc --proto_path=../test/tools/ossfuzz yulProto.proto --cpp_out=../test/tools/ossfuzz
+protoc --proto_path=../test/tools/ossfuzz abiV2Proto.proto --cpp_out=../test/tools/ossfuzz
+protoc --proto_path=../test/tools/ossfuzz solProto.proto --cpp_out=../test/tools/ossfuzz
+cmake .. -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE:-Release}" -DCMAKE_TOOLCHAIN_FILE=../cmake/toolchains/libfuzzer.cmake
+
+[[ "${MAKE_JOBS}" != "" ]] || MAKE_JOBS=4
+make ossfuzz ossfuzz_proto ossfuzz_abiv2 -j "${MAKE_JOBS}"
diff --git a/scripts/ci/buildpack-deps_test_emscripten.sh b/scripts/ci/buildpack-deps_test_emscripten.sh
@@ -0,0 +1 @@
+../../scripts/travis-emscripten/build_emscripten.sh
diff --git a/scripts/ci/buildpack-deps_test_ubuntu1604.clang.ossfuzz.sh b/scripts/ci/buildpack-deps_test_ubuntu1604.clang.ossfuzz.sh
@@ -0,0 +1 @@
+build_ossfuzz.sh
diff --git a/scripts/ci/buildpack-deps_test_ubuntu1804.sh b/scripts/ci/buildpack-deps_test_ubuntu1804.sh
@@ -0,0 +1 @@
+build.sh
diff --git a/scripts/ci/buildpack-deps_test_ubuntu2004.clang.sh b/scripts/ci/buildpack-deps_test_ubuntu2004.clang.sh
@@ -0,0 +1 @@
+build.sh
diff --git a/scripts/ci/buildpack-deps_test_ubuntu2004.sh b/scripts/ci/buildpack-deps_test_ubuntu2004.sh
@@ -0,0 +1 @@
+build.sh
diff --git a/scripts/ci/docker_upgrade.sh b/scripts/ci/docker_upgrade.sh
@@ -0,0 +1,78 @@
+#!/usr/bin/env bash
+set -e
+
+function error() {
+  echo >&2 "ERROR: ${1} Aborting." && false
+}
+
+function warning() {
+  echo >&2 "WARNING: ${1}"
+}
+
+[[ $# == 3 ]] || error "Expected exactly 3 parameters: ${0} <IMAGE_NAME> <IMAGE_VARIANT> <DOCKER_REPOSITORY>"
+
+IMAGE_NAME=${1}
+IMAGE_VARIANT=${2}
+DOCKER_REPOSITORY=${3}
+
+check_version() {
+  echo "-- check_version"
+
+  DOCKERFILE="scripts/docker/${IMAGE_NAME}/Dockerfile.${IMAGE_VARIANT}"
+
+  # continue running the whole workflow only if corresponding Dockerfile was changed.
+  if ! git diff --name-only origin/develop HEAD  -- | grep -e "^${DOCKERFILE}\$"
+  then
+    echo "${DOCKERFILE} was not changed. Nothing to do."
+    exit 0
+  fi
+
+  PREV_VERSION=$(git diff origin/develop HEAD -- "${DOCKERFILE}" | grep -e '^\s*-LABEL\s\+version=".*"\s*$' | awk -F'"' '{ print $2 }')
+  NEXT_VERSION=$(git diff origin/develop HEAD -- "${DOCKERFILE}" | grep -e '^\s*+LABEL\s\+version=".*"\s*$' | awk -F'"' '{ print $2 }')
+
+  [[ $NEXT_VERSION != "" ]] || error "No version label defined in Dockerfile. You may need to add 'LABEL version' in '${DOCKERFILE}'."
+
+  [[ $PREV_VERSION != "" ]] || {
+    warning "no previous version found. Will set \$PREV_VERSION = 0."
+    PREV_VERSION=0
+  }
+
+  if [[ $((PREV_VERSION + 1)) != $((NEXT_VERSION)) ]]; then
+    error "Version label in Dockerfile was not incremented. You may need to change 'LABEL version' in '${DOCKERFILE}'."
+  fi
+}
+
+build_docker() {
+  echo "-- build_docker"
+
+  docker build "scripts/docker/${IMAGE_NAME}" --file "scripts/docker/${IMAGE_NAME}/Dockerfile.${IMAGE_VARIANT}" --tag "${IMAGE_NAME}" || docker build "scripts/docker/${IMAGE_NAME}" --file "scripts/docker/${IMAGE_NAME}/Dockerfile.${IMAGE_VARIANT}" --tag "${IMAGE_NAME}"
+}
+
+test_docker() {
+  echo "-- test_docker @ '${PWD}'"
+
+  docker run --rm --volume "${PWD}:/root/project" "${IMAGE_NAME}" "/root/project/scripts/ci/${IMAGE_NAME}_test_${IMAGE_VARIANT}.sh"
+}
+
+push_docker() {
+  echo "-- push_docker"
+
+  VERSION=$(docker inspect --format='{{.Config.Labels.version}}' "${IMAGE_NAME}")
+  DOCKER_IMAGE_ID="${DOCKER_REPOSITORY}/${IMAGE_NAME}-${IMAGE_VARIANT}"
+
+  docker tag "${IMAGE_NAME}" "${DOCKER_IMAGE_ID}:${VERSION}"
+  docker push "${DOCKER_IMAGE_ID}:${VERSION}"
+
+  REPO_DIGEST=$(docker inspect --format='{{.RepoDigests}}' "${DOCKER_IMAGE_ID}:${VERSION}")
+
+  docker tag "${IMAGE_NAME}" "${DOCKER_IMAGE_ID}:latest"
+  docker push "${DOCKER_IMAGE_ID}:latest"
+
+  echo "::set-env name=DOCKER_IMAGE::${DOCKER_IMAGE_ID}:${VERSION}"
+  echo "::set-env name=DOCKER_REPO_DIGEST::${REPO_DIGEST}"
+}
+
+check_version
+build_docker
+test_docker
+push_docker
diff --git a/.circleci/docker/Dockerfile.emscripten → ...cker/buildpack-deps/Dockerfile.emscripten b/.circleci/docker/Dockerfile.emscripten → ...cker/buildpack-deps/Dockerfile.emscripten
@@ -29,6 +29,7 @@
 #   make version=1.39.15 build
 #
 FROM emscripten/emsdk:1.39.15 AS base
+LABEL version="1"
 
 ADD emscripten.jam /usr/src
 RUN set -ex; \
@@ -63,3 +64,4 @@ RUN set -ex; \
 		cxxflags="-s DISABLE_EXCEPTION_CATCHING=0 -Wno-unused-local-typedef -Wno-variadic-macros -Wno-c99-extensions -Wno-all" \
 	       --prefix=/emsdk/emscripten/sdk/system install; \
 	rm -r /usr/src/boost_1_73_0
+
diff --git a/...ocker/Dockerfile.ubuntu1604.clang.ossfuzz → ...-deps/Dockerfile.ubuntu1604.clang.ossfuzz b/...ocker/Dockerfile.ubuntu1604.clang.ossfuzz → ...-deps/Dockerfile.ubuntu1604.clang.ossfuzz
@@ -22,6 +22,7 @@
 # (c) 2016-2019 solidity contributors.
 #------------------------------------------------------------------------------
 FROM gcr.io/oss-fuzz-base/base-clang as base
+LABEL version="1"
 
 ARG DEBIAN_FRONTEND=noninteractive
 
@@ -99,3 +100,4 @@ FROM base
 COPY --from=libraries /usr/lib /usr/lib
 COPY --from=libraries /usr/bin /usr/bin
 COPY --from=libraries /usr/include /usr/include
+
diff --git a/.circleci/docker/Dockerfile.ubuntu1804 → ...cker/buildpack-deps/Dockerfile.ubuntu1804 b/.circleci/docker/Dockerfile.ubuntu1804 → ...cker/buildpack-deps/Dockerfile.ubuntu1804
@@ -22,6 +22,7 @@
 # (c) 2016-2019 solidity contributors.
 #------------------------------------------------------------------------------
 FROM buildpack-deps:bionic AS base
+LABEL version="1"
 
 ARG DEBIAN_FRONTEND=noninteractive
 
@@ -91,3 +92,4 @@ FROM base
 COPY --from=libraries /usr/lib /usr/lib
 COPY --from=libraries /usr/bin /usr/bin
 COPY --from=libraries /usr/include /usr/include
+
diff --git a/.circleci/docker/Dockerfile.ubuntu2004 → ...cker/buildpack-deps/Dockerfile.ubuntu2004 b/.circleci/docker/Dockerfile.ubuntu2004 → ...cker/buildpack-deps/Dockerfile.ubuntu2004
@@ -22,6 +22,7 @@
 # (c) 2016-2019 solidity contributors.
 #------------------------------------------------------------------------------
 FROM buildpack-deps:focal AS base
+LABEL version="1"
 
 ARG DEBIAN_FRONTEND=noninteractive
 
@@ -60,3 +61,4 @@ FROM base
 COPY --from=libraries /usr/lib /usr/lib
 COPY --from=libraries /usr/bin /usr/bin
 COPY --from=libraries /usr/include /usr/include
+
diff --git a/.circleci/docker/Dockerfile.ubuntu2004.clang → ...uildpack-deps/Dockerfile.ubuntu2004.clang b/.circleci/docker/Dockerfile.ubuntu2004.clang → ...uildpack-deps/Dockerfile.ubuntu2004.clang
@@ -22,6 +22,7 @@
 # (c) 2016-2019 solidity contributors.
 #------------------------------------------------------------------------------
 FROM buildpack-deps:focal AS base
+LABEL version="1"
 
 ARG DEBIAN_FRONTEND=noninteractive
 
@@ -62,3 +63,4 @@ FROM base
 COPY --from=libraries /usr/lib /usr/lib
 COPY --from=libraries /usr/bin /usr/bin
 COPY --from=libraries /usr/include /usr/include
+
diff --git a/scripts/docker/buildpack-deps/README.md b/scripts/docker/buildpack-deps/README.md
@@ -0,0 +1,30 @@
+# buildpack-deps docker images
+
+The `buildpack-deps` docker images are used to compile and test solidity within our CI.
+
+## GitHub Workflow
+
+The creation of the images are triggered by different workflows. For each resulting
+`buildpack-deps` docker image a workflow is defined in `.github/workflows/buildpack-deps`.
+The workflows get triggered, if the workflow file itself, or the Dockerfiles `scripts/docker/buildpack-deps/Dockerfile.*`
+were changed within a PR.
+
+### Versioning
+
+The version of the docker images can be defined within the Dockerfile with `LABEL version`. A new docker image
+will only be created and pushed, if the new version is incremented by `1` compared with the version of the Dockerfile
+located in `develop`.
+
+### Build, Test & Push
+
+If the version check was successful, the docker image will be built using the Dockerfile located in
+`scripts/docker/buildpack-deps/Dockerfile.*`.
+
+The resulting docker image will be tested by executing
+the corresponding `scripts/ci/buildpack-deps_test_*` scripts. These scripts are normally symlinked to `scripts/ci/build.sh`,
+except for the `buildpack-deps-ubuntu1604.clang.ossfuzz` docker image, that is symlinked to `scripts/ci/build_ossfuzz.sh`.
+These scripts `scripts/ci/build.sh` and `scripts/ci/build_ossfuzz.sh` are also used by CircleCI, see `.circleci/config.yml`.
+
+If the tests passed successfully, the docker image will get tagged by the version defined within the corresponding `Dockerfile`.
+Finally, a comment will be added to the PR that contains the full repository, version and repository digest
+of the freshly created docker image.
diff --git a/.circleci/docker/emscripten.jam → scripts/docker/buildpack-deps/emscripten.jam b/.circleci/docker/emscripten.jam → scripts/docker/buildpack-deps/emscripten.jam
diff --git a/scripts/travis-emscripten/build_emscripten.sh b/scripts/travis-emscripten/build_emscripten.sh
@@ -53,7 +53,10 @@ cmake \
   -DBoost_USE_STATIC_RUNTIME=1 \
   -DTESTS=0 \
   ..
-make -j 4 soljson
+
+[[ "${MAKE_JOBS}" != "" ]] || MAKE_JOBS=4
+make -j ${MAKE_JOBS} soljson
+
 # Patch soljson.js for backwards compatibility.
 # TODO: remove this with 0.7.
 # "viiiii" encodes the signature of the callback function.
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		../../scripts/travis-emscripten/build_emscripten.sh