From 34d3b27175f43406835cd956b9f87595f053fc20 Mon Sep 17 00:00:00 2001 From: Crypt Keeper <64215+codefromthecrypt@users.noreply.github.com> Date: Tue, 5 Dec 2023 10:30:40 +0800 Subject: [PATCH] ci: cleans up docker scripts and adds ppc64le note (#78) ppc64le isn't an alpine package, yet, as noted in https://github.com/openzipkin/zipkin/issues/3443 This skips trying that and also merges in the same build-bin/docker/* and build-bin/README.md from docker-alpine Finally, this adds more comments in the Dockerfile about the java_version vs java_home variables. Signed-off-by: Adrian Cole --- Dockerfile | 8 +- build-bin/README.md | 106 ++----------------------- build-bin/deploy | 4 + build-bin/docker/configure_docker | 2 +- build-bin/docker/configure_docker_push | 7 +- build-bin/docker/docker_arch | 7 +- build-bin/docker/docker_args | 10 +-- build-bin/docker/docker_build | 4 +- build-bin/docker/docker_push | 53 ++++++++----- 9 files changed, 71 insertions(+), 130 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7db21fd..52929a9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,13 +8,17 @@ # Use latest version here: https://github.com/orgs/openzipkin/packages/container/package/alpine ARG docker_parent_image=ghcr.io/openzipkin/alpine:3.18.5 -# java_version is hard-coded here to allow the following to work: +# java_version and java_home are hard-coded here to allow the following: # * `docker build https://github.com/openzipkin/docker-java.git` # +# These are overridden via build-bin/docker/docker_args, ensuring the two are +# coherent (e.g. java 21.* has a java_home of java-21-openjdk). +# # When updating, also update the README # * Use current version from https://pkgs.alpinelinux.org/packages?name=openjdk21, stripping # the `-rX` at the end. ARG java_version=21.0.1_p12 +ARG java_home=/usr/lib/jvm/java-21-openjdk # We copy files from the context into a scratch container first to avoid a problem where docker and # docker-compose don't share layer hashes https://github.com/docker/compose/issues/883 normally. @@ -33,7 +37,7 @@ FROM $docker_parent_image as base # This is defined in many places because Docker has no "env" script functionality unless you use # docker-compose: When updating, update everywhere. ARG java_version -ARG java_home=/usr/lib/jvm/java-21-openjdk +ARG java_home LABEL java-version=$java_version LABEL java-home=$java_home diff --git a/build-bin/README.md b/build-bin/README.md index a3618bd..9cf3580 100644 --- a/build-bin/README.md +++ b/build-bin/README.md @@ -1,9 +1,9 @@ # Test and Deploy scripts This is a Docker project, which uses standard conventions for test and deploy. What's notable about -docker-java is that the base layers are only pushed to `ghcr.io`, and tests are simply running the -`-version` command. There's a non-standard `build` script referenced in [../README.md] that allows -easier manual builds of the Dockerfile. +docker-alpine is that the base layers are only pushed to `ghcr.io`, and tests are simply running the +`cat /etc/alpine-release`. There's a non-standard `build` script referenced in [../README.md] that +allows easier manual builds of the Dockerfile. [//]: # (Below here should be standard for all projects) @@ -11,7 +11,7 @@ easier manual builds of the Dockerfile. `build-bin` holds portable scripts used in CI to test and deploy the project. The scripts here are portable. They do not include any CI provider-specific logic or ENV variables. -This helps `.travis.yml` and `test.yml` (GitHub Actions) contain nearly the same contents, even if +This helps `test.yml` (GitHub Actions) and alternatives contain nearly the same contents, even if certain OpenZipkin projects need slight adjustments here. Portability has proven necessary, as OpenZipkin has had to transition CI providers many times due to feature and quota constraints. @@ -64,7 +64,7 @@ jobs: test: steps: - name: Checkout Repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: fetch-depth: 0 # full git history for license check - name: Test @@ -73,49 +73,6 @@ jobs: build-bin/test ``` -### Example Travis setup -`.travis.yml` is a monolithic configuration file broken into stages, of which the default is "test". -A simplest Travis `test` job configures tests in `install` and runs them as `script`, but only on -relevant event conditions. - -The `if:` section obviates job creation and resource usage for irrelevant events. Travis does not -support file conditions. A `before_install` step to skip documentation-only commits will likely -complete in less than a minute (10 credit cost). - -Here's a partial `.travis.yml` including only the aspects mentioned above. -```yaml -git: - depth: false # TRAVIS_COMMIT_RANGE requires full commit history. - -jobs: - include: - - stage: test - if: branch = master AND tag IS blank AND type IN (push, pull_request) - name: Run unit and integration tests - before_install: | # Prevent test build of a documentation-only change. - if [ -n "${TRAVIS_COMMIT_RANGE}" ] && ! git diff --name-only "${TRAVIS_COMMIT_RANGE}" -- | grep -qv '\.md$'; then - echo "Stopping job as changes only affect documentation (ex. README.md)" - travis_terminate 0 - fi - install: ./build-bin/configure_test - script: ./build-bin/test -``` - -When Travis only runs tests (something else does deploy), there's no need to use stages: -```yaml -git: - depth: false # TRAVIS_COMMIT_RANGE requires full commit history. - -if: branch = master AND tag IS blank AND type IN (push, pull_request) -before_install: | # Prevent test build of a documentation-only change. - if [ -n "${TRAVIS_COMMIT_RANGE}" ] && ! git diff --name-only "${TRAVIS_COMMIT_RANGE}" -- | grep -qv '\.md$'; then - echo "Stopping job as changes only affect documentation (ex. README.md)" - travis_terminate 0 - fi -install: ./build-bin/configure_test -script: ./build-bin/test -``` - ## Deploy Deploy builds and pushes artifacts to a remote repository for master and release commits on it. CI @@ -138,17 +95,18 @@ master will happen even on README change. Here's a partial `deploy.yml` including only the aspects mentioned above. Notice env variables are explicitly defined and `on.tags` is a [glob pattern](https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-syntax-for-github-actions#filter-pattern-cheat-sheet). + ```yaml on: push: - tags: '[0-9]+.[0-9]+.[0-9]+**' # Ex. 8.272.10 or 21.0.1_p12 + tags: '[0-9]+.[0-9]+.[0-9]+**' # e.g. 8.272.10 or 15.0.1_p9 branches: master jobs: deploy: steps: - name: Checkout Repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: fetch-depth: 1 # only needed to get the sha label - name: Configure Deploy @@ -160,51 +118,3 @@ jobs: # GITHUB_REF will be refs/heads/master or refs/tags/1.2.3 run: build-bin/deploy $(echo ${GITHUB_REF} | cut -d/ -f 3) ``` -'[0-9]+.[0-9]+.[0-9]+**' -### Example Travis setup -`.travis.yml` is a monolithic configuration file broken into stages. This means `test` and `deploy` -are in the same file. A simplest Travis `deploy` stage has two jobs: one for master pushes and -another for version tags. These jobs are controlled by event conditions. - -The `if:` section obviates job creation and resource usage for irrelevant events. Travis does not -support file conditions. A `before_install` step to skip documentation-only commits will likely -complete in less than a minute (10 credit cost). - -As billing is by the minute, it is most cost effective to combine test and deploy on master push. - -Here's a partial `.travis.yml` including only the aspects mentioned above. Notice YAML anchors work -in Travis and `tag =~` [condition](https://github.com/travis-ci/travis-conditions) is a regular -expression. -```yaml -git: - depth: false # full git history for license check, and doc-only skipping - -_terminate_if_only_docs: &terminate_if_only_docs | - if [ -n "${TRAVIS_COMMIT_RANGE}" ] && ! git diff --name-only "${TRAVIS_COMMIT_RANGE}" -- | grep -qv '\.md$'; then - echo "Stopping job as changes only affect documentation (ex. README.md)" - travis_terminate 0 - fi - -jobs: - include: - - stage: test - if: branch = master AND tag IS blank AND type IN (push, pull_request) - before_install: *terminate_if_only_docs - install: | - if [ "${TRAVIS_SECURE_ENV_VARS}" = "true" ] && [ "${TRAVIS_PULL_REQUEST}" = "false" ]; then - export SHOULD_DEPLOY=true - ./build-bin/configure_deploy - else - export SHOULD_DEPLOY=false - ./build-bin/configure_test - fi - script: - - ./build-bin/test || travis_terminate 1 - - if [ "${SHOULD_DEPLOY}" != "true" ]; then travis_terminate 0; fi - - travis_wait ./build-bin/deploy master - - stage: deploy - # Ex. 8.272.10 or 21.0.1_p12 - if: tag =~ /^[0-9]+\.[0-9]+\.[0-9]+/ AND type = push AND env(GH_TOKEN) IS present - install: ./build-bin/configure_deploy - script: ./build-bin/deploy ${TRAVIS_TAG} -``` diff --git a/build-bin/deploy b/build-bin/deploy index 2dcb143..ee60591 100755 --- a/build-bin/deploy +++ b/build-bin/deploy @@ -12,6 +12,10 @@ fi export DOCKER_RELEASE_REPOS=ghcr.io export DOCKER_FILE=Dockerfile +# Don't attempt ppc64le until there's a package for recent LTS versions. +# See https://github.com/openzipkin/zipkin/issues/3443 +export DOCKER_ARCHS="amd64 arm64 s390x" + # Don't make latest tag DOCKER_TARGET=jdk DOCKER_TAGS=${version} build-bin/docker/docker_push openzipkin/java ${version} DOCKER_TARGET=jre DOCKER_TAGS=${version}-jre build-bin/docker/docker_push openzipkin/java ${version} diff --git a/build-bin/docker/configure_docker b/build-bin/docker/configure_docker index 0c6a045..f71bb1a 100755 --- a/build-bin/docker/configure_docker +++ b/build-bin/docker/configure_docker @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright 2019-2020 The OpenZipkin Authors +# Copyright 2019-2023 The OpenZipkin Authors # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at diff --git a/build-bin/docker/configure_docker_push b/build-bin/docker/configure_docker_push index 196e302..a26fab3 100755 --- a/build-bin/docker/configure_docker_push +++ b/build-bin/docker/configure_docker_push @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright 2019-2020 The OpenZipkin Authors +# Copyright 2019-2023 The OpenZipkin Authors # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -50,5 +50,8 @@ fi # See https://github.com/multiarch/qemu-user-static # # Mirrored image use to avoid docker.io pulls: -# docker tag multiarch/qemu-user-static:5.1.0-7 ghcr.io/openzipkin/multiarch-qemu-user-static:latest +# docker tag multiarch/qemu-user-static:7.2.0-1 ghcr.io/openzipkin/multiarch-qemu-user-static:latest +# +# Note: This image only works on x86_64/amd64 architecture. +# See: https://github.com/multiarch/qemu-user-static#supported-host-architectures docker run --rm --privileged ghcr.io/openzipkin/multiarch-qemu-user-static --reset -p yes diff --git a/build-bin/docker/docker_arch b/build-bin/docker/docker_arch index 911a9a6..d83063e 100755 --- a/build-bin/docker/docker_arch +++ b/build-bin/docker/docker_arch @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright 2019-2020 The OpenZipkin Authors +# Copyright 2019-2023 The OpenZipkin Authors # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -21,6 +21,8 @@ set -ue # Normalize docker_arch to what's available +# +# Note: s390x and ppc64le were added for Knative docker_arch=${DOCKER_ARCH:-$(uname -m)} case ${docker_arch} in amd64* ) @@ -38,6 +40,9 @@ case ${docker_arch} in s390x* ) docker_arch=s390x ;; + ppc64le* ) + docker_arch=ppc64le + ;; * ) >&2 echo "Unsupported DOCKER_ARCH: ${docker_arch}" exit 1; diff --git a/build-bin/docker/docker_args b/build-bin/docker/docker_args index f51a516..7f35ba3 100755 --- a/build-bin/docker/docker_args +++ b/build-bin/docker/docker_args @@ -46,7 +46,7 @@ if [ -n "${DOCKER_TARGET}" ]; then fi # When non-empty, becomes the base layer including tag appropriate for the image being built. -# Ex. ghcr.io/openzipkin/java:21.0.1_p12-jre +# e.g. ghcr.io/openzipkin/java:21.0.1_p12-jre # # This is not required to be a base (FROM scratch) image like ghcr.io/openzipkin/alpine:3.12.3 # See https://docs.docker.com/glossary/#parent-image @@ -54,13 +54,13 @@ if [ -n "${DOCKER_PARENT_IMAGE}" ]; then docker_args="${docker_args} --build-arg docker_parent_image=${DOCKER_PARENT_IMAGE}" fi -# When non-empty, becomes the build-arg alpine_version. Ex. "3.12.3" +# When non-empty, becomes the build-arg alpine_version. e.g. "3.12.3" # Used to align base layers from https://github.com/orgs/openzipkin/packages/container/package/alpine if [ -n "${ALPINE_VERSION}" ]; then docker_args="${docker_args} --build-arg alpine_version=${ALPINE_VERSION}" fi -# When non-empty, becomes the build-arg java_version. Ex. "21.0.1_p12" +# When non-empty, becomes the build-arg java_version. e.g. "21.0.1_p12" # Used to align base layers from https://github.com/orgs/openzipkin/packages/container/package/java if [ -n "${JAVA_VERSION}" ]; then docker_args="${docker_args} --build-arg java_version=${JAVA_VERSION}" @@ -69,13 +69,13 @@ if [ -n "${JAVA_VERSION}" ]; then java_major_version=$(echo ${JAVA_VERSION}| cut -f1 -d .) case ${java_major_version} in 8) java_home=/usr/lib/jvm/java-1.8-openjdk;; - 1?) java_home=/usr/lib/jvm/java-${java_major_version}-openjdk;; + 1?|2?|3?) java_home=/usr/lib/jvm/java-${java_major_version}-openjdk;; esac if [ -n "${java_home}" ]; then docker_args="${docker_args} --build-arg java_home=${java_home}"; fi fi -# When non-empty, becomes the build-arg maven_classifier. Ex. "module" or "exec" +# When non-empty, becomes the build-arg maven_classifier. e.g. "module" or "exec" # Used as the classifier arg to ./build-bin/maven/maven_unjar. Allows building two images with the # same Dockerfile, varying on classifier, like openzipkin/zipkin vs openzipkin/zipkin-slim if [ -n "${MAVEN_CLASSIFIER}" ]; then diff --git a/build-bin/docker/docker_build b/build-bin/docker/docker_build index 9ce3fdd..05d5450 100755 --- a/build-bin/docker/docker_build +++ b/build-bin/docker/docker_build @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright 2019-2020 The OpenZipkin Authors +# Copyright 2019-2023 The OpenZipkin Authors # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -20,4 +20,4 @@ version=${2:-} docker_args=$($(dirname "$0")/docker_args ${version}) echo "Building image ${docker_tag}" -DOCKER_BUILDKIT=1 docker build --pull ${docker_args} --tag ${docker_tag} . +DOCKER_BUILDKIT=1 docker build --network=host --pull ${docker_args} --tag ${docker_tag} . diff --git a/build-bin/docker/docker_push b/build-bin/docker/docker_push index cc2f258..3d3c8f9 100755 --- a/build-bin/docker/docker_push +++ b/build-bin/docker/docker_push @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright 2019-2020 The OpenZipkin Authors +# Copyright 2019-2023 The OpenZipkin Authors # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -64,11 +64,13 @@ for repo in ${docker_repos}; do done docker_args=$($(dirname "$0")/docker_args ${version}) -docker_archs=${DOCKER_ARCHS:-amd64 arm64 s390x} +# Note: s390x and ppc64le were added for Knative +docker_archs=${DOCKER_ARCHS:-amd64 arm64 s390x ppc64le} echo "Will build the following architectures: ${docker_archs}" docker_tag0="$(echo ${docker_tags} | awk '{print $1;}')" +docker_arch0="$(echo ${docker_archs} | awk '{print $1;}')" arch_tags="" for docker_arch in ${docker_archs}; do arch_tag=${docker_image}:${docker_tag0}-${docker_arch} @@ -79,24 +81,37 @@ done echo "Will push the following tags:\n${tags}" -for tag in $(echo ${tags} | xargs); do - manifest_tags="" - for arch_tag in ${arch_tags}; do - docker_arch=$(echo ${arch_tag} | sed 's/.*-//g') - manifest_tag=${tag}-${docker_arch} - docker tag ${arch_tag} ${manifest_tag} - echo "Pushing tag ${manifest_tag}..." - docker push ${manifest_tag} - manifest_tags="${manifest_tags} ${manifest_tag}" +if [ "${docker_arch0}" = "${docker_archs}" ]; then + # single architecture + arch_tag=${docker_image}:${docker_tag0}-${docker_arch0} + + for tag in $(echo ${tags} | xargs); do + docker tag ${arch_tag} ${tag} + echo "Pushing tag ${tag}..." + docker push ${tag} done - docker manifest create ${tag} ${manifest_tags} +else + # multi-architecture: make a manifest + for tag in $(echo ${tags} | xargs); do + manifest_tags="" + for arch_tag in ${arch_tags}; do + docker_arch=$(echo ${arch_tag} | sed 's/.*-//g') + manifest_tag=${tag}-${docker_arch} + docker tag ${arch_tag} ${manifest_tag} + echo "Pushing tag ${manifest_tag}..." + docker push ${manifest_tag} + manifest_tags="${manifest_tags} ${manifest_tag}" + done - for manifest_tag in ${manifest_tags}; do - docker_arch=$(echo ${manifest_tag} | sed 's/.*-//g') - docker manifest annotate ${tag} ${manifest_tag} --os linux --arch ${docker_arch} - done + docker manifest create ${tag} ${manifest_tags} - echo "Pushing manifest ${manifest_tag}..." - docker manifest push -p ${tag} -done + for manifest_tag in ${manifest_tags}; do + docker_arch=$(echo ${manifest_tag} | sed 's/.*-//g') + docker manifest annotate ${tag} ${manifest_tag} --os linux --arch ${docker_arch} + done + + echo "Pushing manifest ${manifest_tag}..." + docker manifest push -p ${tag} + done +fi