Skip to content

Commit

Permalink
Update containerd to be properly multiarch
Browse files Browse the repository at this point in the history
  • Loading branch information
tianon committed Sep 3, 2023
1 parent 6afd7cf commit 1531fa8
Show file tree
Hide file tree
Showing 6 changed files with 225 additions and 15 deletions.
26 changes: 26 additions & 0 deletions .libs/lib.jq
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
def unique_unsorted:
# https://unix.stackexchange.com/a/738744/153467
reduce .[] as $a ([]; if IN(.[]; $a) then . else . += [$a] end)
;
def intersection:
# add all arrays together and bit-by-bit remove anything unique from each array
reduce .[] as $arr (add | unique_unsorted; . - (. - $arr))
;
def deb_arch:
{
# https://salsa.debian.org/dpkg-team/dpkg/-/blob/main/data/cputable
# https://wiki.debian.org/ArchitectureSpecificsMemo#Architecture_baselines
# http://deb.debian.org/debian/dists/unstable/main/
# http://deb.debian.org/debian/dists/stable/main/
# https://deb.debian.org/debian-ports/dists/unstable/main/
amd64: "amd64",
arm32v5: "armel",
arm32v7: "armhf",
arm64v8: "arm64",
i386: "i386",
mips64le: "mips64el",
ppc64le: "ppc64el",
riscv64: "riscv64",
s390x: "s390x",
}[.]
;
30 changes: 24 additions & 6 deletions containerd/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -25,18 +25,36 @@ RUN set -eux; \
# https://github.com/opencontainers/runc/releases
ENV RUNC_VERSION 1.1.9
RUN set -eux; \
# TODO multiple architectures
wget -O /usr/local/bin/runc "https://github.com/opencontainers/runc/releases/download/v${RUNC_VERSION}/runc.amd64" --progress=dot:giga; \
# TODO .asc / .sha256sum
dpkgArch="$(dpkg --print-architecture)"; \
case "$dpkgArch" in \
'amd64') url='https://github.com/opencontainers/runc/releases/download/v1.1.9/runc.amd64'; sha256='b9bfdd4cb27cddbb6172a442df165a80bfc0538a676fbca1a6a6c8f4c6933b43' ;; \
'arm64') url='https://github.com/opencontainers/runc/releases/download/v1.1.9/runc.arm64'; sha256='b43e9f561e85906f469eef5a7b7992fc586f750f44a0e011da4467e7008c33a0' ;; \
'armhf') url='https://github.com/opencontainers/runc/releases/download/v1.1.9/runc.armhf'; sha256='3c33909a3e2b40d879a1c07b89dc18c7dc727de61a6c381173e8c5c46b200c3d' ;; \
'ppc64el') url='https://github.com/opencontainers/runc/releases/download/v1.1.9/runc.ppc64le'; sha256='065cf4f84b5acc0acdb017af2955743dfb5f5e1f49a493eea3e8206f33bf6fe6' ;; \
'riscv64') url='https://github.com/opencontainers/runc/releases/download/v1.1.9/runc.riscv64'; sha256='177a64486b8ce372938692f44e4aa6a6d9d3476a1f77b152142f02cf881a4817' ;; \
's390x') url='https://github.com/opencontainers/runc/releases/download/v1.1.9/runc.s390x'; sha256='363b317d44a44591205439e29f70f94f01eb51e58047b092b03b1cd2cdad533a' ;; \
*) echo >&2 "error: unsupported architecture: '$dpkgArch'"; exit 1 ;; \
esac; \
# TODO $url.asc (see https://github.com/opencontainers/runc/releases)
wget -O /usr/local/bin/runc "$url" --progress=dot:giga; \
echo "$sha256 */usr/local/bin/runc" | sha256sum --strict --check -; \
chmod +x /usr/local/bin/runc; \
runc --version

# https://github.com/containerd/containerd/releases
ENV CONTAINERD_VERSION 1.7.5
RUN set -eux; \
# TODO multiple architectures
wget -O containerd.tar.gz "https://github.com/containerd/containerd/releases/download/v${CONTAINERD_VERSION}/containerd-${CONTAINERD_VERSION}-linux-amd64.tar.gz" --progress=dot:giga; \
# TODO .sha256sum
dpkgArch="$(dpkg --print-architecture)"; \
case "$dpkgArch" in \
'amd64') url='https://github.com/containerd/containerd/releases/download/v1.7.5/containerd-1.7.5-linux-amd64.tar.gz'; sha256='33609ae2d5838bc5798306a1ac30d7f2c6a8cff785ca6253d2be8a8b3ccbab25' ;; \
'arm64') url='https://github.com/containerd/containerd/releases/download/v1.7.5/containerd-1.7.5-linux-arm64.tar.gz'; sha256='98fc6990820d52d45b56ea2cda808157d4e61bb30ded96887634644c03025fa9' ;; \
'ppc64el') url='https://github.com/containerd/containerd/releases/download/v1.7.5/containerd-1.7.5-linux-ppc64le.tar.gz'; sha256='2496e24a95fa74750363a8a7e2ac36acf8d41ee2e4b67a452154ad4c8efbc4bc' ;; \
'riscv64') url='https://github.com/containerd/containerd/releases/download/v1.7.5/containerd-1.7.5-linux-riscv64.tar.gz'; sha256='9b2835df49f8fd9c2562a9173f2cd33412712e0e8e0e792fa8d187dec901bbb1' ;; \
's390x') url='https://github.com/containerd/containerd/releases/download/v1.7.5/containerd-1.7.5-linux-s390x.tar.gz'; sha256='0cd81b8844a2854b746af0c30d00fe3df5018e0985c56175dd9dca3b67a71238' ;; \
*) echo >&2 "error: unsupported architecture: '$dpkgArch'"; exit 1 ;; \
esac; \
wget -O containerd.tar.gz "$url" --progress=dot:giga; \
echo "$sha256 *containerd.tar.gz" | sha256sum --strict --check -; \
tar -xvf containerd.tar.gz -C /usr/local/; \
rm containerd.tar.gz; \
containerd --version
Expand Down
25 changes: 19 additions & 6 deletions containerd/Dockerfile.template
Original file line number Diff line number Diff line change
Expand Up @@ -19,18 +19,31 @@ RUN set -eux; \
# https://github.com/opencontainers/runc/releases
ENV RUNC_VERSION {{ .runc.version }}
RUN set -eux; \
# TODO multiple architectures
wget -O /usr/local/bin/runc "https://github.com/opencontainers/runc/releases/download/v${RUNC_VERSION}/runc.amd64" --progress=dot:giga; \
# TODO .asc / .sha256sum
dpkgArch="$(dpkg --print-architecture)"; \
case "$dpkgArch" in \
{{ .runc.arches | map( -}}
{{ .dpkgArch // empty | @sh }}) url={{ .url | @sh }}; sha256={{ .sha256 | @sh }} ;; \
{{ ) | add -}}
*) echo >&2 "error: unsupported architecture: '$dpkgArch'"; exit 1 ;; \
esac; \
# TODO $url.asc (see https://github.com/opencontainers/runc/releases)
wget -O /usr/local/bin/runc "$url" --progress=dot:giga; \
echo "$sha256 */usr/local/bin/runc" | sha256sum --strict --check -; \
chmod +x /usr/local/bin/runc; \
runc --version

# https://github.com/containerd/containerd/releases
ENV CONTAINERD_VERSION {{ .version }}
RUN set -eux; \
# TODO multiple architectures
wget -O containerd.tar.gz "https://github.com/containerd/containerd/releases/download/v${CONTAINERD_VERSION}/containerd-${CONTAINERD_VERSION}-linux-amd64.tar.gz" --progress=dot:giga; \
# TODO .sha256sum
dpkgArch="$(dpkg --print-architecture)"; \
case "$dpkgArch" in \
{{ .arches | map( -}}
{{ .dpkgArch // empty | @sh }}) url={{ .url | @sh }}; sha256={{ .sha256 | @sh }} ;; \
{{ ) | add -}}
*) echo >&2 "error: unsupported architecture: '$dpkgArch'"; exit 1 ;; \
esac; \
wget -O containerd.tar.gz "$url" --progress=dot:giga; \
echo "$sha256 *containerd.tar.gz" | sha256sum --strict --check -; \
tar -xvf containerd.tar.gz -C /usr/local/; \
rm containerd.tar.gz; \
containerd --version
Expand Down
17 changes: 17 additions & 0 deletions containerd/gsl.sh
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,27 @@ dir="$(basename "$PWD")"
cd ..

version="$(jq -r '.version' "$dir/versions.json")"
from="$(awk '$1 == "FROM" { print $2; exit }' "$dir/Dockerfile")" # TODO multi-stage build??
fromArches="$(bashbrew remote arches --json "$from" | jq -c '.arches | keys')"
arches="$(jq -r -L "$dir/../.libs" --argjson fromArches "$fromArches" '
include "lib"
;
[
$fromArches,
(.arches | map_values(select(.dpkgArch)) | keys),
(.runc.arches | map_values(select(.dpkgArch)) | keys),
empty
]
| intersection
| join(", ")
' "$dir/versions.json")"
[ -n "$arches" ]

source gsl-libs.sh

globalEntry
echo "Architectures: $arches"

versionedTagsEntry "$dir" "$version" latest

dockerfile='Dockerfile.c8dind' versionedVariantEntry "$dir" c8dind "$version" c8dind
66 changes: 65 additions & 1 deletion containerd/versions.json
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,75 @@
"ref": "refs/tags/v1.7.5^{}",
"tag": "v1.7.5",
"version": "1.7.5",
"arches": {
"amd64": {
"url": "https://github.com/containerd/containerd/releases/download/v1.7.5/containerd-1.7.5-linux-amd64.tar.gz",
"sha256": "33609ae2d5838bc5798306a1ac30d7f2c6a8cff785ca6253d2be8a8b3ccbab25",
"dpkgArch": "amd64"
},
"arm64v8": {
"url": "https://github.com/containerd/containerd/releases/download/v1.7.5/containerd-1.7.5-linux-arm64.tar.gz",
"sha256": "98fc6990820d52d45b56ea2cda808157d4e61bb30ded96887634644c03025fa9",
"dpkgArch": "arm64"
},
"ppc64le": {
"url": "https://github.com/containerd/containerd/releases/download/v1.7.5/containerd-1.7.5-linux-ppc64le.tar.gz",
"sha256": "2496e24a95fa74750363a8a7e2ac36acf8d41ee2e4b67a452154ad4c8efbc4bc",
"dpkgArch": "ppc64el"
},
"riscv64": {
"url": "https://github.com/containerd/containerd/releases/download/v1.7.5/containerd-1.7.5-linux-riscv64.tar.gz",
"sha256": "9b2835df49f8fd9c2562a9173f2cd33412712e0e8e0e792fa8d187dec901bbb1",
"dpkgArch": "riscv64"
},
"s390x": {
"url": "https://github.com/containerd/containerd/releases/download/v1.7.5/containerd-1.7.5-linux-s390x.tar.gz",
"sha256": "0cd81b8844a2854b746af0c30d00fe3df5018e0985c56175dd9dca3b67a71238",
"dpkgArch": "s390x"
},
"windows-amd64": {
"url": "https://github.com/containerd/containerd/releases/download/v1.7.5/containerd-1.7.5-windows-amd64.tar.gz",
"sha256": "48f3dc0d7d54f92c65573d5d6545f6bbe21f16967f817fef8cdecee62d0a1970",
"dpkgArch": null
}
},
"runc": {
"commit": "ccaecfcbc907d70a7aa870a6650887b901b25b82",
"ref": "refs/tags/v1.1.9^{}",
"tag": "v1.1.9",
"version": "1.1.9"
"version": "1.1.9",
"arches": {
"amd64": {
"url": "https://github.com/opencontainers/runc/releases/download/v1.1.9/runc.amd64",
"sha256": "b9bfdd4cb27cddbb6172a442df165a80bfc0538a676fbca1a6a6c8f4c6933b43",
"dpkgArch": "amd64"
},
"arm64v8": {
"url": "https://github.com/opencontainers/runc/releases/download/v1.1.9/runc.arm64",
"sha256": "b43e9f561e85906f469eef5a7b7992fc586f750f44a0e011da4467e7008c33a0",
"dpkgArch": "arm64"
},
"arm32v7": {
"url": "https://github.com/opencontainers/runc/releases/download/v1.1.9/runc.armhf",
"sha256": "3c33909a3e2b40d879a1c07b89dc18c7dc727de61a6c381173e8c5c46b200c3d",
"dpkgArch": "armhf"
},
"ppc64le": {
"url": "https://github.com/opencontainers/runc/releases/download/v1.1.9/runc.ppc64le",
"sha256": "065cf4f84b5acc0acdb017af2955743dfb5f5e1f49a493eea3e8206f33bf6fe6",
"dpkgArch": "ppc64el"
},
"riscv64": {
"url": "https://github.com/opencontainers/runc/releases/download/v1.1.9/runc.riscv64",
"sha256": "177a64486b8ce372938692f44e4aa6a6d9d3476a1f77b152142f02cf881a4817",
"dpkgArch": "riscv64"
},
"s390x": {
"url": "https://github.com/opencontainers/runc/releases/download/v1.1.9/runc.s390x",
"sha256": "363b317d44a44591205439e29f70f94f01eb51e58047b092b03b1cd2cdad533a",
"dpkgArch": "s390x"
}
}
},
"dind": {
"version": "d58df1fc6c866447ce2cd129af10e5b507705624"
Expand Down
76 changes: 74 additions & 2 deletions containerd/versions.sh
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,80 @@ source "$dir/../.libs/git.sh"

versions_hooks+=( hook_no-prereleases )

containerd="$(git-tags 'https://github.com/containerd/containerd.git')"
runc="$(git-tags 'https://github.com/opencontainers/runc.git')"
containerd="$(
# https://github.com/containerd/containerd/releases
upstreamArches=(
amd64
arm64v8
ppc64le
riscv64
s390x
windows-amd64
)
hook_containerd-arches() {
local version="$3"
local json='{}' arch
for arch in "${upstreamArches[@]}"; do
local upstreamArch
case "$arch" in
arm64v8) upstreamArch='linux-arm64' ;;
windows-*) upstreamArch="$arch" ;;
*) upstreamArch="linux-$arch" ;;
esac
local sha256 url="https://github.com/containerd/containerd/releases/download/v$version/containerd-$version-$upstreamArch.tar.gz"
sha256="$(wget -qO- "$url.sha256sum")" || continue
sha256="${sha256%% *}"
json="$(jq <<<"$json" -c --arg arch "$arch" --arg url "$url" --arg sha256 "$sha256" -L"$dir/../.libs" '
include "lib"
;
.arches[$arch] = {
url: $url,
sha256: $sha256,
dpkgArch: ($arch | deb_arch),
}
')"
done
jq <<<"$json" -e '.arches? | has("amd64") and has("arm64v8")' > /dev/null || return 1
[ "$json" = '{}' ] || printf '%s\n' "$json"
}
versions_hooks+=( hook_containerd-arches )
git-tags 'https://github.com/containerd/containerd.git'
)"

runc="$(
hook_runc-arches() {
local version="$3"
local sha256 urlBase="https://github.com/opencontainers/runc/releases/download/v$version"
sha256="$(wget -qO- "$urlBase/runc.sha256sum")" || return 1
local json
json="$(jq <<<"$sha256" -csR --arg urlBase "$urlBase" -L"$dir/../.libs" '
include "lib"
;
split("\n")
| map(
capture("^(?<sha256>[0-9a-f]{64}) [ *](?<file>runc.(?<arch>[^.]+))$")
| select(.arch != "armel") # https://github.com/opencontainers/runc/blob/8feecba2bb293267c0dee854c86d291852b86388/script/lib.sh#L14-L16 ("arm-linux-gnueabi" + GOARM=6 does not make a ton of sense but also does not map to a Debian arch, so we do not need it)
| .arch |= ({
arm64: "arm64v8",
armhf: "arm32v7",
}[.] // .)
| { (.arch): {
url: ($urlBase + "/" + .file),
sha256: .sha256,
dpkgArch: (.arch | deb_arch),
} }
)
| { arches: add }
')" || return 1
jq <<<"$json" -e '.arches | has("amd64") and has("arm64v8")' > /dev/null || return 1
printf '%s\n' "$json"
}
versions_hooks+=( hook_runc-arches )
git-tags 'https://github.com/opencontainers/runc.git'
)"

dind="$(github-file-commit 'moby/moby' 'HEAD' 'hack/dind')"

jq <<<"$containerd" --argjson runc "$runc" --argjson dind "$dind" '
Expand Down

0 comments on commit 1531fa8

Please sign in to comment.