Merge pull request #507 from self-five/c8d-multiarch

Update containerd to be properly multiarch
tianon · Sep 3, 2023 · c11d566 · c11d566
2 parents 6afd7cf + 1531fa8
commit c11d566
Show file tree

Hide file tree

Showing 6 changed files with 225 additions and 15 deletions.
diff --git a/.libs/lib.jq b/.libs/lib.jq
@@ -0,0 +1,26 @@
+def unique_unsorted:
+	# https://unix.stackexchange.com/a/738744/153467
+	reduce .[] as $a ([]; if IN(.[]; $a) then . else . += [$a] end)
+;
+def intersection:
+	# add all arrays together and bit-by-bit remove anything unique from each array
+	reduce .[] as $arr (add | unique_unsorted; . - (. - $arr))
+;
+def deb_arch:
+	{
+		# https://salsa.debian.org/dpkg-team/dpkg/-/blob/main/data/cputable
+		# https://wiki.debian.org/ArchitectureSpecificsMemo#Architecture_baselines
+		# http://deb.debian.org/debian/dists/unstable/main/
+		# http://deb.debian.org/debian/dists/stable/main/
+		# https://deb.debian.org/debian-ports/dists/unstable/main/
+		amd64: "amd64",
+		arm32v5: "armel",
+		arm32v7: "armhf",
+		arm64v8: "arm64",
+		i386: "i386",
+		mips64le: "mips64el",
+		ppc64le: "ppc64el",
+		riscv64: "riscv64",
+		s390x: "s390x",
+	}[.]
+;
diff --git a/containerd/Dockerfile b/containerd/Dockerfile
@@ -25,18 +25,36 @@ RUN set -eux; \
 # https://github.com/opencontainers/runc/releases
 ENV RUNC_VERSION 1.1.9
 RUN set -eux; \
-# TODO multiple architectures
-	wget -O /usr/local/bin/runc "https://github.com/opencontainers/runc/releases/download/v${RUNC_VERSION}/runc.amd64" --progress=dot:giga; \
-# TODO .asc / .sha256sum
+	dpkgArch="$(dpkg --print-architecture)"; \
+	case "$dpkgArch" in \
+		'amd64') url='https://github.com/opencontainers/runc/releases/download/v1.1.9/runc.amd64'; sha256='b9bfdd4cb27cddbb6172a442df165a80bfc0538a676fbca1a6a6c8f4c6933b43' ;; \
+		'arm64') url='https://github.com/opencontainers/runc/releases/download/v1.1.9/runc.arm64'; sha256='b43e9f561e85906f469eef5a7b7992fc586f750f44a0e011da4467e7008c33a0' ;; \
+		'armhf') url='https://github.com/opencontainers/runc/releases/download/v1.1.9/runc.armhf'; sha256='3c33909a3e2b40d879a1c07b89dc18c7dc727de61a6c381173e8c5c46b200c3d' ;; \
+		'ppc64el') url='https://github.com/opencontainers/runc/releases/download/v1.1.9/runc.ppc64le'; sha256='065cf4f84b5acc0acdb017af2955743dfb5f5e1f49a493eea3e8206f33bf6fe6' ;; \
+		'riscv64') url='https://github.com/opencontainers/runc/releases/download/v1.1.9/runc.riscv64'; sha256='177a64486b8ce372938692f44e4aa6a6d9d3476a1f77b152142f02cf881a4817' ;; \
+		's390x') url='https://github.com/opencontainers/runc/releases/download/v1.1.9/runc.s390x'; sha256='363b317d44a44591205439e29f70f94f01eb51e58047b092b03b1cd2cdad533a' ;; \
+		*) echo >&2 "error: unsupported architecture: '$dpkgArch'"; exit 1 ;; \
+	esac; \
+# TODO $url.asc (see https://github.com/opencontainers/runc/releases)
+	wget -O /usr/local/bin/runc "$url" --progress=dot:giga; \
+	echo "$sha256 */usr/local/bin/runc" | sha256sum --strict --check -; \
 	chmod +x /usr/local/bin/runc; \
 	runc --version
 
 # https://github.com/containerd/containerd/releases
 ENV CONTAINERD_VERSION 1.7.5
 RUN set -eux; \
-# TODO multiple architectures
-	wget -O containerd.tar.gz "https://github.com/containerd/containerd/releases/download/v${CONTAINERD_VERSION}/containerd-${CONTAINERD_VERSION}-linux-amd64.tar.gz" --progress=dot:giga; \
-# TODO .sha256sum
+	dpkgArch="$(dpkg --print-architecture)"; \
+	case "$dpkgArch" in \
+		'amd64') url='https://github.com/containerd/containerd/releases/download/v1.7.5/containerd-1.7.5-linux-amd64.tar.gz'; sha256='33609ae2d5838bc5798306a1ac30d7f2c6a8cff785ca6253d2be8a8b3ccbab25' ;; \
+		'arm64') url='https://github.com/containerd/containerd/releases/download/v1.7.5/containerd-1.7.5-linux-arm64.tar.gz'; sha256='98fc6990820d52d45b56ea2cda808157d4e61bb30ded96887634644c03025fa9' ;; \
+		'ppc64el') url='https://github.com/containerd/containerd/releases/download/v1.7.5/containerd-1.7.5-linux-ppc64le.tar.gz'; sha256='2496e24a95fa74750363a8a7e2ac36acf8d41ee2e4b67a452154ad4c8efbc4bc' ;; \
+		'riscv64') url='https://github.com/containerd/containerd/releases/download/v1.7.5/containerd-1.7.5-linux-riscv64.tar.gz'; sha256='9b2835df49f8fd9c2562a9173f2cd33412712e0e8e0e792fa8d187dec901bbb1' ;; \
+		's390x') url='https://github.com/containerd/containerd/releases/download/v1.7.5/containerd-1.7.5-linux-s390x.tar.gz'; sha256='0cd81b8844a2854b746af0c30d00fe3df5018e0985c56175dd9dca3b67a71238' ;; \
+		*) echo >&2 "error: unsupported architecture: '$dpkgArch'"; exit 1 ;; \
+	esac; \
+	wget -O containerd.tar.gz "$url" --progress=dot:giga; \
+	echo "$sha256 *containerd.tar.gz" | sha256sum --strict --check -; \
 	tar -xvf containerd.tar.gz -C /usr/local/; \
 	rm containerd.tar.gz; \
 	containerd --version

diff --git a/containerd/Dockerfile.template b/containerd/Dockerfile.template
@@ -19,18 +19,31 @@ RUN set -eux; \
 # https://github.com/opencontainers/runc/releases
 ENV RUNC_VERSION {{ .runc.version }}
 RUN set -eux; \
-# TODO multiple architectures
-	wget -O /usr/local/bin/runc "https://github.com/opencontainers/runc/releases/download/v${RUNC_VERSION}/runc.amd64" --progress=dot:giga; \
-# TODO .asc / .sha256sum
+	dpkgArch="$(dpkg --print-architecture)"; \
+	case "$dpkgArch" in \
+{{ .runc.arches | map( -}}
+		{{ .dpkgArch // empty | @sh }}) url={{ .url | @sh }}; sha256={{ .sha256 | @sh }} ;; \
+{{ ) | add -}}
+		*) echo >&2 "error: unsupported architecture: '$dpkgArch'"; exit 1 ;; \
+	esac; \
+# TODO $url.asc (see https://github.com/opencontainers/runc/releases)
+	wget -O /usr/local/bin/runc "$url" --progress=dot:giga; \
+	echo "$sha256 */usr/local/bin/runc" | sha256sum --strict --check -; \
 	chmod +x /usr/local/bin/runc; \
 	runc --version
 
 # https://github.com/containerd/containerd/releases
 ENV CONTAINERD_VERSION {{ .version }}
 RUN set -eux; \
-# TODO multiple architectures
-	wget -O containerd.tar.gz "https://github.com/containerd/containerd/releases/download/v${CONTAINERD_VERSION}/containerd-${CONTAINERD_VERSION}-linux-amd64.tar.gz" --progress=dot:giga; \
-# TODO .sha256sum
+	dpkgArch="$(dpkg --print-architecture)"; \
+	case "$dpkgArch" in \
+{{ .arches | map( -}}
+		{{ .dpkgArch // empty | @sh }}) url={{ .url | @sh }}; sha256={{ .sha256 | @sh }} ;; \
+{{ ) | add -}}
+		*) echo >&2 "error: unsupported architecture: '$dpkgArch'"; exit 1 ;; \
+	esac; \
+	wget -O containerd.tar.gz "$url" --progress=dot:giga; \
+	echo "$sha256 *containerd.tar.gz" | sha256sum --strict --check -; \
 	tar -xvf containerd.tar.gz -C /usr/local/; \
 	rm containerd.tar.gz; \
 	containerd --version

diff --git a/containerd/gsl.sh b/containerd/gsl.sh
@@ -6,10 +6,27 @@ dir="$(basename "$PWD")"
 cd ..
 
 version="$(jq -r '.version' "$dir/versions.json")"
+from="$(awk '$1 == "FROM" { print $2; exit }' "$dir/Dockerfile")" # TODO multi-stage build??
+fromArches="$(bashbrew remote arches --json "$from" | jq -c '.arches | keys')"
+arches="$(jq -r -L "$dir/../.libs" --argjson fromArches "$fromArches" '
+	include "lib"
+	;
+	[
+		$fromArches,
+		(.arches | map_values(select(.dpkgArch)) | keys),
+		(.runc.arches | map_values(select(.dpkgArch)) | keys),
+		empty
+	]
+	| intersection
+	| join(", ")
+' "$dir/versions.json")"
+[ -n "$arches" ]
 
 source gsl-libs.sh
 
 globalEntry
+echo "Architectures: $arches"
 
 versionedTagsEntry "$dir" "$version" latest
+
 dockerfile='Dockerfile.c8dind' versionedVariantEntry "$dir" c8dind "$version" c8dind
diff --git a/containerd/versions.json b/containerd/versions.json
@@ -3,11 +3,75 @@
   "ref": "refs/tags/v1.7.5^{}",
   "tag": "v1.7.5",
   "version": "1.7.5",
+  "arches": {
+    "amd64": {
+      "url": "https://github.com/containerd/containerd/releases/download/v1.7.5/containerd-1.7.5-linux-amd64.tar.gz",
+      "sha256": "33609ae2d5838bc5798306a1ac30d7f2c6a8cff785ca6253d2be8a8b3ccbab25",
+      "dpkgArch": "amd64"
+    },
+    "arm64v8": {
+      "url": "https://github.com/containerd/containerd/releases/download/v1.7.5/containerd-1.7.5-linux-arm64.tar.gz",
+      "sha256": "98fc6990820d52d45b56ea2cda808157d4e61bb30ded96887634644c03025fa9",
+      "dpkgArch": "arm64"
+    },
+    "ppc64le": {
+      "url": "https://github.com/containerd/containerd/releases/download/v1.7.5/containerd-1.7.5-linux-ppc64le.tar.gz",
+      "sha256": "2496e24a95fa74750363a8a7e2ac36acf8d41ee2e4b67a452154ad4c8efbc4bc",
+      "dpkgArch": "ppc64el"
+    },
+    "riscv64": {
+      "url": "https://github.com/containerd/containerd/releases/download/v1.7.5/containerd-1.7.5-linux-riscv64.tar.gz",
+      "sha256": "9b2835df49f8fd9c2562a9173f2cd33412712e0e8e0e792fa8d187dec901bbb1",
+      "dpkgArch": "riscv64"
+    },
+    "s390x": {
+      "url": "https://github.com/containerd/containerd/releases/download/v1.7.5/containerd-1.7.5-linux-s390x.tar.gz",
+      "sha256": "0cd81b8844a2854b746af0c30d00fe3df5018e0985c56175dd9dca3b67a71238",
+      "dpkgArch": "s390x"
+    },
+    "windows-amd64": {
+      "url": "https://github.com/containerd/containerd/releases/download/v1.7.5/containerd-1.7.5-windows-amd64.tar.gz",
+      "sha256": "48f3dc0d7d54f92c65573d5d6545f6bbe21f16967f817fef8cdecee62d0a1970",
+      "dpkgArch": null
+    }
+  },
   "runc": {
     "commit": "ccaecfcbc907d70a7aa870a6650887b901b25b82",
     "ref": "refs/tags/v1.1.9^{}",
     "tag": "v1.1.9",
-    "version": "1.1.9"
+    "version": "1.1.9",
+    "arches": {
+      "amd64": {
+        "url": "https://github.com/opencontainers/runc/releases/download/v1.1.9/runc.amd64",
+        "sha256": "b9bfdd4cb27cddbb6172a442df165a80bfc0538a676fbca1a6a6c8f4c6933b43",
+        "dpkgArch": "amd64"
+      },
+      "arm64v8": {
+        "url": "https://github.com/opencontainers/runc/releases/download/v1.1.9/runc.arm64",
+        "sha256": "b43e9f561e85906f469eef5a7b7992fc586f750f44a0e011da4467e7008c33a0",
+        "dpkgArch": "arm64"
+      },
+      "arm32v7": {
+        "url": "https://github.com/opencontainers/runc/releases/download/v1.1.9/runc.armhf",
+        "sha256": "3c33909a3e2b40d879a1c07b89dc18c7dc727de61a6c381173e8c5c46b200c3d",
+        "dpkgArch": "armhf"
+      },
+      "ppc64le": {
+        "url": "https://github.com/opencontainers/runc/releases/download/v1.1.9/runc.ppc64le",
+        "sha256": "065cf4f84b5acc0acdb017af2955743dfb5f5e1f49a493eea3e8206f33bf6fe6",
+        "dpkgArch": "ppc64el"
+      },
+      "riscv64": {
+        "url": "https://github.com/opencontainers/runc/releases/download/v1.1.9/runc.riscv64",
+        "sha256": "177a64486b8ce372938692f44e4aa6a6d9d3476a1f77b152142f02cf881a4817",
+        "dpkgArch": "riscv64"
+      },
+      "s390x": {
+        "url": "https://github.com/opencontainers/runc/releases/download/v1.1.9/runc.s390x",
+        "sha256": "363b317d44a44591205439e29f70f94f01eb51e58047b092b03b1cd2cdad533a",
+        "dpkgArch": "s390x"
+      }
+    }
   },
   "dind": {
     "version": "d58df1fc6c866447ce2cd129af10e5b507705624"

diff --git a/containerd/versions.sh b/containerd/versions.sh
@@ -9,8 +9,80 @@ source "$dir/../.libs/git.sh"
 
 versions_hooks+=( hook_no-prereleases )
 
-containerd="$(git-tags 'https://github.com/containerd/containerd.git')"
-runc="$(git-tags 'https://github.com/opencontainers/runc.git')"
+containerd="$(
+	# https://github.com/containerd/containerd/releases
+	upstreamArches=(
+		amd64
+		arm64v8
+		ppc64le
+		riscv64
+		s390x
+		windows-amd64
+	)
+	hook_containerd-arches() {
+		local version="$3"
+		local json='{}' arch
+		for arch in "${upstreamArches[@]}"; do
+			local upstreamArch
+			case "$arch" in
+				arm64v8) upstreamArch='linux-arm64' ;;
+				windows-*) upstreamArch="$arch" ;;
+				*) upstreamArch="linux-$arch" ;;
+			esac
+			local sha256 url="https://github.com/containerd/containerd/releases/download/v$version/containerd-$version-$upstreamArch.tar.gz"
+			sha256="$(wget -qO- "$url.sha256sum")" || continue
+			sha256="${sha256%% *}"
+			json="$(jq <<<"$json" -c --arg arch "$arch" --arg url "$url" --arg sha256 "$sha256" -L"$dir/../.libs" '
+				include "lib"
+				;
+				.arches[$arch] = {
+					url: $url,
+					sha256: $sha256,
+					dpkgArch: ($arch | deb_arch),
+				}
+			')"
+		done
+		jq <<<"$json" -e '.arches? | has("amd64") and has("arm64v8")' > /dev/null || return 1
+		[ "$json" = '{}' ] || printf '%s\n' "$json"
+	}
+	versions_hooks+=( hook_containerd-arches )
+
+	git-tags 'https://github.com/containerd/containerd.git'
+)"
+
+runc="$(
+	hook_runc-arches() {
+		local version="$3"
+		local sha256 urlBase="https://github.com/opencontainers/runc/releases/download/v$version"
+		sha256="$(wget -qO- "$urlBase/runc.sha256sum")" || return 1
+		local json
+		json="$(jq <<<"$sha256" -csR --arg urlBase "$urlBase" -L"$dir/../.libs" '
+			include "lib"
+			;
+			split("\n")
+			| map(
+				capture("^(?<sha256>[0-9a-f]{64}) [ *](?<file>runc.(?<arch>[^.]+))$")
+				| select(.arch != "armel") # https://github.com/opencontainers/runc/blob/8feecba2bb293267c0dee854c86d291852b86388/script/lib.sh#L14-L16 ("arm-linux-gnueabi" + GOARM=6 does not make a ton of sense but also does not map to a Debian arch, so we do not need it)
+				| .arch |= ({
+					arm64: "arm64v8",
+					armhf: "arm32v7",
+				}[.] // .)
+				| { (.arch): {
+					url: ($urlBase + "/" + .file),
+					sha256: .sha256,
+					dpkgArch: (.arch | deb_arch),
+				} }
+			)
+			| { arches: add }
+		')" || return 1
+		jq <<<"$json" -e '.arches | has("amd64") and has("arm64v8")' > /dev/null || return 1
+		printf '%s\n' "$json"
+	}
+	versions_hooks+=( hook_runc-arches )
+
+	git-tags 'https://github.com/opencontainers/runc.git'
+)"
+
 dind="$(github-file-commit 'moby/moby' 'HEAD' 'hack/dind')"
 
 jq <<<"$containerd" --argjson runc "$runc" --argjson dind "$dind" '