Feat: docker-kind-support (#428)

* add docker and kind * fix, 24.0.5 not signed yet (?) * install s6 * need the dockerfiles in the context * s6 services config for docker * docker as nonroot * rename * fix typo * fix iptables issue * add modprobe * use rootless install script * add env vars * fix * fix * fix * rm debug code * try with fuse-overlayfs * test * test dockerfile systemd * fix build * add missing iproute2 * pod yamls * jammy test with plural login * update image * sysbox priv * add user/contents.d * test podman build * this at least works with --privileged * rm dockerd s6 service, clogs logs * try on sysbox runc * shouldn't matter, but let's see * unset mount program, be specific about defaults * test with new storage.conf * simplify * for some reason this does make a difference * try with vfs * makes it worse * fix perm on volume * add back mknod * force newer version of podman, crun dep is ancient * trigger build * build with docker-ce-cli only * dind sidecar approach * clean up * create dind dockerfile with systemd conf * add make target * dont ignore systemd files * add dind build gh wf step * rm trest image dockerfile * rm debug yamnls * clean up * clean up, not needed with correct unit file * rm dependencies of podman * clean up * rm get-docker.sh was need for debugging * update yaml * delete sidecar yaml example, not needed in repo
pluralsh · Aug 22, 2023 · b79724d · b79724d
1 parent f8a468d
commit b79724d
Show file tree

Hide file tree

Showing 7 changed files with 241 additions and 1 deletion.
diff --git a/.dockerignore b/.dockerignore
@@ -15,4 +15,5 @@ packer/
 hack/
 Makefile
 Dockerfile
-dockerfiles/
+dockerfiles
+!dockerfiles/dind-config
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -206,6 +206,107 @@ jobs:
         uses: github/codeql-action/upload-sarif@v2
         with:
           sarif_file: 'trivy-results.sarif'
+  dind:
+    name: Build dind image
+    runs-on: ubuntu-latest
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+      packages: 'write'
+      security-events: write
+      actions: read
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-region: us-east-2
+          role-to-assume: arn:aws:iam::312272277431:role/github-actions/buildx-deployments
+          role-session-name: PluralCLI
+      - name: setup kubectl
+        uses: azure/setup-kubectl@v3
+      - name: Get EKS credentials
+        run: aws eks update-kubeconfig --name pluraldev
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          # list of Docker images to use as base name for tags
+          images: |
+            ghcr.io/pluralsh/plural-dind
+          # generate Docker tags based on the following events/attributes
+          tags: |
+            type=sha
+            type=ref,event=pr
+            type=ref,event=branch
+      - name: Set up Docker Buildx
+        id: builder
+        uses: docker/setup-buildx-action@v2
+        with:
+          driver: kubernetes
+          platforms: linux/amd64
+          driver-opts: |
+            namespace=buildx
+            requests.cpu=1.5
+            requests.memory=3.5Gi
+            "nodeselector=plural.sh/scalingGroup=buildx-spot-x86"
+            "tolerations=key=plural.sh/capacityType,value=SPOT,effect=NoSchedule;key=plural.sh/reserved,value=BUILDX,effect=NoSchedule"
+      - name: Append ARM buildx builder from AWS
+        run: |
+          docker buildx create \
+            --append \
+            --bootstrap \
+            --name ${{ steps.builder.outputs.name }} \
+            --driver=kubernetes \
+            --platform linux/arm64 \
+            --node=${{ steps.builder.outputs.name }}-arm64 \
+            --buildkitd-flags "--allow-insecure-entitlement security.insecure --allow-insecure-entitlement network.host" \
+            --driver-opt namespace=buildx \
+            --driver-opt requests.cpu=1.5 \
+            --driver-opt requests.memory=3.5Gi \
+            '--driver-opt="nodeselector=plural.sh/scalingGroup=buildx-spot-arm64"' \
+            '--driver-opt="tolerations=key=plural.sh/capacityType,value=SPOT,effect=NoSchedule;key=plural.sh/reserved,value=BUILDX,effect=NoSchedule"'
+      - name: Login to GHCR
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Get current date
+        id: date
+        run: echo "date=$(date -u +'%Y-%m-%dT%H:%M:%S%z')" >> $GITHUB_OUTPUT
+      - uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: ./dockerfiles/Dockerfile.dind
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          platforms: linux/amd64,linux/arm64
+          # cache-from: type=gha
+          # cache-to: type=gha,mode=max
+          build-args: |
+            APP_VSN=dev
+            APP_COMMIT=${{ github.sha }}
+            APP_DATE=${{ steps.date.outputs.date }}
+      - name: Run Trivy vulnerability scanner on dind image
+        uses: aquasecurity/trivy-action@master
+        with:
+          scan-type: 'image'
+          image-ref: ${{ fromJSON(steps.meta.outputs.json).tags[0] }}
+          hide-progress: false
+          format: 'sarif'
+          output: 'trivy-results.sarif'
+          scanners: 'vuln'
+          timeout: 10m
+          ignore-unfixed: true
+           #severity: 'CRITICAL,HIGH'
+      - name: Upload Trivy scan results to GitHub Security tab
+        uses: github/codeql-action/upload-sarif@v2
+        with:
+          sarif_file: 'trivy-results.sarif'
+
   trivy-scan:
     name: Trivy fs scan
     runs-on: ubuntu-latest

diff --git a/Makefile b/Makefile
@@ -101,6 +101,17 @@ build-cloud: ## build the cloud docker image
 		-t gcr.io/$(GCP_PROJECT)/$(APP_NAME)-cloud:$(APP_VSN) \
 		-t $(DKR_HOST)/plural/$(APP_NAME)-cloud:$(APP_VSN) -f dockerfiles/Dockerfile.cloud  .
 
+.PHONY: build-dind
+build-dind: ## build the dind docker image
+	docker build --build-arg APP_NAME=$(APP_NAME) \
+		--build-arg APP_VSN=$(APP_VSN) \
+		--build-arg APP_DATE=$(APP_DATE) \
+		--build-arg APP_COMMIT=$(BUILD) \
+		-t $(APP_NAME)-cloud:$(APP_VSN) \
+		-t $(APP_NAME)-cloud:latest \
+		-t gcr.io/$(GCP_PROJECT)/$(APP_NAME)-cloud:$(APP_VSN) \
+		-t $(DKR_HOST)/plural/$(APP_NAME)-dind:$(APP_VSN) -f dockerfiles/Dockerfile.dind  .
+
 .PHONY: push
 push: ## push to gcr
 	docker push gcr.io/$(GCP_PROJECT)/$(APP_NAME):$(APP_VSN)

diff --git a/dockerfiles/Dockerfile.cloud b/dockerfiles/Dockerfile.cloud
@@ -110,6 +110,22 @@ RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.
       google-cloud-sdk-gke-gcloud-auth-plugin && \
       gcloud --help
 
+
+# install kind
+# K8s.io KinD
+ENV KIND_VERSION=v0.20.0
+RUN curl -Lo ./kind https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-linux-amd64 \
+    && chmod +x ./kind \
+    && mv ./kind /usr/bin/kind
+
+# install docker cli
+RUN install -m 0755 -d /etc/apt/keyrings && \
+    curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg && \
+    chmod a+r /etc/apt/keyrings/docker.gpg && \
+    echo "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null && \
+    apt-get -yq update && apt-get -yq install docker-ce-cli
+
+
 WORKDIR /root
 
 ENV TERM=xterm-256color

diff --git a/dockerfiles/Dockerfile.dind b/dockerfiles/Dockerfile.dind
@@ -0,0 +1,58 @@
+FROM ubuntu:jammy
+
+#
+# Systemd installation
+#
+RUN apt-get update &&                            \
+    apt-get install -y --no-install-recommends   \
+            systemd                              \
+            systemd-sysv                         \
+            libsystemd0                          \
+            ca-certificates                      \
+            dbus                                 \
+            iptables                             \
+            iproute2                             \
+            kmod                                 \
+            locales                              \
+            sudo                                 \
+            udev &&                              \
+                                                 \
+    # Prevents journald from reading kernel messages from /dev/kmsg
+    echo "ReadKMsg=no" >> /etc/systemd/journald.conf &&               \
+                                                                      \
+    # Housekeeping
+    apt-get clean -y &&                                               \
+    rm -rf                                                            \
+       /var/cache/debconf/*                                           \
+       /var/lib/apt/lists/*                                           \
+       /var/log/*                                                     \
+       /tmp/*                                                         \
+       /var/tmp/*                                                     \
+       /usr/share/doc/*                                               \
+       /usr/share/man/*                                               \
+       /usr/share/local/*                                          
+
+# Disable systemd services/units that are unnecessary within a container.
+RUN systemctl mask systemd-udevd.service \
+                   systemd-udevd-kernel.socket \
+                   systemd-udevd-control.socket \
+                   systemd-modules-load.service \
+                   sys-kernel-debug.mount \
+                   sys-kernel-tracing.mount
+
+# Make use of stopsignal (instead of sigterm) to stop systemd containers.
+STOPSIGNAL SIGRTMIN+3
+
+
+# Install Docker
+RUN apt-get update && apt-get install -y curl \
+    && rm -rf /var/lib/apt/lists/* \
+    && curl -fsSL https://get.docker.com -o get-docker.sh && sh get-docker.sh \
+ADD https://raw.githubusercontent.com/docker/docker-ce/master/components/cli/contrib/completion/bash/docker /etc/bash_completion.d/docker.sh
+
+COPY dockerfiles/dind-config/docker.service /lib/systemd/system/docker.service
+COPY dockerfiles/dind-config/daemon.json /etc/docker/daemon.json
+
+
+# Set systemd as entrypoint.
+ENTRYPOINT [ "/sbin/init", "--log-level=err" ]
diff --git a/dockerfiles/dind-config/daemon.json b/dockerfiles/dind-config/daemon.json
@@ -0,0 +1,3 @@
+{
+    "hosts": ["tcp://0.0.0.0:2375", "unix:///var/run/docker.sock"]
+}
diff --git a/dockerfiles/dind-config/docker.service b/dockerfiles/dind-config/docker.service
@@ -0,0 +1,50 @@
+[Unit]
+Description=Docker Application Container Engine
+Documentation=https://docs.docker.com
+After=network-online.target docker.socket firewalld.service containerd.service time-set.target
+Wants=network-online.target containerd.service
+Requires=docker.socket
+
+[Service]
+Type=notify
+# the default is not to use systemd for cgroups because the delegate issues still
+# exists and systemd currently does not support the cgroup feature set required
+# for containers run by docker
+#ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock 
+#### BEGIN MOD ####
+ExecStart=/usr/bin/dockerd --config-file /etc/docker/daemon.json --containerd=/run/containerd/containerd.sock
+#### END MOD ######
+ExecReload=/bin/kill -s HUP $MAINPID
+TimeoutStartSec=0
+RestartSec=2
+Restart=always
+
+# Note that StartLimit* options were moved from "Service" to "Unit" in systemd 229.
+# Both the old, and new location are accepted by systemd 229 and up, so using the old location
+# to make them work for either version of systemd.
+StartLimitBurst=3
+
+# Note that StartLimitInterval was renamed to StartLimitIntervalSec in systemd 230.
+# Both the old, and new name are accepted by systemd 230 and up, so using the old name to make
+# this option work for either version of systemd.
+StartLimitInterval=60s
+
+# Having non-zero Limit*s causes performance problems due to accounting overhead
+# in the kernel. We recommend using cgroups to do container-local accounting.
+LimitNOFILE=infinity
+LimitNPROC=infinity
+LimitCORE=infinity
+
+# Comment TasksMax if your systemd version does not support it.
+# Only systemd 226 and above support this option.
+TasksMax=infinity
+
+# set delegate yes so that systemd does not reset the cgroups of docker containers
+Delegate=yes
+
+# kill only the docker process, not all processes in the cgroup
+KillMode=process
+OOMScoreAdjust=-500
+
+[Install]
+WantedBy=multi-user.target