Initial release

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,27 @@
+<!--
+Thank you for your pull request. Please provide a description above and
+review the checklist below.
+
+Contributors guide: ./CONTRIBUTING.md
+-->
+
+## Checklist
+<!--
+Remove items that do not apply. For completed items, change [ ] to [x].
+-->
+
+- [ ] The PR has a meaningful title. It will be used to auto-generate the
+      changelog.
+      The PR has a meaningful description that sums up the change. It will be
+      linked in the changelog.
+- [ ] PR contains a single logical change (to build a better changelog).
+- [ ] Update the documentation.
+- [ ] Categorize the PR by adding one of the following labels so it shows up
+      in the correct changelog section:
+      `bug`, `enhancement`, `documentation`, `change`, `breaking`, `dependency`, `internal`
+- [ ] Link this PR to related issues or PRs.
+
+<!--
+NOTE: these things are not required to open a PR and can be done afterwards,
+while the PR is open.
+-->

.github/changelog-configuration.json

@@ -0,0 +1,34 @@
+{
+    "pr_template": "- ${{TITLE}} (#${{NUMBER}})",
+    "categories": [
+        {
+            "title": "## 🔎 Breaking Changes",
+            "labels": ["breaking"]
+        },
+        {
+            "title": "## 🚀 Features",
+            "labels": ["enhancement", "feature"]
+        },
+        {
+            "title": "## 🛠️ Minor Changes",
+            "labels": ["change"]
+        },
+        {
+            "title": "## 🪛 Internal Changes",
+            "labels": ["internal"]
+        },
+        {
+            "title": "## 🐛 Fixes",
+            "labels": ["bug", "fix"]
+        },
+        {
+            "title": "## 📄 Documentation",
+            "labels": ["documentation"]
+        },
+        {
+            "title": "## 🔗 Dependency Updates",
+            "labels": ["dependency"]
+        }
+    ],
+    "template": "${{CATEGORIZED_COUNT}} changes since ${{FROM_TAG}}\n\n${{CHANGELOG}}"
+}

.github/workflows/gitlab-container-image.yml

@@ -0,0 +1,48 @@
+---
+name: Build & Push GitLab Container Image
+
+"on":
+  push:
+    branches:
+      - main
+    tags:
+      - v*
+  pull_request: {}
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    env:
+      IMAGE: ${{ github.repository }}/gitlab
+    steps:
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Set image version latest
+        if: github.ref == 'refs/heads/main'
+        run: echo "VERSION=latest" >> ${GITHUB_ENV}
+      - name: Set image version for PRs to branch name
+        if: github.event_name == 'pull_request'
+        run: echo "VERSION=${GITHUB_HEAD_REF//\//-}" >> ${GITHUB_ENV}
+      - name: Set image version from tag
+        if: startsWith(github.ref, 'refs/tags/v')
+        run: echo "VERSION=$(echo ${GITHUB_REF#refs/tags/})" >> ${GITHUB_ENV}
+
+      - name: Login to ghcr.io
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push
+        id: docker_build
+        uses: docker/build-push-action@v6
+        with:
+          context: '{{defaultContext}}:gitlab'
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: 'ghcr.io/${{ env.IMAGE }}:${{ env.VERSION }}'

.github/workflows/prune-old-container-images.yml

@@ -0,0 +1,19 @@
+---
+name: Prune old container images
+
+"on":
+  schedule:
+    - cron: "17 8 * * 1-5"
+  workflow_dispatch: {}
+
+jobs:
+  prune:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/delete-package-versions@v5
+        with:
+          package-name: "${{ github.repository }}-gitlab"
+          package-type: container
+          # never delete release versions or latest
+          ignore-versions: "^(v.*|latest)$"
+          min-versions-to-keep: 3

.github/workflows/release.yml

@@ -0,0 +1,34 @@
+---
+name: Release
+"on":
+  push:
+    tags:
+      - v*
+
+jobs:
+  dist:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: '0'
+      - name: Build changelog from PRs with labels
+        id: build_changelog
+        uses: mikepenz/release-changelog-builder-action@v4
+        with:
+          configuration: '.github/changelog-configuration.json'
+          # PreReleases still get a changelog, but the next full release gets a
+          # diff since the last full release, combining possible changelogs of
+          # all previous PreReleases in between. PreReleases show a partial
+          # changelog since last PreRelease.
+          ignorePreReleases: "${{ !contains(github.ref, '-rc') }}"
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Create Release
+        uses: ncipollo/release-action@v1
+        with:
+          body: ${{steps.build_changelog.outputs.changelog}}
+          prerelease: "${{ contains(github.ref, '-rc') }}"
+          # Ensure target branch for release is "main"
+          commit: main
+          token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/test.yml

@@ -0,0 +1,33 @@
+---
+name: Lint & Test
+
+"on":
+  pull_request: {}
+
+jobs:
+  yamllint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: reviewdog/action-yamllint@v1
+        with:
+          github_token: ${{ secrets.github_token }}
+          reporter: github-pr-review
+
+  jsonnetfmt:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - run: |
+          make -C gitlab jsonnetfmt_check
+
+  shellcheck:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: reviewdog/action-shellcheck@v1
+        with:
+          reporter: github-pr-review
+          pattern: |
+            *.sh
+            step-*

.yamllint.yml

@@ -0,0 +1,6 @@
+---
+extends: default
+rules:
+  line-length: disable
+  trailing-spaces:
+    level: warning

README.md

@@ -2,3 +2,6 @@
 
 This repository holds CI pipeline definitions that are suitable for compiling
 Project Syn cluster catalogs from a Project Syn tenant repository.
+
+Currently, the repository only contains CI pipeline definitions for GitLab.
+See [gitlab/README.md](/gitlab/README.md) for details.

gitlab/Dockerfile

@@ -0,0 +1,5 @@
+FROM docker.io/library/alpine:latest
+
+RUN apk add --no-cache yamllint jsonnet jq curl
+COPY ./bin/* /usr/local/bin/
+COPY ./commodore-compile.jsonnet /opt/commodore-compile.jsonnet

gitlab/Makefile

@@ -0,0 +1,14 @@
+MAKEFLAGS += --warn-undefined-variables
+SHELL := bash
+.SHELLFLAGS := -eu -o pipefail -c
+
+include Makefile.vars.mk
+
+.PHONY: jsonnetfmt_check jsonnetfmt
+jsonnetfmt_check: JSONNET_ENTRYPOINT=jsonnetfmt
+jsonnetfmt_check:
+	$(JSONNET_DOCKER) --test --pad-arrays -- *.jsonnet
+
+jsonnetfmt: JSONNET_ENTRYPOINT=jsonnetfmt
+jsonnetfmt:
+	$(JSONNET_DOCKER) --in-place --pad-arrays -- *.jsonnet

gitlab/Makefile.vars.mk

@@ -0,0 +1,12 @@
+ifneq "$(shell which docker 2>/dev/null)" ""
+	DOCKER_CMD    ?= $(shell which docker)
+	DOCKER_USERNS ?= ""
+else
+	DOCKER_CMD    ?= podman
+	DOCKER_USERNS ?= keep-id
+endif
+DOCKER_ARGS ?= run --rm -u "$$(id -u):$$(id -g)" --userns=$(DOCKER_USERNS) -w /work -e HOME="/work"
+
+JSONNET_IMAGE      ?= docker.io/bitnami/jsonnet:latest
+JSONNET_ENTRYPOINT ?= bash
+JSONNET_DOCKER     ?= $(DOCKER_CMD) $(DOCKER_ARGS) -v "$${PWD}:/work" --entrypoint=$(JSONNET_ENTRYPOINT) $(JSONNET_IMAGE)

gitlab/README.md

@@ -0,0 +1,109 @@
+# GitLab Commodore compile pipeline
+
+This is a pipeline definition which is suitable to build and push commodore catalogs from a Project Syn tenant repository hosted on a GitLab instance.
+
+Features:
+
+* Show diffs on commits, MRs and master/main
+* Push changes automatically from master/main
+* Override CI job memory limits for individual clusters (if using the GitLab K8s runner)
+
+## Usage
+
+The pipeline can be used in a Project Syn tenant repository as follows:
+
+1. Copy the `gitlab-ci.yml.default` from this directory to `.gitlab-ci.yml` in the tenant repo.
+1. Add the clusters to compile to the `CLUSTERS` in the `.gitlab-ci.yml` in the tenant repo.
+1. Create a project access token for the cluster catalog repository of each cluster listed in `CLUSTERS`.
+   Set the "role" to "Maintainer" and select the `write_repository` scope.
+1. Create a CI/CD variable named `ACCESS_TOKEN_CLUSTERNAME` for each cluster in `CLUSTERS`, where `CLUSTERNAME` is the name of the cluster with `-` replaced by `_`. 
+   Set each variable's value to the corresponding catalog project access token you created before.
+1. Create CI/CD variables `COMMODORE_API_URL` and `COMMODORE_API_TOKEN` which contain the Lieutenant API URL and a suitable API token for the tenant.
+
+> [!NOTE]
+> Project access tokens for catalog repositories are only required for cluster catalog repositories which are hosted on the same GitLab instance as the tenant repo.
+> See below for configuring access to external cluster catalog repositories via SSH.
+
+> [!TIP]
+> If the pipeline needs to clone projects other than the cluster's catalog repo from the local GitLab instance, you need to deactivate the feature _"Limit access to this project"_ in "Settings > CI/CD > Token Access" on those repositories.
+> Alternatively, you can allow access for the job tokens of each tenant repository that needs to access the project.
+
+> [!TIP]
+> Lieutenant supports managing the CI pipeline configuration for "managed" tenant and cluster catalog repositories.
+> See the [Lieutenant documentation](link todo) for details.
+
+### Commodore API Token
+
+To get the `COMMODORE_API_TOKEN`, connect to the Kubernetes cluster hosting your Lieutenant instance and run the following command:
+
+```bash
+TENANT_NAME=t-tenant-id-1234 # Replace with actual tenant id
+kubectl get secret -n lieutenant ${TENANT_NAME} -o go-template='{{.data.token|base64decode}}'
+```
+
+Alternatively, configure the Tenant to manage the `COMMODORE_API_TOKEN` CI/CD variable by adding the following in the `Tenant` resource (for example with `kubectl -n lieutenant edit tenant t-tenant-id-1234`):
+
+```bash
+spec:
+  gitRepoTemplate:
+    ciVariables:
+      - name: COMMODORE_API_TOKEN
+        valueFrom:
+          secretKeyRef:
+            key: token
+            name: t-tenant-id-1234
+        gitlabOptions:
+          masked: true
+```
+
+### External cluster catalog
+
+If the cluster catalog is not hosted on the same GitLab instance as the tenant repo, you can specify an SSH key which has access to the cluster catalog and the relevant known hosts entry via CI/CD variables on the tenant repo:
+
+1. Create a CI/CD variable named `SSH_PRIVATE_KEY` containing the SSH private key.
+1. Create a CI/CD varaible named `SSH_KNOWN_HOSTS` containing the know hosts entry.
+1. (optional) Create a CI/CD variable named `SSH_CONFIG` containing any required SSH configuration.
+
+### Test new pipeline generation image
+
+The image used to generate the compile and deploy pipelines can be adjusted by setting the following variables.
+
+```yaml
+variables:
+  PIPELINE_GENERATION_IMAGE_NAME: ghcr.io/projectsyn/commodore-compile-pipelines/gitlab
+  PIPELINE_GENERATION_IMAGE_TAG: mytestbranch
+```
+
+## FAQ
+
+### How can the compile pipeline fetch components hosted on the local GitLab instance
+
+The pipeline is configured to use the GitLab CI `CI_JOB_TOKEN` token when fetching repos from the local GitLab instance.
+The `CI_JOB_TOKEN` token has the same permissions to access the API as the user that caused the job to run.
+Therefore, the compile pipeline can access components hosted in all GitLab projects to which that user has access.
+
+
+### Why do pipelines for some MRs fail
+
+One common cause for pipeline failures for MRs is that the GitLab user who created the MR doesn't have access to the cluster catalog repo or another repo hosted on the local GitLab instance.
+To fix the issue:
+
+* Add the MR creator to the cluster catalog repositories as a "Developer" for read-only, or "Maintainer" for read-write access.
+* Add the MR creator to other repositories as a "Developer".
+
+### Configure cpu requests and limits
+
+The following options can be configured as CI/CD variables when the GitLab instance uses a K8s CI runner:
+
+* `CPU_REQUESTS`, which defaults to `800m`
+* `CPU_LIMITS`, which defaults to `2`
+* `MEMORY_LIMITS`, which defaults to `2Gi`
+
+The job generator expects that each of these variables has space-separated entries of the form `c-cluster-id-1234=value` if it's present.
+
+Example:
+
+```yaml
+variables:
+  MEMORY_LIMITS: "c-my-cluster=3Gi c-my-other-cluster=3Gi"
+```

gitlab/bin/step-render-pipeline

@@ -0,0 +1,19 @@
+#!/bin/sh
+set -e -u -x
+
+cluster_catalog_urls=""
+for c in ${CLUSTERS}; do
+  r=$(curl -sH"Authorization: Bearer ${COMMODORE_API_TOKEN}" "${COMMODORE_API_URL}/clusters/${c}" | jq -r .gitRepo.url)
+  cluster_catalog_urls="${cluster_catalog_urls}${c}=${r} "
+done
+
+jsonnet \
+  --ext-str clusters="${CLUSTERS}" \
+  --ext-str memory_limits="${MEMORY_LIMITS:-}" \
+  --ext-str cpu_limits="${CPU_LIMITS:-}" \
+  --ext-str cpu_requests="${CPU_REQUESTS:-}" \
+  --ext-str cluster_catalog_urls="${cluster_catalog_urls}" \
+  --ext-str server_fqdn="${CI_SERVER_FQDN}" \
+  --ext-str server_ssh_host="${CI_SERVER_SHELL_SSH_HOST}" \
+  /opt/commodore-compile.jsonnet \
+  > generated-commodore-compile.yml

gitlab/bin/step-yamllint

@@ -0,0 +1,9 @@
+#!/bin/sh
+set -e -u -x
+
+if [ "$(find . -name '.yamllint' -or -name '.yamllint.y*ml' | wc -l)" -ge "1" ]; then
+  # yamllint will autoload this file
+  exec yamllint .
+else
+  exec yamllint -d "$YAMLLINT_DEFAULTS" .
+fi