New gen-client.sh for offline generation

.ci/ansible/start_container.yaml

@@ -58,7 +58,7 @@
             follow_redirects: none
           register: result
           until: result.status == 200
-          retries: 6
+          retries: 20
           delay: 5
       rescue:
         - name: "Output pulp container log"

.github/workflows/scripts/install.sh

@@ -9,52 +9,26 @@
 
 set -euv
 
-mkdir .ci/ansible/vars || true
-echo "---" > .ci/ansible/vars/main.yaml
+# make sure this script runs at the repo root
+cd "$(dirname "$(realpath -e "$0")")"/../../..
 
+TAG="${TAG:-latest}"
+
+mkdir -p .ci/ansible/vars
 cd .ci/ansible/
 
-TAG=ci_build
-
-if [ -n "${GITHUB_REF##*/}" ]; then
-  # Install the plugin only and use published PyPI packages for the rest
-  # Quoting ${TAG} ensures Ansible casts the tag as a string.
-  cat >> vars/main.yaml << VARSYAML
-image:
-  name: pulp
-  tag: "${TAG}"
-plugins:
-  - name: pulpcore
-    source: pulpcore
-  - name: pulp_file
-    source:  "pulp_file"
-services:
-  - name: pulp
-    image: "pulp:${TAG}"
-    volumes:
-      - ./settings:/etc/pulp
-VARSYAML
-else
-  cat >> vars/main.yaml << VARSYAML
-image:
-  name: pulp
-  tag: "${TAG}"
-plugins:
-  - name: pulp_file
-    source: "pulp_file"
-  - name: pulpcore
-    source: ./pulpcore
+cat >> vars/main.yaml << VARSYAML
+---
 services:
   - name: pulp
-    image: "pulp:${TAG}"
+    image: "ghcr.io/pulp/pulp:${TAG}"
     volumes:
       - ./settings:/etc/pulp
 VARSYAML
-fi
 
 cat >> vars/main.yaml << VARSYAML
 pulp_settings: {"allowed_content_checksums": ["sha1", "sha224", "sha256", "sha384", "sha512"], "allowed_export_paths": ["/tmp"], "allowed_import_paths": ["/tmp"]}
 VARSYAML
 
-ansible-playbook build_container.yaml
+# ansible-playbook build_container.yaml
 ansible-playbook start_container.yaml

.github/workflows/scripts/script.sh

@@ -10,39 +10,35 @@
 
 set -mveuo pipefail
 
+# make sure this script runs at the repo root
+cd "$(dirname "$(realpath -e "$0")")"/../../..
+
 # Needed for both starting the service and building the docs.
 # Gets set in .github/settings.yml, but doesn't seem to inherited by
 # this script.
-export DJANGO_SETTINGS_MODULE=pulpcore.app.settings
-export PULP_SETTINGS=$GITHUB_WORKSPACE/.ci/ansible/settings/settings.py
-
+export DJANGO_SETTINGS_MODULE="pulpcore.app.settings"
+export PULP_SETTINGS="$PWD/.ci/ansible/settings/settings.py"
 export PULP_URL="http://pulp"
 
-cd $GITHUB_WORKSPACE
-
 ./generate.sh pulpcore python
 pip install ./pulpcore-client
 ./generate.sh pulp_file python
 pip install ./pulp_file-client
 
-python $GITHUB_WORKSPACE/.github/workflows/scripts/test_bindings.py
-cd ../pulp-openapi-generator
+python .github/workflows/scripts/test_bindings.py
 
 rm -rf ./pulpcore-client
+./generate.sh pulpcore ruby
+pushd pulpcore-client
+  gem build pulpcore_client
+  gem install --both ./pulpcore_client-*.gem
+popd
 
-./generate.sh pulpcore ruby 0
-cd pulpcore-client
-gem build pulpcore_client
-gem install --both ./pulpcore_client-0.gem
-cd ..
 rm -rf ./pulp_file-client
-
-./generate.sh pulp_file ruby 0
-
-cd pulp_file-client
+./generate.sh pulp_file ruby
+pushd pulp_file-client
 gem build pulp_file_client
-gem install --both ./pulp_file_client-0.gem
-cd ..
-ruby $GITHUB_WORKSPACE/.github/workflows/scripts/test_bindings.rb
+gem install --both ./pulp_file_client-*.gem
+popd
 
-exit
+ruby .github/workflows/scripts/test_bindings.rb

.gitignore

@@ -65,3 +65,6 @@ target/
 
 # written in generate.sh
 .openapi-generator-ignore
+
+# generated client packages
+/*-client/

gen-client.sh

@@ -0,0 +1,120 @@
+#!/bin/bash
+
+set -eu
+
+if [ $# -eq 0 ]
+then
+  echo "No arguments provided"
+  echo "Usage: $0 <api_spec> <component> [<language> [<package>]]"
+  exit 1
+fi
+
+API_SPEC="$1"
+COMPONENT="$2"
+LANGUAGE="${3:-python}"
+PACKAGE="${4:-pulp_${COMPONENT//-/_}}"
+
+DOMAIN_ENABLED="$(jq -r '.info."x-pulp-domain-enabled" // false' < "${API_SPEC}")"
+VERSION="$(jq -r --arg component "$COMPONENT" '.info."x-pulp-app-versions"[$component]' < "${API_SPEC}")"
+
+echo "Version: ${VERSION}"
+[ "${VERSION}" != "none" ]
+
+OPENAPI_PYTHON_IMAGE="${OPENAPI_PYTHON_IMAGE:-docker.io/openapitools/openapi-generator-cli:v4.3.1}"
+OPENAPI_RUBY_IMAGE="${OPENAPI_RUBY_IMAGE:-docker.io/openapitools/openapi-generator-cli:v4.3.1}"
+OPENAPI_TYPESCRIPT_IMAGE="${OPENAPI_TYPESCRIPT_IMAGE:-docker.io/openapitools/openapi-generator-cli:v5.2.1}"
+
+if command -v podman > /dev/null
+then
+  CONTAINER_EXEC=podman
+  if [[ -n "${PULP_MCS_LABEL:-}" ]]
+  then
+    USER_COMMAND=("--userns=keep-id" "--security-opt" "label=level:$PULP_MCS_LABEL")
+  else
+    USER_COMMAND=("--userns=keep-id")
+  fi
+  ULIMIT_COMMAND=()
+else
+  CONTAINER_EXEC=docker
+  if [[ -n "${PULP_MCS_LABEL:-}" ]]
+  then
+    USER_COMMAND=("-u" "$(id -u)" "--security-opt" "label=level:$PULP_MCS_LABEL")
+  else
+    USER_COMMAND=("-u" "$(id -u)")
+  fi
+  ULIMIT_COMMAND=("--ulimit" "nofile=122880:122880")
+fi
+
+# Mount volumes from parent container with `--volumes-from` option if `PARENT_CONTAINER_ID` is set.
+if [ -z "${PARENT_CONTAINER_ID:-}" ]
+then
+  if command -v getenforce > /dev/null && [ "$(getenforce)" == "Enforcing" ]
+  then
+    VOLUME_NAME="/local:Z"
+  else
+    VOLUME_NAME="/local"
+  fi
+  VOLUME_OPTION=("--volume" "${PWD}:${VOLUME_NAME}")
+  VOLUME_DIR="/local"
+else
+  VOLUME_OPTION=("--volumes-from" "${PARENT_CONTAINER_ID}:rw")
+  VOLUME_DIR="${PWD}"
+fi
+
+if [ "$LANGUAGE" = "python" ]
+then
+  $CONTAINER_EXEC run \
+    "${ULIMIT_COMMAND[@]}" \
+    "${USER_COMMAND[@]}" \
+    --rm \
+    "${VOLUME_OPTION[@]}" \
+    "$OPENAPI_PYTHON_IMAGE" generate \
+    -i "${VOLUME_DIR}/${API_SPEC}" \
+    -g python \
+    -o "${VOLUME_DIR}/${PACKAGE}-client" \
+    "--additional-properties=packageName=pulpcore.client.${PACKAGE},projectName=${PACKAGE}-client,packageVersion=${VERSION},domainEnabled=${DOMAIN_ENABLED}" \
+    -t "${VOLUME_DIR}/templates/python" \
+    --skip-validate-spec \
+    --strict-spec=false
+  cp python/__init__.py "${PACKAGE}-client/pulpcore/"
+  cp python/__init__.py "${PACKAGE}-client/pulpcore/client/"
+fi
+
+if [ "$LANGUAGE" = "ruby" ]
+then
+  # https://github.com/OpenAPITools/openapi-generator/wiki/FAQ#how-to-skip-certain-files-during-code-generation
+  mkdir -p "${PACKAGE}-client"
+  echo git_push.sh > "${PACKAGE}-client/.openapi-generator-ignore"
+
+  python3 remove-cookie-auth.py
+  $CONTAINER_EXEC run \
+    "${ULIMIT_COMMAND[@]}" \
+    "${USER_COMMAND[@]}" \
+    --rm \
+    "${VOLUME_OPTION[@]}" \
+    "$OPENAPI_RUBY_IMAGE" generate \
+    -i "${VOLUME_DIR}/${API_SPEC}" \
+    -g ruby \
+    -o "${VOLUME_DIR}/${PACKAGE}-client" \
+    "--additional-properties=gemName=${PACKAGE}_client,gemLicense="GPLv2+",gemVersion=${VERSION},gemHomepage=https://github.com/pulp/${PACKAGE}" \
+    --library=faraday \
+    -t "${VOLUME_DIR}/templates/ruby" \
+    --skip-validate-spec \
+    --strict-spec=false
+fi
+
+if [ "$LANGUAGE" = "typescript" ]
+then
+  $CONTAINER_EXEC run \
+    "${ULIMIT_COMMAND[@]}" \
+    "${USER_COMMAND[@]}" \
+    --rm \
+    "${VOLUME_OPTION[@]}" \
+    "$OPENAPI_TYPESCRIPT_IMAGE" generate \
+    -i "${VOLUME_DIR}/${API_SPEC}" \
+    -g typescript-axios \
+    -o "${VOLUME_DIR}/${PACKAGE}-client" \
+    -t "${VOLUME_DIR}/templates/typescript-axios" \
+    --skip-validate-spec \
+    --strict-spec=false
+fi