diff --git a/.ci/ansible/start_container.yaml b/.ci/ansible/start_container.yaml index bb8abbf..688fa34 100644 --- a/.ci/ansible/start_container.yaml +++ b/.ci/ansible/start_container.yaml @@ -58,7 +58,7 @@ follow_redirects: none register: result until: result.status == 200 - retries: 6 + retries: 20 delay: 5 rescue: - name: "Output pulp container log" diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index 7222362..8b49505 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -9,52 +9,26 @@ set -euv -mkdir .ci/ansible/vars || true -echo "---" > .ci/ansible/vars/main.yaml +# make sure this script runs at the repo root +cd "$(dirname "$(realpath -e "$0")")"/../../.. +TAG="${TAG:-latest}" + +mkdir -p .ci/ansible/vars cd .ci/ansible/ -TAG=ci_build - -if [ -n "${GITHUB_REF##*/}" ]; then - # Install the plugin only and use published PyPI packages for the rest - # Quoting ${TAG} ensures Ansible casts the tag as a string. - cat >> vars/main.yaml << VARSYAML -image: - name: pulp - tag: "${TAG}" -plugins: - - name: pulpcore - source: pulpcore - - name: pulp_file - source: "pulp_file" -services: - - name: pulp - image: "pulp:${TAG}" - volumes: - - ./settings:/etc/pulp -VARSYAML -else - cat >> vars/main.yaml << VARSYAML -image: - name: pulp - tag: "${TAG}" -plugins: - - name: pulp_file - source: "pulp_file" - - name: pulpcore - source: ./pulpcore +cat >> vars/main.yaml << VARSYAML +--- services: - name: pulp - image: "pulp:${TAG}" + image: "ghcr.io/pulp/pulp:${TAG}" volumes: - ./settings:/etc/pulp VARSYAML -fi cat >> vars/main.yaml << VARSYAML pulp_settings: {"allowed_content_checksums": ["sha1", "sha224", "sha256", "sha384", "sha512"], "allowed_export_paths": ["/tmp"], "allowed_import_paths": ["/tmp"]} VARSYAML -ansible-playbook build_container.yaml +# ansible-playbook build_container.yaml ansible-playbook start_container.yaml diff --git a/.github/workflows/scripts/script.sh b/.github/workflows/scripts/script.sh index 08b08cb..52eef66 100755 --- a/.github/workflows/scripts/script.sh +++ b/.github/workflows/scripts/script.sh @@ -10,39 +10,35 @@ set -mveuo pipefail +# make sure this script runs at the repo root +cd "$(dirname "$(realpath -e "$0")")"/../../.. + # Needed for both starting the service and building the docs. # Gets set in .github/settings.yml, but doesn't seem to inherited by # this script. -export DJANGO_SETTINGS_MODULE=pulpcore.app.settings -export PULP_SETTINGS=$GITHUB_WORKSPACE/.ci/ansible/settings/settings.py - +export DJANGO_SETTINGS_MODULE="pulpcore.app.settings" +export PULP_SETTINGS="$PWD/.ci/ansible/settings/settings.py" export PULP_URL="http://pulp" -cd $GITHUB_WORKSPACE - ./generate.sh pulpcore python pip install ./pulpcore-client ./generate.sh pulp_file python pip install ./pulp_file-client -python $GITHUB_WORKSPACE/.github/workflows/scripts/test_bindings.py -cd ../pulp-openapi-generator +python .github/workflows/scripts/test_bindings.py rm -rf ./pulpcore-client +./generate.sh pulpcore ruby +pushd pulpcore-client + gem build pulpcore_client + gem install --both ./pulpcore_client-*.gem +popd -./generate.sh pulpcore ruby 0 -cd pulpcore-client -gem build pulpcore_client -gem install --both ./pulpcore_client-0.gem -cd .. rm -rf ./pulp_file-client - -./generate.sh pulp_file ruby 0 - -cd pulp_file-client +./generate.sh pulp_file ruby +pushd pulp_file-client gem build pulp_file_client -gem install --both ./pulp_file_client-0.gem -cd .. -ruby $GITHUB_WORKSPACE/.github/workflows/scripts/test_bindings.rb +gem install --both ./pulp_file_client-*.gem +popd -exit +ruby .github/workflows/scripts/test_bindings.rb diff --git a/.gitignore b/.gitignore index 88fbedb..5be0c53 100644 --- a/.gitignore +++ b/.gitignore @@ -65,3 +65,6 @@ target/ # written in generate.sh .openapi-generator-ignore + +# generated client packages +/*-client/ diff --git a/gen-client.sh b/gen-client.sh new file mode 100755 index 0000000..e05be46 --- /dev/null +++ b/gen-client.sh @@ -0,0 +1,120 @@ +#!/bin/bash + +set -eu + +if [ $# -eq 0 ] +then + echo "No arguments provided" + echo "Usage: $0 [ []]" + exit 1 +fi + +API_SPEC="$1" +COMPONENT="$2" +LANGUAGE="${3:-python}" +PACKAGE="${4:-pulp_${COMPONENT//-/_}}" + +DOMAIN_ENABLED="$(jq -r '.info."x-pulp-domain-enabled" // false' < "${API_SPEC}")" +VERSION="$(jq -r --arg component "$COMPONENT" '.info."x-pulp-app-versions"[$component]' < "${API_SPEC}")" + +echo "Version: ${VERSION}" +[ "${VERSION}" != "none" ] + +OPENAPI_PYTHON_IMAGE="${OPENAPI_PYTHON_IMAGE:-docker.io/openapitools/openapi-generator-cli:v4.3.1}" +OPENAPI_RUBY_IMAGE="${OPENAPI_RUBY_IMAGE:-docker.io/openapitools/openapi-generator-cli:v4.3.1}" +OPENAPI_TYPESCRIPT_IMAGE="${OPENAPI_TYPESCRIPT_IMAGE:-docker.io/openapitools/openapi-generator-cli:v5.2.1}" + +if command -v podman > /dev/null +then + CONTAINER_EXEC=podman + if [[ -n "${PULP_MCS_LABEL:-}" ]] + then + USER_COMMAND=("--userns=keep-id" "--security-opt" "label=level:$PULP_MCS_LABEL") + else + USER_COMMAND=("--userns=keep-id") + fi + ULIMIT_COMMAND=() +else + CONTAINER_EXEC=docker + if [[ -n "${PULP_MCS_LABEL:-}" ]] + then + USER_COMMAND=("-u" "$(id -u)" "--security-opt" "label=level:$PULP_MCS_LABEL") + else + USER_COMMAND=("-u" "$(id -u)") + fi + ULIMIT_COMMAND=("--ulimit" "nofile=122880:122880") +fi + +# Mount volumes from parent container with `--volumes-from` option if `PARENT_CONTAINER_ID` is set. +if [ -z "${PARENT_CONTAINER_ID:-}" ] +then + if command -v getenforce > /dev/null && [ "$(getenforce)" == "Enforcing" ] + then + VOLUME_NAME="/local:Z" + else + VOLUME_NAME="/local" + fi + VOLUME_OPTION=("--volume" "${PWD}:${VOLUME_NAME}") + VOLUME_DIR="/local" +else + VOLUME_OPTION=("--volumes-from" "${PARENT_CONTAINER_ID}:rw") + VOLUME_DIR="${PWD}" +fi + +if [ "$LANGUAGE" = "python" ] +then + $CONTAINER_EXEC run \ + "${ULIMIT_COMMAND[@]}" \ + "${USER_COMMAND[@]}" \ + --rm \ + "${VOLUME_OPTION[@]}" \ + "$OPENAPI_PYTHON_IMAGE" generate \ + -i "${VOLUME_DIR}/${API_SPEC}" \ + -g python \ + -o "${VOLUME_DIR}/${PACKAGE}-client" \ + "--additional-properties=packageName=pulpcore.client.${PACKAGE},projectName=${PACKAGE}-client,packageVersion=${VERSION},domainEnabled=${DOMAIN_ENABLED}" \ + -t "${VOLUME_DIR}/templates/python" \ + --skip-validate-spec \ + --strict-spec=false + cp python/__init__.py "${PACKAGE}-client/pulpcore/" + cp python/__init__.py "${PACKAGE}-client/pulpcore/client/" +fi + +if [ "$LANGUAGE" = "ruby" ] +then + # https://github.com/OpenAPITools/openapi-generator/wiki/FAQ#how-to-skip-certain-files-during-code-generation + mkdir -p "${PACKAGE}-client" + echo git_push.sh > "${PACKAGE}-client/.openapi-generator-ignore" + + python3 remove-cookie-auth.py + $CONTAINER_EXEC run \ + "${ULIMIT_COMMAND[@]}" \ + "${USER_COMMAND[@]}" \ + --rm \ + "${VOLUME_OPTION[@]}" \ + "$OPENAPI_RUBY_IMAGE" generate \ + -i "${VOLUME_DIR}/${API_SPEC}" \ + -g ruby \ + -o "${VOLUME_DIR}/${PACKAGE}-client" \ + "--additional-properties=gemName=${PACKAGE}_client,gemLicense="GPLv2+",gemVersion=${VERSION},gemHomepage=https://github.com/pulp/${PACKAGE}" \ + --library=faraday \ + -t "${VOLUME_DIR}/templates/ruby" \ + --skip-validate-spec \ + --strict-spec=false +fi + +if [ "$LANGUAGE" = "typescript" ] +then + $CONTAINER_EXEC run \ + "${ULIMIT_COMMAND[@]}" \ + "${USER_COMMAND[@]}" \ + --rm \ + "${VOLUME_OPTION[@]}" \ + "$OPENAPI_TYPESCRIPT_IMAGE" generate \ + -i "${VOLUME_DIR}/${API_SPEC}" \ + -g typescript-axios \ + -o "${VOLUME_DIR}/${PACKAGE}-client" \ + -t "${VOLUME_DIR}/templates/typescript-axios" \ + --skip-validate-spec \ + --strict-spec=false +fi diff --git a/generate.sh b/generate.sh index 67d30d2..fab7741 100755 --- a/generate.sh +++ b/generate.sh @@ -1,156 +1,48 @@ -#!/bin/bash -e +#!/bin/bash -if [ $# -eq 0 ]; then - echo "No arguments provided" - exit 1 -fi +set -eu -if command -v podman > /dev/null +if [ $# -eq 0 ] then - container_exec=podman - ULIMIT_COMMAND= - if [[ -n $PULP_MCS_LABEL ]] - then - USER_COMMAND="--userns=keep-id --security-opt label=level:$PULP_MCS_LABEL" - else - USER_COMMAND="--userns=keep-id" - fi -else - container_exec=docker - if [[ -n $PULP_MCS_LABEL ]] - then - USER_COMMAND="-u $(id -u) --security-opt label=level:$PULP_MCS_LABEL" - else - USER_COMMAND="-u $(id -u)" - fi - ULIMIT_COMMAND="--ulimit nofile=122880:122880" + echo "No arguments provided" + exit 1 fi -if command -v getenforce > /dev/null +# match the component name by removing the "pulp/pulp_" prefix +if [ "$1" = "pulpcore" ] then - if [ "$(getenforce)" == "Enforcing" ] - then - volume_name="/local:Z" - else - volume_name="/local" - fi + COMPONENT="core" else - volume_name="/local" + COMPONENT=${1#"pulp_"} fi # Skip downloading the api.json if `USE_LOCAL_API_JSON` is set. -if [[ -z $USE_LOCAL_API_JSON ]] +if [[ -z "${USE_LOCAL_API_JSON:-}" ]] then PULP_URL="${PULP_URL:-http://localhost:24817}" - PULP_API_ROOT="${PULP_API_ROOT:-/pulp/}" - PULP_URL="${PULP_URL}${PULP_API_ROOT}api/v3/" # Download the schema - retry_count=0 - until curl --fail-with-body -k -o api.json "${PULP_URL}docs/api.json?bindings&plugin=$1" + RETRY_COUNT=0 + until curl --fail-with-body -k -o api.json "${PULP_URL}docs/api.json?bindings&component=${COMPONENT}" do - if [ $retry_count -eq 10 ] + if [ $RETRY_COUNT -eq 10 ] then break fi sleep 2 - ((retry_count++)) + ((RETRY_COUNT++)) done # Get the version of the pulpcore or plugin as reported by status API fi -DOMAIN_ENABLED=$(jq -r '.info | ."x-pulp-domain-enabled" // false' < api.json) -export DOMAIN_ENABLED - -if [ $# -gt 2 ]; -then - export VERSION=$3 -else - # match the component name by removing the "pulp/pulp_" prefix - if [ $1 = 'pulpcore' ] - then - COMPONENT_NAME="core" - else - COMPONENT_NAME=${1#"pulp_"} - fi - - curl --fail-with-body -k -o status.json ${PULP_URL}status/ - VERSION=$(jq --arg plugin $COMPONENT_NAME -r '.versions[] | select(.component == $plugin) | .version' < status.json) - export VERSION - rm status.json -fi - -# Mount volumes from parent container with `--volumes-from` option if the -# `PARENT_CONTAINER_ID` is set. -if [ -z $PARENT_CONTAINER_ID ] -then - VOLUME_OPTION="--volume ${PWD}:${volume_name}" - VOLUME_DIR="/local" -else - VOLUME_OPTION="--volumes-from ${PARENT_CONTAINER_ID}:rw" - VOLUME_DIR="${PWD}" -fi - -OPENAPI_PYTHON_IMAGE="${OPENAPI_PYTHON_IMAGE:-docker.io/openapitools/openapi-generator-cli:v4.3.1}" -OPENAPI_RUBY_IMAGE="${OPENAPI_RUBY_IMAGE:-docker.io/openapitools/openapi-generator-cli:v4.3.1}" -OPENAPI_TYPESCRIPT_IMAGE="${OPENAPI_TYPESCRIPT_IMAGE:-docker.io/openapitools/openapi-generator-cli:v5.2.1}" - echo ::group::BINDINGS -if [ $2 = 'python' ] -then - $container_exec run \ - $ULIMIT_COMMAND \ - $USER_COMMAND \ - --rm \ - ${VOLUME_OPTION} \ - $OPENAPI_PYTHON_IMAGE generate \ - -i ${VOLUME_DIR}/api.json \ - -g python \ - -o ${VOLUME_DIR}/$1-client \ - --additional-properties=packageName=pulpcore.client.$1,projectName=$1-client,packageVersion=${VERSION},domainEnabled=${DOMAIN_ENABLED} \ - -t ${VOLUME_DIR}/templates/python \ - --skip-validate-spec \ - --strict-spec=false - cp python/__init__.py $1-client/pulpcore/ - cp python/__init__.py $1-client/pulpcore/client -fi -if [ $2 = 'ruby' ] -then - # https://github.com/OpenAPITools/openapi-generator/wiki/FAQ#how-to-skip-certain-files-during-code-generation - mkdir -p $1-client - echo git_push.sh > $1-client/.openapi-generator-ignore - python3 remove-cookie-auth.py - $container_exec run \ - $ULIMIT_COMMAND \ - $USER_COMMAND \ - --rm ${VOLUME_OPTION} \ - $OPENAPI_RUBY_IMAGE generate \ - -i ${VOLUME_DIR}/api.json \ - -g ruby \ - -o ${VOLUME_DIR}/$1-client \ - --additional-properties=gemName=$1_client,gemLicense="GPLv2+",gemVersion=${VERSION},gemHomepage=https://github.com/pulp/$1 \ - --library=faraday \ - -t ${VOLUME_DIR}/templates/ruby \ - --skip-validate-spec \ - --strict-spec=false -fi -if [ $2 = 'typescript' ] -then - $container_exec run \ - $ULIMIT_COMMAND \ - $USER_COMMAND \ - --rm ${VOLUME_OPTION} \ - $OPENAPI_TYPESCRIPT_IMAGE generate \ - -i ${VOLUME_DIR}/api.json \ - -g typescript-axios \ - -o ${VOLUME_DIR}/$1-client \ - -t ${VOLUME_DIR}/templates/typescript-axios \ - --skip-validate-spec \ - --strict-spec=false -fi +./gen-client.sh api.json "${COMPONENT}" "${2:-python}" "${1}" echo ::endgroup:: -rm api.json +if [[ -z "${USE_LOCAL_API_JSON:-}" ]] +then + rm api.json +fi