Skip to content

Commit

Permalink
Update CI files
Browse files Browse the repository at this point in the history
[noissue]
  • Loading branch information
pulpbot committed Jun 30, 2024
1 parent eed48d5 commit 2cff19b
Show file tree
Hide file tree
Showing 9 changed files with 252 additions and 101 deletions.
1 change: 1 addition & 0 deletions .ci/ansible/Containerfile.j2
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ RUN pip3 install
{{ " " }}-r ./{{ item.name }}/ci_requirements.txt
{%- endif -%}
{%- endfor %}
{{ " " }}-c ./{{ plugins[0].name }}/.ci/assets/ci_constraints.txt

{% if pulp_env is defined and pulp_env %}
{% for key, value in pulp_env.items() %}
Expand Down
7 changes: 7 additions & 0 deletions .ci/assets/ci_constraints.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
# Pulpcore versions without the openapi command do no longer work in the CI
pulpcore>=3.21.30,!=3.23.*,!=3.24.*,!=3.25.*,!=3.26.*,!=3.27.*,!=3.29.*,!=3.30.*,!=3.31.*,!=3.32.*,!=3.33.*,!=3.34.*,!=3.35.*,!=3.36.*,!=3.37.*,!=3.38.*,!=3.40.*,!=3.41.*,!=3.42.*,!=3.43.*,!=3.44.*,!=3.45.*,!=3.46.*,!=3.47.*,!=3.48.*,!=3.50.*,!=3.51.*,!=3.52.*,!=3.53.*,!=3.54.*



tablib!=3.6.0
# 3.6.0: This release introduced a regression removing the "html" optional dependency.
2 changes: 1 addition & 1 deletion .github/template_gitref
Original file line number Diff line number Diff line change
@@ -1 +1 @@
2021.08.26-341-g5cb6e90
2021.08.26-347-gc4a2504
70 changes: 69 additions & 1 deletion .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,18 +23,30 @@ jobs:
with:
fetch-depth: 1
path: "pulp_container"
- uses: "actions/checkout@v4"
with:
fetch-depth: 1
repository: "pulp/pulp-openapi-generator"
path: "pulp-openapi-generator"
- uses: "actions/setup-python@v5"
with:
python-version: "3.11"
- name: "Install python dependencies"
run: |
echo ::group::PYDEPS
pip install packaging twine wheel
pip install packaging twine wheel mkdocs jq
echo ::endgroup::
- name: "Build package"
run: |
python3 setup.py sdist bdist_wheel --python-tag py3
twine check dist/*
- name: "Install built packages"
run: |
pip install dist/pulp_container-*-py3-none-any.whl -c .ci/assets/ci_constraints.txt
- name: "Generate api specs"
run: |
pulpcore-manager openapi --file "api.json"
pulpcore-manager openapi --bindings --component "container" --file "container-api.json"
- name: "Upload Package whl"
uses: "actions/upload-artifact@v4"
with:
Expand All @@ -43,3 +55,59 @@ jobs:
if-no-files-found: "error"
retention-days: 5
overwrite: true
- name: "Upload API specs"
uses: "actions/upload-artifact@v4"
with:
name: "api_spec"
path: |
pulp_container/api.json
pulp_container/container-api.json
if-no-files-found: "error"
retention-days: 5
overwrite: true
- name: "Build Python bindings packages"
run: |
.github/workflows/scripts/build_python_client.sh
shell: "bash"
env:
PY_COLORS: "1"
ANSIBLE_FORCE_COLOR: "1"
GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}"
- name: "Upload python client packages"
uses: "actions/upload-artifact@v4"
with:
name: "python-client.tar"
path: |
pulp_container/container-python-client.tar
if-no-files-found: "error"
retention-days: 5
overwrite: true
- name: "Upload python client docs"
uses: "actions/upload-artifact@v4"
with:
name: "python-client-docs.tar"
path: |
pulp_container/container-python-client-docs.tar
if-no-files-found: "error"
retention-days: 5
overwrite: true
- name: "Build Ruby bindings packages"
run: |
.github/workflows/scripts/build_ruby_client.sh
shell: "bash"
env:
PY_COLORS: "1"
ANSIBLE_FORCE_COLOR: "1"
GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}"
- name: "Upload Ruby client"
uses: "actions/upload-artifact@v4"
with:
name: "ruby-client.tar"
path: |
pulp_container/container-ruby-client.tar
if-no-files-found: "error"
retention-days: 5
overwrite: true
...
56 changes: 56 additions & 0 deletions .github/workflows/scripts/build_python_client.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
#!/bin/bash

# This script expects all <app_label>-api.json files to exist in the plugins root directory.
# It produces a <app_label>-python-client.tar and <app_label>-python-client-docs.tar file in the plugins root directory.

# WARNING: DO NOT EDIT!
#
# This file was generated by plugin_template, and is managed by it. Please use
# './plugin-template --github pulp_container' to update this file.
#
# For more info visit https://github.com/pulp/plugin_template

set -mveuo pipefail

# make sure this script runs at the repo root
cd "$(dirname "$(realpath -e "$0")")"/../../..

pushd ../pulp-openapi-generator
rm -rf "pulp_container-client"

# We need to copy that over to be visible in the container.
cp "../pulp_container/container-api.json" .
./gen-client.sh "container-api.json" "container" python "pulp_container"

pushd pulp_container-client
python setup.py sdist bdist_wheel --python-tag py3

twine check "dist/pulp_container_client-"*"-py3-none-any.whl"
twine check "dist/pulp_container-client-"*".tar.gz"

tar cvf "../../pulp_container/container-python-client.tar" ./dist

find ./docs/* -exec sed -i 's/Back to README/Back to HOME/g' {} \;
find ./docs/* -exec sed -i 's/README//g' {} \;
cp README.md docs/index.md
sed -i 's/docs\///g' docs/index.md
find ./docs/* -exec sed -i 's/\.md//g' {} \;

cat >> mkdocs.yml << DOCSYAML
---
site_name: PulpContainer Client
site_description: Container bindings
site_author: Pulp Team
site_url: https://docs.pulpproject.org/pulp_container_client/
repo_name: pulp/pulp_container
repo_url: https://github.com/pulp/pulp_container
theme: readthedocs
DOCSYAML

# Building the bindings docs
mkdocs build

# Pack the built site.
tar cvf ../../pulp_container/container-python-client-docs.tar ./site
popd
popd
35 changes: 35 additions & 0 deletions .github/workflows/scripts/build_ruby_client.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
#!/bin/bash

# This script expects all <app_label>-api.json files to exist in the plugins root directory.
# It produces a <app_label>-ruby-client.tar file in the plugins root directory.

# WARNING: DO NOT EDIT!
#
# This file was generated by plugin_template, and is managed by it. Please use
# './plugin-template --github pulp_container' to update this file.
#
# For more info visit https://github.com/pulp/plugin_template

set -mveuo pipefail

# make sure this script runs at the repo root
cd "$(dirname "$(realpath -e "$0")")"/../../..

pushd ../pulp-openapi-generator
rm -rf "pulp_container-client"

# We need to copy that over to be visible in the container.
#cp "../pulp_container/container-api.json" .
#./gen-client.sh "container-api.json" "container" ruby "pulp_container"

# -------------
# The generator still needs to have it called api.json at this time...
cp "../pulp_container/api.json" .
./gen-client.sh "api.json" "container" ruby "pulp_container"
# -------------

pushd pulp_container-client
gem build pulp_container_client
tar cvf "../../pulp_container/container-ruby-client.tar" "./pulp_container_client-"*".gem"
popd
popd
115 changes: 57 additions & 58 deletions .github/workflows/scripts/script.sh
Original file line number Diff line number Diff line change
Expand Up @@ -52,75 +52,74 @@ password password
# Some commands like ansible-galaxy specifically require 600
cmd_prefix bash -c "chmod 600 ~pulp/.netrc"

# Generate and install binding
# Generate bindings
###################

echo "::group::Generate bindings"

touch bindings_requirements.txt
pushd ../pulp-openapi-generator
if pulp debug has-plugin --name "core" --specifier ">=3.44.0.dev"
then
# Use app_label to generate api.json and package to produce the proper package name.

if [ "$(jq -r '.domain_enabled' <<<"$REPORTED_STATUS")" = "true" ]
# Workaround: Domains are not supported by the published bindings.
# Sadly: Different pulpcore-versions aren't either...
# So we exclude the prebuilt ones only for domains disabled.
if [ "$(jq -r '.domain_enabled' <<<"${REPORTED_STATUS}")" = "true" ] || [ "$(jq -r '.online_workers[0].pulp_href|startswith("/pulp/api/v3/")' <<< "${REPORTED_STATUS}")" = "false" ]
then
# Workaround: Domains are not supported by the published bindings.
# Generate new bindings for all packages.
for item in $(jq -r '.versions[] | tojson' <<<"$REPORTED_STATUS")
do
echo $item
COMPONENT="$(jq -r '.component' <<<"$item")"
VERSION="$(jq -r '.version' <<<"$item")"
MODULE="$(jq -r '.module' <<<"$item")"
PACKAGE="${MODULE%%.*}"
curl --fail-with-body -k -o api.json "${PULP_URL}${PULP_API_ROOT}api/v3/docs/api.json?bindings&component=$COMPONENT"
USE_LOCAL_API_JSON=1 ./generate.sh "${PACKAGE}" python "${VERSION}"
cmd_prefix pip3 install "/root/pulp-openapi-generator/${PACKAGE}-client"
sudo rm -rf "./${PACKAGE}-client"
done
BUILT_CLIENTS=""
else
# Sadly: Different pulpcore-versions aren't either...
for item in $(jq -r '.versions[]| select(.component!="container")| tojson' <<<"$REPORTED_STATUS")
do
echo $item
COMPONENT="$(jq -r '.component' <<<"$item")"
VERSION="$(jq -r '.version' <<<"$item")"
MODULE="$(jq -r '.module' <<<"$item")"
PACKAGE="${MODULE%%.*}"
curl --fail-with-body -k -o api.json "${PULP_URL}${PULP_API_ROOT}api/v3/docs/api.json?bindings&component=$COMPONENT"
USE_LOCAL_API_JSON=1 ./generate.sh "${PACKAGE}" python "${VERSION}"
cmd_prefix pip3 install "/root/pulp-openapi-generator/${PACKAGE}-client"
sudo rm -rf "./${PACKAGE}-client"
done
BUILT_CLIENTS=" container "
fi
else
# Infer the client name from the package name by replacing "-" with "_".
# Use the component to infer the package name on older versions of pulpcore.

if [ "$(echo "$REPORTED_STATUS" | jq -r '.domain_enabled')" = "true" ]
then
# Workaround: Domains are not supported by the published bindings.
# Generate new bindings for all packages.
for item in $(echo "$REPORTED_STATUS" | jq -r '.versions[]|(.package // ("pulp_" + .component)|sub("pulp_core"; "pulpcore"))|sub("-"; "_")')
do
./generate.sh "${item}" python
cmd_prefix pip3 install "/root/pulp-openapi-generator/${item}-client"
sudo rm -rf "./${item}-client"
done
else
# Sadly: Different pulpcore-versions aren't either...
for item in $(echo "$REPORTED_STATUS" | jq -r '.versions[]|select(.component!="container")|(.package // ("pulp_" + .component)|sub("pulp_core"; "pulpcore"))|sub("-"; "_")')
do
./generate.sh "${item}" python
cmd_prefix pip3 install "/root/pulp-openapi-generator/${item}-client"
sudo rm -rf "./${item}-client"
done
fi
fi
for ITEM in $(jq -r '.versions[] | tojson' <<<"${REPORTED_STATUS}")
do
COMPONENT="$(jq -r '.component' <<<"${ITEM}")"
VERSION="$(jq -r '.version' <<<"${ITEM}" | python3 -c "from packaging.version import Version; print(Version(input()))")"
# On older status endpoints, the module was not provided, but the package should be accurate
# there, because we did not merge plugins into pulpcore back then.
MODULE="$(jq -r '.module // (.package|gsub("-"; "_"))' <<<"${ITEM}")"
PACKAGE="${MODULE%%.*}"
cmd_prefix pulpcore-manager openapi --bindings --component "${COMPONENT}" > "${COMPONENT}-api.json"
if [[ ! " ${BUILT_CLIENTS} " =~ "${COMPONENT}" ]]
then
rm -rf "./${PACKAGE}-client"
./gen-client.sh "${COMPONENT}-api.json" "${COMPONENT}" python "${PACKAGE}"
pushd "${PACKAGE}-client"
python setup.py sdist bdist_wheel --python-tag py3
popd
else
if [ ! -f "${PACKAGE}-client/dist/${PACKAGE}_client-${VERSION}-py3-none-any.whl" ]
then
ls -lR "${PACKAGE}-client/"
echo "Error: Client bindings for ${COMPONENT} not found."
echo "File ${PACKAGE}-client/dist/${PACKAGE}_client-${VERSION}-py3-none-any.whl missing."
exit 1
fi
fi
echo "/root/pulp-openapi-generator/${PACKAGE}-client/dist/${PACKAGE}_client-${VERSION}-py3-none-any.whl" >> "../pulp_container/bindings_requirements.txt"
done
popd

# At this point, this is a safeguard only, so let's not make too much fuzz about the old status format.
echo "$REPORTED_STATUS" | jq -r '.versions[]|select(.package)|(.package|sub("_"; "-")) + "-client==" + .version' > bindings_requirements.txt
echo "::endgroup::"

echo "::group::Debug bindings diffs"

# Bindings diff for container
jq '(.paths[][].parameters|select(.)) |= sort_by(.name)' < "container-api.json" > "build-api.json"
jq '(.paths[][].parameters|select(.)) |= sort_by(.name)' < "../pulp-openapi-generator/container-api.json" > "test-api.json"
jsondiff --indent 2 build-api.json test-api.json || true
echo "::endgroup::"

# Install test requirements
###########################

# Add a safeguard to make sure the proper versions of the clients are installed.
echo "$REPORTED_STATUS" | jq -r '.versions[]|select(.package)|(.package|sub("_"; "-")) + "-client==" + .version' > bindings_constraints.txt
cmd_stdin_prefix bash -c "cat > /tmp/unittest_requirements.txt" < unittest_requirements.txt
cmd_stdin_prefix bash -c "cat > /tmp/functest_requirements.txt" < functest_requirements.txt
cmd_stdin_prefix bash -c "cat > /tmp/bindings_requirements.txt" < bindings_requirements.txt
cmd_prefix pip3 install -r /tmp/unittest_requirements.txt -r /tmp/functest_requirements.txt -r /tmp/bindings_requirements.txt
cmd_stdin_prefix bash -c "cat > /tmp/bindings_constraints.txt" < bindings_constraints.txt
cmd_prefix pip3 install -r /tmp/unittest_requirements.txt -r /tmp/functest_requirements.txt -r /tmp/bindings_requirements.txt -c /tmp/bindings_constraints.txt

CERTIFI=$(cmd_prefix python3 -c 'import certifi; print(certifi.where())')
cmd_prefix bash -c "cat /etc/pulp/certs/pulp_webserver.crt >> '$CERTIFI'"
Expand Down Expand Up @@ -151,7 +150,7 @@ else
else
cmd_user_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_container.tests.functional -m parallel -n 8"
cmd_user_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_container.tests.functional -m 'not parallel'"
fi
fi
fi
pushd ../pulp-cli
pip install -r test_requirements.txt
Expand Down
Loading

0 comments on commit 2cff19b

Please sign in to comment.