From 23624c2eabf6a58ce814f92aa2bb6e1784ad8f7d Mon Sep 17 00:00:00 2001 From: Quentin Bourgerie Date: Wed, 27 Nov 2024 11:37:40 +0100 Subject: [PATCH] test docs --- .../workflows/concrete_python_release_cpu.yml | 616 +++++++++--------- .github/workflows/scripts/make_apidocs.sh | 70 ++ 2 files changed, 389 insertions(+), 297 deletions(-) create mode 100755 .github/workflows/scripts/make_apidocs.sh diff --git a/.github/workflows/concrete_python_release_cpu.yml b/.github/workflows/concrete_python_release_cpu.yml index 3f775b0faa..c05252fcfc 100644 --- a/.github/workflows/concrete_python_release_cpu.yml +++ b/.github/workflows/concrete_python_release_cpu.yml @@ -2,6 +2,7 @@ name: concrete-python release-cpu on: workflow_dispatch: + pull_request: push: tags: - 'v[0-9]+.[0-9]+.[0-9]+*' @@ -38,29 +39,10 @@ jobs: backend: aws profile: release - release-checks: - needs: setup-instance - runs-on: ${{ needs.setup-instance.outputs.runner-name }} - steps: - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - submodules: recursive - fetch-depth: 0 - - name: Check python api doc is up to date - run: ci/scripts/make_apidocs.sh - - name: Slack Notification - if: ${{ failure() }} - continue-on-error: true - uses: rtCamp/action-slack-notify@c33737706dea87cd7784c687dadc9adf1be59990 - env: - SLACK_COLOR: ${{ job.status }} - SLACK_MESSAGE: "release-checks finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" - build-linux-x86: strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.8"] needs: setup-instance runs-on: ${{ needs.setup-instance.outputs.runner-name }} steps: @@ -138,174 +120,18 @@ jobs: SLACK_COLOR: ${{ job.status }} SLACK_MESSAGE: "build-linux-x86 finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" - build-macos: + release-checks: strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] - runs-on: ["aws-mac1-metal", "aws-mac2-metal"] - runs-on: ${{ matrix.runs-on }} + python-version: ["3.8"] + needs: [ build-linux-x86 ] + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: submodules: recursive fetch-depth: 0 - - name: Install OS Dependencies - run: | - brew install ninja ccache - - name: Setup rust toolchain for concrete-cpu - uses: ./.github/actions/setup_rust_toolchain_for_concrete_cpu - - name: Set release version (nightly) - if: ${{ env.RELEASE_TYPE == 'nightly' }} - run: | - NIGHTLY_VERSION=$(date +"%Y%m%d") - LATEST_RELEASE_VERSION=$(git tag -l |grep "v.*" |sort |tail -n 1 | grep -e '[0-9].*' -o) - echo "__version__ = \"${LATEST_RELEASE_VERSION}-dev${NIGHTLY_VERSION}\"" >| frontends/concrete-python/version.txt - - name: Set release version (public) - if: ${{ env.RELEASE_TYPE == 'public' }} - run: echo "__version__ = \"$(git describe --tags --abbrev=0 | grep -e '[0-9].*' -o)\"" >| frontends/concrete-python/version.txt - - name: Expose release version from Python - run: cp frontends/concrete-python/version.txt frontends/concrete-python/concrete/fhe/version.py - - name: Build wheel - run: | - CONCRETE_PYTHON=$(pwd)/frontends/concrete-python - CONCRETE_COMPILER=$(pwd)/compilers/concrete-compiler/compiler - export COMPILER_BUILD_DIRECTORY=$CONCRETE_COMPILER/build - export PYTHON=${{ format('python{0}', matrix.python-version) }} - echo "Using $PYTHON" - - # Setup pkg-config to find OpenBLAS (scipy need it) - export PKG_CONFIG_PATH="/opt/homebrew/opt/openblas/lib/pkgconfig" - - # Setup vitual environment - rm -rf .venv - $PYTHON -m venv .venv && . .venv/bin/activate - - # Install requirements - pip install -r "${CONCRETE_PYTHON}"/requirements.txt - pip install -r "${CONCRETE_PYTHON}"/requirements.dev.txt - - # Build python bindings of concrete compiler - cd "${CONCRETE_COMPILER}" || exit - echo "Debug: ccache statistics (prior to the build):" && ccache -s - make Python3_EXECUTABLE="$(which python)" python-bindings - echo "Debug: ccache statistics (after the build):" && ccache -s - - # Build wheel - cd "${CONCRETE_PYTHON}" || exit - rm -rf dist - mkdir -p dist - pip wheel -v --no-deps -w dist . - delocate-wheel -v dist/*macos*.whl - - deactivate - - name: Upload wheel - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 - with: - name: ${{ format('wheel-{0}-{1}', matrix.python-version, matrix.runs-on) }} - path: frontends/concrete-python/dist/*macos*.whl - retention-days: 3 - - name: Slack Notification - if: ${{ failure() }} - continue-on-error: true - uses: rtCamp/action-slack-notify@c33737706dea87cd7784c687dadc9adf1be59990 - env: - SLACK_COLOR: ${{ job.status }} - SLACK_MESSAGE: "build-macos finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" - - hash: - # Generate hashes for the wheels, used later for provenance. - needs: [build-linux-x86, build-macos] - runs-on: ubuntu-latest - outputs: - hash: ${{ steps.hash.outputs.hash }} - steps: - - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 - with: - path: frontends/concrete-python/dist - pattern: wheel-* - merge-multiple: true - - name: generate hash - id: hash - run: cd frontends/concrete-python/dist && echo "hash=$(sha256sum ./*.whl | base64 -w0)" >> "${GITHUB_OUTPUT}" - - provenance: - needs: [hash] - permissions: - actions: read - id-token: write - contents: write - # Can't pin with hash due to how this workflow works. - uses: slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@v2.0.0 - with: - base64-subjects: ${{ needs.hash.outputs.hash }} - - push: - needs: [build-linux-x86, build-macos, provenance] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 0 - - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 - with: - path: wheels - pattern: 'wheel-*' - merge-multiple: true - - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 - with: - pattern: '*.intoto.jsonl' - # When building a new public tag, create a new draft release. - - name: create draft release - if: ${{ env.RELEASE_TYPE == 'public'}} - run: | - TAG=$(git describe --tags --abbrev=0) - export TAG - echo "${TAG}" - gh release create --draft --repo ${{ github.repository }} \ - --verify-tag "${TAG}" \ - --title "${TAG}" \ - wheels/* ./*.intoto.jsonl/* - env: - GH_TOKEN: ${{ github.token }} - - name: Upload wheels to S3 - if: ${{ env.RELEASE_TYPE == 'public' || env.RELEASE_TYPE == 'nightly' }} - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_IAM_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_IAM_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }} - S3_BUCKET_NAME: ${{ secrets.AWS_S3_PYPI_BUCKET_NAME }} - CLOUDFRONT_DISTRIBUTION_ID: ${{ secrets.AWS_CLOUDFRONT_PYPI_DISTRIBUTION_ID }} - run: | - pip install boto3 bigtree - # upload wheels - aws s3 sync ./wheels/ "s3://${S3_BUCKET_NAME}/cpu/concrete-python" - # update indexes and invalidate cloudfront cache - python .github/workflows/scripts/s3_update_html_indexes.py - - name: Slack Notification - if: ${{ failure() }} - continue-on-error: true - uses: rtCamp/action-slack-notify@c33737706dea87cd7784c687dadc9adf1be59990 - env: - SLACK_COLOR: ${{ job.status }} - SLACK_MESSAGE: "push finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" - - test-linux-x86: - needs: [setup-instance, build-linux-x86] - continue-on-error: true - strategy: - matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] - runs-on: ${{ needs.setup-instance.outputs.runner-name }} - steps: - # HOME is needed by actions-rs/toolchain - - run: | - echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}" - - name: Install rust - uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # v1.0.7 - with: - toolchain: nightly - default: true - name: Setup Python uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 with: @@ -315,50 +141,246 @@ jobs: with: name: ${{ format('wheel-{0}-linux-x86', matrix.python-version) }} path: ${{ format('wheel-{0}-linux-x86', matrix.python-version) }} - - name: Checkout the repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - submodules: recursive - path: repo - - name: Test wheel + - name: Check documentation run: | WHEEL_DIR=$(pwd)/${{ format('wheel-{0}-linux-x86', matrix.python-version) }} - CONCRETE_PYTHON=$(pwd)/repo/frontends/concrete-python - - # Initialize an empty test environment - cd "$(mktemp -d)" - python -m venv .testenv && source .testenv/bin/activate - - # Install the concrete-python wheel - pip install "${WHEEL_DIR}"/*.whl - - # Install extra requirements for tests - sudo apt update -y - sudo apt install -y graphviz libgraphviz-dev - pip install -r "${CONCRETE_PYTHON}"/requirements.extra-full.txt - pip install -r "${CONCRETE_PYTHON}"/requirements.dev.txt - - # TODO - check for version - - # Copy test files - cp -R "${CONCRETE_PYTHON}"/tests . - cp -R "${CONCRETE_PYTHON}"/examples . - cp -R "${CONCRETE_PYTHON}"/pytest.ini . - cp "${CONCRETE_PYTHON}"/Makefile . - - # Running tests - make tfhers-utils - pytest tests -svv -n auto - - name: Slack Notification + WHEEL_CONCRETE="${WHEEL_DIR}"/*.whl .github/workflows/scripts/make_apidocs.sh + - name: Upload docs.patch if failure if: ${{ failure() }} - continue-on-error: true - uses: rtCamp/action-slack-notify@c33737706dea87cd7784c687dadc9adf1be59990 - env: - SLACK_COLOR: ${{ job.status }} - SLACK_MESSAGE: "test-linux-x86 (${{ matrix.python-version }}) finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" + uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 + with: + name: docs.patch + path: docs.patch + retention-days: 3 + # - name: Slack Notification + # if: ${{ failure() }} + # continue-on-error: true + # uses: rtCamp/action-slack-notify@c33737706dea87cd7784c687dadc9adf1be59990 + # env: + # SLACK_COLOR: ${{ job.status }} + # SLACK_MESSAGE: "release-checks finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" + + # build-macos: + # strategy: + # matrix: + # python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + # runs-on: ["aws-mac1-metal", "aws-mac2-metal"] + # runs-on: ${{ matrix.runs-on }} + # steps: + # - name: Checkout + # uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + # with: + # submodules: recursive + # fetch-depth: 0 + # - name: Install OS Dependencies + # run: | + # brew install ninja ccache + # - name: Setup rust toolchain for concrete-cpu + # uses: ./.github/actions/setup_rust_toolchain_for_concrete_cpu + # - name: Set release version (nightly) + # if: ${{ env.RELEASE_TYPE == 'nightly' }} + # run: | + # NIGHTLY_VERSION=$(date +"%Y%m%d") + # LATEST_RELEASE_VERSION=$(git tag -l |grep "v.*" |sort |tail -n 1 | grep -e '[0-9].*' -o) + # echo "__version__ = \"${LATEST_RELEASE_VERSION}-dev${NIGHTLY_VERSION}\"" >| frontends/concrete-python/version.txt + # - name: Set release version (public) + # if: ${{ env.RELEASE_TYPE == 'public' }} + # run: echo "__version__ = \"$(git describe --tags --abbrev=0 | grep -e '[0-9].*' -o)\"" >| frontends/concrete-python/version.txt + # - name: Expose release version from Python + # run: cp frontends/concrete-python/version.txt frontends/concrete-python/concrete/fhe/version.py + # - name: Build wheel + # run: | + # CONCRETE_PYTHON=$(pwd)/frontends/concrete-python + # CONCRETE_COMPILER=$(pwd)/compilers/concrete-compiler/compiler + # export COMPILER_BUILD_DIRECTORY=$CONCRETE_COMPILER/build + # export PYTHON=${{ format('python{0}', matrix.python-version) }} + # echo "Using $PYTHON" + + # # Setup pkg-config to find OpenBLAS (scipy need it) + # export PKG_CONFIG_PATH="/opt/homebrew/opt/openblas/lib/pkgconfig" + + # # Setup vitual environment + # rm -rf .venv + # $PYTHON -m venv .venv && . .venv/bin/activate + + # # Install requirements + # pip install -r "${CONCRETE_PYTHON}"/requirements.txt + # pip install -r "${CONCRETE_PYTHON}"/requirements.dev.txt + + # # Build python bindings of concrete compiler + # cd "${CONCRETE_COMPILER}" || exit + # echo "Debug: ccache statistics (prior to the build):" && ccache -s + # make Python3_EXECUTABLE="$(which python)" python-bindings + # echo "Debug: ccache statistics (after the build):" && ccache -s + + # # Build wheel + # cd "${CONCRETE_PYTHON}" || exit + # rm -rf dist + # mkdir -p dist + # pip wheel -v --no-deps -w dist . + # delocate-wheel -v dist/*macos*.whl + + # deactivate + # - name: Upload wheel + # uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 + # with: + # name: ${{ format('wheel-{0}-{1}', matrix.python-version, matrix.runs-on) }} + # path: frontends/concrete-python/dist/*macos*.whl + # retention-days: 3 + # - name: Slack Notification + # if: ${{ failure() }} + # continue-on-error: true + # uses: rtCamp/action-slack-notify@c33737706dea87cd7784c687dadc9adf1be59990 + # env: + # SLACK_COLOR: ${{ job.status }} + # SLACK_MESSAGE: "build-macos finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" + + # hash: + # # Generate hashes for the wheels, used later for provenance. + # needs: [build-linux-x86, build-macos] + # runs-on: ubuntu-latest + # outputs: + # hash: ${{ steps.hash.outputs.hash }} + # steps: + # - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + # with: + # path: frontends/concrete-python/dist + # pattern: wheel-* + # merge-multiple: true + # - name: generate hash + # id: hash + # run: cd frontends/concrete-python/dist && echo "hash=$(sha256sum ./*.whl | base64 -w0)" >> "${GITHUB_OUTPUT}" + + # provenance: + # needs: [hash] + # permissions: + # actions: read + # id-token: write + # contents: write + # # Can't pin with hash due to how this workflow works. + # uses: slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@v2.0.0 + # with: + # base64-subjects: ${{ needs.hash.outputs.hash }} + + # push: + # needs: [build-linux-x86, build-macos, provenance] + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + # with: + # fetch-depth: 0 + # - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + # with: + # path: wheels + # pattern: 'wheel-*' + # merge-multiple: true + # - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + # with: + # pattern: '*.intoto.jsonl' + # # When building a new public tag, create a new draft release. + # - name: create draft release + # if: ${{ env.RELEASE_TYPE == 'public'}} + # run: | + # TAG=$(git describe --tags --abbrev=0) + # export TAG + # echo "${TAG}" + # gh release create --draft --repo ${{ github.repository }} \ + # --verify-tag "${TAG}" \ + # --title "${TAG}" \ + # wheels/* ./*.intoto.jsonl/* + # env: + # GH_TOKEN: ${{ github.token }} + # - name: Upload wheels to S3 + # if: ${{ env.RELEASE_TYPE == 'public' || env.RELEASE_TYPE == 'nightly' }} + # env: + # AWS_ACCESS_KEY_ID: ${{ secrets.AWS_IAM_ID }} + # AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_IAM_KEY }} + # AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }} + # S3_BUCKET_NAME: ${{ secrets.AWS_S3_PYPI_BUCKET_NAME }} + # CLOUDFRONT_DISTRIBUTION_ID: ${{ secrets.AWS_CLOUDFRONT_PYPI_DISTRIBUTION_ID }} + # run: | + # pip install boto3 bigtree + # # upload wheels + # aws s3 sync ./wheels/ "s3://${S3_BUCKET_NAME}/cpu/concrete-python" + # # update indexes and invalidate cloudfront cache + # python .github/workflows/scripts/s3_update_html_indexes.py + # - name: Slack Notification + # if: ${{ failure() }} + # continue-on-error: true + # uses: rtCamp/action-slack-notify@c33737706dea87cd7784c687dadc9adf1be59990 + # env: + # SLACK_COLOR: ${{ job.status }} + # SLACK_MESSAGE: "push finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" + + # test-linux-x86: + # needs: [setup-instance, build-linux-x86] + # continue-on-error: true + # strategy: + # matrix: + # python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + # runs-on: ${{ needs.setup-instance.outputs.runner-name }} + # steps: + # # HOME is needed by actions-rs/toolchain + # - run: | + # echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}" + # - name: Install rust + # uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # v1.0.7 + # with: + # toolchain: nightly + # default: true + # - name: Setup Python + # uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 + # with: + # python-version: ${{ matrix.python-version }} + # - name: Download wheels + # uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + # with: + # name: ${{ format('wheel-{0}-linux-x86', matrix.python-version) }} + # path: ${{ format('wheel-{0}-linux-x86', matrix.python-version) }} + # - name: Checkout the repository + # uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + # with: + # submodules: recursive + # path: repo + # - name: Test wheel + # run: | + # WHEEL_DIR=$(pwd)/${{ format('wheel-{0}-linux-x86', matrix.python-version) }} + # CONCRETE_PYTHON=$(pwd)/repo/frontends/concrete-python + + # # Initialize an empty test environment + # cd "$(mktemp -d)" + # python -m venv .testenv && source .testenv/bin/activate + + # # Install the concrete-python wheel + # pip install "${WHEEL_DIR}"/*.whl + + # # Install extra requirements for tests + # sudo apt update -y + # sudo apt install -y graphviz libgraphviz-dev + # pip install -r "${CONCRETE_PYTHON}"/requirements.extra-full.txt + # pip install -r "${CONCRETE_PYTHON}"/requirements.dev.txt + + # # TODO - check for version + + # # Copy test files + # cp -R "${CONCRETE_PYTHON}"/tests . + # cp -R "${CONCRETE_PYTHON}"/examples . + # cp -R "${CONCRETE_PYTHON}"/pytest.ini . + # cp "${CONCRETE_PYTHON}"/Makefile . + + # # Running tests + # make tfhers-utils + # pytest tests -svv -n auto + # - name: Slack Notification + # if: ${{ failure() }} + # continue-on-error: true + # uses: rtCamp/action-slack-notify@c33737706dea87cd7784c687dadc9adf1be59990 + # env: + # SLACK_COLOR: ${{ job.status }} + # SLACK_MESSAGE: "test-linux-x86 (${{ matrix.python-version }}) finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" teardown-instance: - needs: [ setup-instance, test-linux-x86 ] + needs: [ setup-instance, build-linux-x86 ] if: ${{ always() && needs.setup-instance.result != 'skipped' }} runs-on: ubuntu-latest steps: @@ -380,81 +402,81 @@ jobs: SLACK_COLOR: ${{ job.status }} SLACK_MESSAGE: "Instance teardown finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" - test-macos: - needs: [build-macos] - continue-on-error: true - strategy: - matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] - runs-on: ["aws-mac1-metal", "aws-mac2-metal"] - runs-on: ${{ matrix.runs-on }} - outputs: - slack_message: ${{ steps.prepare_slack_notif.outputs.slack_message }} - slack_color: ${{ steps.prepare_slack_notif.outputs.slack_color }} - steps: - - name: Download wheels - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 - with: - name: ${{ format('wheel-{0}-{1}', matrix.python-version, matrix.runs-on) }} - path: ${{ format('wheel-{0}-{1}', matrix.python-version, matrix.runs-on) }} - - name: Checkout the repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - submodules: recursive - path: repo - - name: Test wheel - run: | - WHEEL_DIR=$(pwd)/${{ format('wheel-{0}-{1}', matrix.python-version, matrix.runs-on) }} - CONCRETE_PYTHON=$(pwd)/repo/frontends/concrete-python - PYTHON=${{ format('python{0}', matrix.python-version) }} - - # Initialize an empty test environment - TEST_TMP_DIR=$(mktemp -d) - echo "TEST_TMP_DIR=${TEST_TMP_DIR}" >> "${GITHUB_ENV}" - cd "${TEST_TMP_DIR}" || exit - - # Activate virtual environment - $PYTHON -m venv .testenv && source .testenv/bin/activate - - # Install extra requirements for tests - pip install "${WHEEL_DIR}"/*macos*.whl - pip install -r "${CONCRETE_PYTHON}"/requirements.dev.txt - - "${CONCRETE_PYTHON}"/../../.github/workflows/scripts/fix_multi_omp_bug_macos.sh - - # Copy test files - cp -R "${CONCRETE_PYTHON}"/tests . - cp -R "${CONCRETE_PYTHON}"/examples . - cp -R "${CONCRETE_PYTHON}"/pytest.ini . - cp "${CONCRETE_PYTHON}"/Makefile . - - # Fix: this is because cargo cannot be found - . "$HOME/.cargo/env" - - # Running tests - make tfhers-utils - mkdir ./KeySetCache - pytest tests -svv -n auto --key-cache "./KeySetCache" -m "not dataflow and not graphviz" - - name: Cleanup host - if: success() || failure() - run: | - rm -rf "${TEST_TMP_DIR}" - - name: Prepare Slack Notification - id: prepare_slack_notif - if: ${{ failure() }} - continue-on-error: true - run: | - echo "slack_message=test-macos (${{matrix.runs-on}}/${{ matrix.python-version }}) finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" >> "$GITHUB_OUTPUT" - echo "slack_color=${{ job.status }}" >> "$GITHUB_OUTPUT" - - slack-notif-macos: - needs: ["test-macos"] - runs-on: "ubuntu-latest" - steps: - - name: Slack Notification - # we want to check that prepare_slack_notif was run - if: ${{ needs.test-macos.outputs.slack_color != '' }} - uses: rtCamp/action-slack-notify@c33737706dea87cd7784c687dadc9adf1be59990 - env: - SLACK_COLOR: ${{ needs.test-macos.outputs.slack_color }} - SLACK_MESSAGE: ${{ needs.test-macos.outputs.slack_message }} + # test-macos: + # needs: [build-macos] + # continue-on-error: true + # strategy: + # matrix: + # python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + # runs-on: ["aws-mac1-metal", "aws-mac2-metal"] + # runs-on: ${{ matrix.runs-on }} + # outputs: + # slack_message: ${{ steps.prepare_slack_notif.outputs.slack_message }} + # slack_color: ${{ steps.prepare_slack_notif.outputs.slack_color }} + # steps: + # - name: Download wheels + # uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + # with: + # name: ${{ format('wheel-{0}-{1}', matrix.python-version, matrix.runs-on) }} + # path: ${{ format('wheel-{0}-{1}', matrix.python-version, matrix.runs-on) }} + # - name: Checkout the repository + # uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + # with: + # submodules: recursive + # path: repo + # - name: Test wheel + # run: | + # WHEEL_DIR=$(pwd)/${{ format('wheel-{0}-{1}', matrix.python-version, matrix.runs-on) }} + # CONCRETE_PYTHON=$(pwd)/repo/frontends/concrete-python + # PYTHON=${{ format('python{0}', matrix.python-version) }} + + # # Initialize an empty test environment + # TEST_TMP_DIR=$(mktemp -d) + # echo "TEST_TMP_DIR=${TEST_TMP_DIR}" >> "${GITHUB_ENV}" + # cd "${TEST_TMP_DIR}" || exit + + # # Activate virtual environment + # $PYTHON -m venv .testenv && source .testenv/bin/activate + + # # Install extra requirements for tests + # pip install "${WHEEL_DIR}"/*macos*.whl + # pip install -r "${CONCRETE_PYTHON}"/requirements.dev.txt + + # "${CONCRETE_PYTHON}"/../../.github/workflows/scripts/fix_multi_omp_bug_macos.sh + + # # Copy test files + # cp -R "${CONCRETE_PYTHON}"/tests . + # cp -R "${CONCRETE_PYTHON}"/examples . + # cp -R "${CONCRETE_PYTHON}"/pytest.ini . + # cp "${CONCRETE_PYTHON}"/Makefile . + + # # Fix: this is because cargo cannot be found + # . "$HOME/.cargo/env" + + # # Running tests + # make tfhers-utils + # mkdir ./KeySetCache + # pytest tests -svv -n auto --key-cache "./KeySetCache" -m "not dataflow and not graphviz" + # - name: Cleanup host + # if: success() || failure() + # run: | + # rm -rf "${TEST_TMP_DIR}" + # - name: Prepare Slack Notification + # id: prepare_slack_notif + # if: ${{ failure() }} + # continue-on-error: true + # run: | + # echo "slack_message=test-macos (${{matrix.runs-on}}/${{ matrix.python-version }}) finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" >> "$GITHUB_OUTPUT" + # echo "slack_color=${{ job.status }}" >> "$GITHUB_OUTPUT" + + # slack-notif-macos: + # needs: ["test-macos"] + # runs-on: "ubuntu-latest" + # steps: + # - name: Slack Notification + # # we want to check that prepare_slack_notif was run + # if: ${{ needs.test-macos.outputs.slack_color != '' }} + # uses: rtCamp/action-slack-notify@c33737706dea87cd7784c687dadc9adf1be59990 + # env: + # SLACK_COLOR: ${{ needs.test-macos.outputs.slack_color }} + # SLACK_MESSAGE: ${{ needs.test-macos.outputs.slack_message }} diff --git a/.github/workflows/scripts/make_apidocs.sh b/.github/workflows/scripts/make_apidocs.sh new file mode 100755 index 0000000000..1fa7a87ce9 --- /dev/null +++ b/.github/workflows/scripts/make_apidocs.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash + +set -e + +PYTHON=${PYTHON:-python} +PIP=${PIP:-${PYTHON} -m pip} +VENV_DIR=${PWD}/.venv-docs + +# Remove old documentation +rm -rf docs/dev/api/* + +# Create virtual env and install concrete and docs tools +${PYTHON} -m venv "${VENV_DIR}" +source "${VENV_DIR}"/bin/activate +if [ -z "${CONCRETE_WHEEL}" ]; then + echo "You must specify the CONCRETE_WHEEL environment variable" + exit 1 +fi +${PIP} install "${CONCRETE_WHEEL}" +${PIP} install lazydocs + +# Generate the API doc +lazydocs --output-path="./docs/dev/api" --overview-file="README.md" --src-base-url="../../" --no-watermark concrete + + +# Update documentation paths +SED_OPT='-i' +if [ $(uname) == "Darwin" ]; then + SED_OPT='-i ""' +fi + +# Fixing the path issues, to point on files in GitHub +WHICH_PYTHON_VERSION=$(${PYTHON} --version | cut -f 2 -d " " | cut -f 1-2 -d ".") +sed $SED_OPT -e "s@../../.venvtrash/lib.*/python$WHICH_PYTHON_VERSION/site-packages/@../../../compilers/concrete-compiler/compiler/lib/Bindings/Python/@g" docs/dev/api/concrete.compiler*.md docs/dev/api/concrete.lang*.md +sed $SED_OPT -e "s@../../.venvtrash/lib.*/python$WHICH_PYTHON_VERSION/site-packages/@../../frontends/concrete-python/@g" docs/dev/api/concrete.fhe*.md + +# Fixing absolute path in doc +sed $SED_OPT -e "s@$PWD/@./@g" docs/dev/api/concrete.compiler.library_support.md + +# Fixing the links in README.md, which fails (missing .'s for some reason): remove the #headers +sed $SED_OPT -e "s@.md#module-.*)@.md)@g" docs/dev/api/README.md +sed $SED_OPT -e "s@.md#function-.*)@.md)@g" docs/dev/api/README.md +sed $SED_OPT -e "s@.md#class-.*)@.md)@g" docs/dev/api/README.md + +# Removed the "object addresses" and "function addresses", since they are not constant +sed $SED_OPT -e "s@object at 0x[a-zA-Z0-9]*@object at ADDRESS@g" docs/*.md +sed $SED_OPT -e "s@object at 0x[a-zA-Z0-9]*@object at ADDRESS@g" docs/*/*.md +sed $SED_OPT -e "s@object at 0x[a-zA-Z0-9]*@object at ADDRESS@g" docs/*/*/*.md + +sed $SED_OPT -e "s@function Int at 0x[a-zA-Z0-9]*@function Int at ADDRESS@g" docs/*.md +sed $SED_OPT -e "s@function Int at 0x[a-zA-Z0-9]*@function Int at ADDRESS@g" docs/*/*.md +sed $SED_OPT -e "s@function Int at 0x[a-zA-Z0-9]*@function Int at ADDRESS@g" docs/*/*/*.md + +# FIXME: remove this once the PR has been merged once +sed $SED_OPT -e "s@https://github.com/zama-ai/concrete-compiler-internal/blob/main/LICENSE.txt@https://github.com/zama-ai/concrete/blob/main/LICENSE.txt@g" ./docs/dev/api/concrete.lang.dialects.md ./docs/dev/api/concrete.compiler.md ./docs/dev/api/concrete.lang.md + +# Create the patch file +git add -N docs/dev/api/* +git diff docs &> docs.patch + +# Was there changes? +if [ -s docs.patch ] then + echo "The documentation us up to date, congrats" + exit 0 +else + echo "There is a difference in the docs, please commit the changes, here the change:" + cat docs.patch + exit -1 +fi +