From b36159322efb7ba3a2e9cb06538671e24f72102d Mon Sep 17 00:00:00 2001 From: Ilya Matiach Date: Fri, 12 Jan 2024 16:03:49 -0500 Subject: [PATCH] update the pypi release script to use github action with auth token and (#593) update github action versions --- .github/workflows/CI-python.yml | 12 +- .github/workflows/Code-Scan.yml | 2 +- .github/workflows/python-linting.yml | 2 +- .../workflows/release-interpret-community.yml | 100 +++++++++ devops/PyPI-Release.yml | 200 ------------------ 5 files changed, 108 insertions(+), 208 deletions(-) create mode 100644 .github/workflows/release-interpret-community.yml delete mode 100644 devops/PyPI-Release.yml diff --git a/.github/workflows/CI-python.yml b/.github/workflows/CI-python.yml index 19e09dfe..d7dddce0 100644 --- a/.github/workflows/CI-python.yml +++ b/.github/workflows/CI-python.yml @@ -19,8 +19,8 @@ jobs: runs-on: ${{ matrix.operatingSystem }} steps: - - uses: actions/checkout@v3 - - uses: conda-incubator/setup-miniconda@v2 + - uses: actions/checkout@v4 + - uses: conda-incubator/setup-miniconda@v3 with: auto-update-conda: true python-version: ${{ matrix.pythonVersion }} @@ -67,13 +67,13 @@ jobs: shell: bash -l {0} run: | pytest ./tests -m "not notebooks" -s -v --cov=${{ matrix.packageDirectory }} --cov-report=xml --cov-report=html - - name: Upload code coverage results - uses: actions/upload-artifact@v3 + + - if: ${{ matrix.pythonVersion == '3.8' }} + name: Upload code coverage results + uses: actions/upload-artifact@v4 with: name: ${{ matrix.packageDirectory }}-code-coverage-results path: htmlcov - # Use always() to always run this step to publish test results when there are test failures - if: ${{ always() }} - if: ${{ matrix.pythonVersion == '3.8' }} name: Upload to codecov diff --git a/.github/workflows/Code-Scan.yml b/.github/workflows/Code-Scan.yml index 8ccfddb5..ecb3d2c2 100644 --- a/.github/workflows/Code-Scan.yml +++ b/.github/workflows/Code-Scan.yml @@ -27,7 +27,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/python-linting.yml b/.github/workflows/python-linting.yml index 94247cc5..90f9e5f0 100644 --- a/.github/workflows/python-linting.yml +++ b/.github/workflows/python-linting.yml @@ -16,7 +16,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python 3.11 uses: actions/setup-python@v5 with: diff --git a/.github/workflows/release-interpret-community.yml b/.github/workflows/release-interpret-community.yml new file mode 100644 index 00000000..ac867285 --- /dev/null +++ b/.github/workflows/release-interpret-community.yml @@ -0,0 +1,100 @@ +name: Release interpret-community to PyPI + +# trigger manually only ("collaborator" or more permissions required) +on: + workflow_dispatch: + inputs: + releaseType: + description: "Test or Prod PyPI?" + required: true + default: "Test" + +jobs: + release-build: + runs-on: ubuntu-latest + + steps: + - name: fail if Test nor Prod + if: ${{ ! (github.event.inputs.releaseType == 'Test' || github.event.inputs.releaseType == 'Prod') }} + run: | + echo "Only Test or Prod can be used." + exit 1 + + - uses: actions/checkout@v4 + + - uses: conda-incubator/setup-miniconda@v3 + with: + auto-update-conda: true + python-version: 3.8 + + - name: Install pytorch on non-MacOS + shell: bash -l {0} + run: | + conda install --yes --quiet pytorch torchvision cpuonly -c pytorch + + - name: update and upgrade pip, setuptools, wheel, and twine + shell: bash -l {0} + run: | + python -m pip install --upgrade pip + pip install --upgrade setuptools wheel twine + + - name: Install dev dependencies + shell: bash -l {0} + run: | + pip install -r requirements-dev.txt + + - name: Install visualization dependencies + shell: bash -l {0} + # install scikit-learn to workaround raiwidgets dependency + run: | + pip install raiwidgets + pip install -r requirements-vis.txt + pip install --upgrade scikit-learn + pip install --upgrade "shap<=0.44.0" + + - name: Install test dependencies + shell: bash -l {0} + run: | + pip install -r requirements-test.txt + + - name: pip freeze + shell: bash -l {0} + run: pip freeze + + - name: build wheel for interpret-community + shell: bash -l {0} + run: python setup.py sdist bdist_wheel + working-directory: python + + # run tests before publishing to PyPI + - name: install interpret-community wheel locally + shell: bash -l {0} + run: find ./dist/ -name '*.whl' -exec pip install {} \; + working-directory: python + + - name: run interpret-community tests + shell: bash -l {0} + run: pytest ./tests -m "not notebooks" -s -v + + - name: Upload a interpret-community build result + uses: actions/upload-artifact@v4 + with: + name: interpret_community + path: python/dist/ + + # publish to PyPI + - name: Publish interpret-community package to Test PyPI + if: ${{ github.event.inputs.releaseType == 'Test' }} + uses: pypa/gh-action-pypi-publish@release/v1 + with: + user: __token__ + password: ${{ secrets.TEST_PYPI_API_TOKEN_INTERPRET_COMMUNITY }} + repository_url: https://test.pypi.org/legacy/ + packages_dir: python/dist/ + - name: Publish interpret-community package to PyPI + if: ${{ github.event.inputs.releaseType == 'Prod' }} + uses: pypa/gh-action-pypi-publish@release/v1 + with: + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN_INTERPRET_COMMUNITY }} + packages_dir: python/dist/ \ No newline at end of file diff --git a/devops/PyPI-Release.yml b/devops/PyPI-Release.yml deleted file mode 100644 index dceadda0..00000000 --- a/devops/PyPI-Release.yml +++ /dev/null @@ -1,200 +0,0 @@ -# Simplified PyPI release pipeline - -# At queue time, the user selects a Test or Production deployment. The following stages -# then run: -# - Predeployment validation (run a set of tests against the repository) -# - Creates a wheel and stores in Pipeline Artifact -# - Download wheel file from Artifact, pip install, and run tests -# - Upload the wheel to PyPI (Test or Production as specified at queue time) -# - Install from PyPI and run tests - -parameters: -- name: releaseType - displayName: Release Type - type: string - default: Test - values: - - Test - - Production - -variables: - poolImage: "ubuntu-latest" - poolPythonVersion: 3.7 - packageArtifactName: Wheels - versionArtifactName: Version - versionFileName: versionInfo.txt - -trigger: none # No CI build - -pr: none # Not for pull requests - -# ================================================================================================== - -stages: -- stage: PredeploymentValidation - displayName: Predeployment Validation - pool: - vmImage: $(poolImage) - - jobs: - - template: templates/all-tests-job-template.yml - parameters: - platforms: { Linux: ubuntu-latest, MacOS: macos-latest, Windows: windows-latest } - pyVersions: [3.7, 3.8] - installationType: PipLocal - envArtifactStem: PredeployFreeze - envFileStem: redeploy-requirements - -# ================================================================================================== - -- stage: CreateWheel - displayName: Create Wheel Artifact - dependsOn: PredeploymentValidation - pool: - vmImage: $(poolImage) - - variables: - wheelEnvName: WheelEnvironment - - jobs: - - job: CreateWheel - displayName: Build and publish wheel - pool: - vmImage: $(poolImage) - - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python $(poolPythonVersion)' - inputs: - versionSpec: $(poolPythonVersion) - addToPath: true - - - template: templates/create-env-step-template.yml - parameters: - pythonVersion: $(poolPythonVersion) - envInfoArtifact: CreateWheelFreeze - envInfoFileBase: createwheel-freeze - condaEnv: $(wheelEnvName) - - - bash: | - source activate $(wheelEnvName) - pip install --upgrade wheel - displayName: 'Install wheel' - - - bash: | - source activate $(wheelEnvName) - python ./tools/build_wheels.py --version-filename $(versionFilename) - displayName: 'Build wheels' - - - task: PublishPipelineArtifact@1 - displayName: "Publish wheels" - inputs: - path: $(System.DefaultWorkingDirectory)/python/dist - artifact: $(packageArtifactName) - - - task: PublishPipelineArtifact@1 - displayName: "Publish version information file" - inputs: - path: '$(System.DefaultWorkingDirectory)/$(versionFilename)' - artifact: $(versionArtifactName) - -# ================================================================================================== - -- stage: TestWheel - displayName: Test Wheel from Artifact - dependsOn: CreateWheel - pool: - vmImage: $(poolImage) - - jobs: - - template: templates/all-tests-job-template.yml - parameters: - platforms: { Linux: ubuntu-latest, MacOS: macos-latest, Windows: windows-latest } - pyVersions: [3.7, 3.8] - installationType: 'WheelArtifact' - envArtifactStem: TestWheelFreeze - envFileStem: requirements-wheel-test - wheelArtifactName: $(packageArtifactName) - -# ================================================================================================== - -- stage: UploadWheel - displayName: Upload Wheel to PyPI (${{parameters.releaseType}}) - dependsOn: TestWheel - pool: - vmImage: $(poolImage) - - variables: - ${{ if eq(parameters.releaseType, 'Test')}}: - twineConnection: PyPI-Test - twineEndpoint: PyPITest - ${{ if eq(parameters.releaseType, 'Production')}}: - twineConnection: PyPI-Prod - twineEndpoint: PyPIProd - - jobs: - - deployment: 'PyPI_${{parameters.releaseType}}_Upload' - displayName: PyPI ${{parameters.releaseType}} Upload - ${{ if eq(parameters.releaseType, 'Test')}}: - environment: 'PyPI-Test Deployment' - ${{ if eq(parameters.releaseType, 'Production')}}: - environment: 'PyPI Deployment' - pool: - vmImage: $(poolImage) - - strategy: - runOnce: - deploy: - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python $(poolPythonVersion)' - inputs: - versionSpec: $(poolPythonVersion) - addToPath: true - - - script: pip install twine - displayName: 'Install twine' - - - task: TwineAuthenticate@0 - inputs: - externalFeeds: ${{variables.twineConnection}} - - - script: 'twine upload --verbose -r $(twineEndpoint) --config-file $(PYPIRC_PATH) $(Pipeline.Workspace)/$(packageArtifactName)/*' - displayName: Upload to ${{parameters.releaseType}} PyPI - - # TODO: Add GitHub Release task, so links in PyPI ReadMe will work without manual intervention (Prod only) - - - job: PyPI_Pause - pool: server - dependsOn: 'PyPI_${{parameters.releaseType}}_Upload' - displayName: PyPI Pause - - steps: - - task: Delay@1 - displayName: "Pause to allow PyPI updates to complete" - inputs: - delayForMinutes: "5" - -# # ================================================================================================== - -- stage: TestFromPyPI - displayName: Test package from ${{parameters.releaseType}} PyPI - dependsOn: UploadWheel - pool: - vmImage: $(poolImage) - - variables: - envInfoArtifact: TestPyPIFreeze - envInfoFileBase: requirements-pypi-test - - jobs: - - template: templates/all-tests-job-template.yml - parameters: - platforms: { Linux: ubuntu-latest, MacOS: macos-latest, Windows: windows-latest } - pyVersions: [3.7, 3.8] - envArtifactStem: TestPyPIFreeze - envFileStem: requirements-pypi-test - installationType: 'PyPI' - targetType: ${{parameters.releaseType}} - versionArtifactName: $(versionArtifactName) - versionArtifactFile: $(versionFileName) \ No newline at end of file