dispatch job #1
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Dispatch Job | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| repo: | |
| description: 'The https github url for the recipe feedstock' | |
| required: true | |
| ref: | |
| description: 'The tag or branch to target in your recipe repo' | |
| required: true | |
| default: 'main' | |
| bucket: | |
| description: 'This job runner leverages s3fs.S3FileSystem for your recipe cache and output. Choices currently are: "default"' | |
| required: true | |
| default: 'default' | |
| prune: | |
| description: 'Only run the first two time steps' | |
| required: true | |
| default: false | |
| parallelism: | |
| description: 'Number of task managers to spin up' | |
| required: true | |
| default: '1' | |
| jobs: | |
| build: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: checkout repository | |
| uses: actions/checkout@v2 | |
| - name: set up python 3.10 | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: '3.10' | |
| - name: convert parallelism input to integer | |
| id: cast_step | |
| run: | | |
| INPUT_NUMBER=${{ github.event.inputs.parallelism }} | |
| PARALLELISM_NUMBER=$(awk 'BEGIN{printf "%d\n", '"$INPUT_NUMBER"'}') | |
| echo "::set-output name=parallelism::$PARALLELISM_NUMBER" | |
| - name: echo server | |
| run: | | |
| echo "Manually triggered workflow: \ | |
| ${{ github.event.inputs.repo }} \ | |
| ${{ github.event.inputs.ref }} \ | |
| ${{ github.event.inputs.bucket }} \ | |
| ${{ github.event.inputs.parallelism }} \ | |
| ${{ github.event.inputs.prune }}" | |
| - name: install deps | |
| run: | | |
| # TODO: move to requirements file | |
| python -m pip install --upgrade pip | |
| pip install s3fs apache-beam==2.52.0 pangeo-forge-recipes>=0.9.1 | |
| - name: resolve bucket choices | |
| run: | | |
| # TODO: finish logic | |
| echo '' | |
| # - name: deploy recipes | |
| # uses: pangeo-forge/deploy-recipe-action@storage-cls-args | |
| # with: | |
| # select_recipe_by_label: "false" | |
| # pangeo_forge_runner_config: > | |
| # { | |
| # "Bake": { | |
| # "prune": ${{ github.event.inputs.prune }}", | |
| # "bakery_class": "pangeo_forge_runner.bakery.flink.FlinkOperatorBakery", | |
| # "container_image": "apache/beam_python3.10_sdk:2.52.0" | |
| # }, | |
| # "FlinkOperatorBakery": { | |
| # "parallelism": ${{ steps.cast_step.outputs.integer }}, | |
| # "enable_job_archiving": true, | |
| # "job_manager_resources": "{\"memory\": \"1536m\", \"cpu\": 0.5}", | |
| # "task_manager_resources": "{\"memory\": \"1536m\", \"cpu\": 0.5}", | |
| # "flink_configuration": "{\"taskmanager.numberOfTaskSlots\": \"1\", \"taskmanager.memory.flink.size\": \"1280m\"}" | |
| # }, | |
| # "TargetStorage": { | |
| # "fsspec_class": "s3fs.S3FileSystem", | |
| # "fsspec_args": { | |
| # "key": "${{ secrets.S3_DEFAULT_AWS_ACCESS_KEY_ID }}", | |
| # "secret": "${{ secrets.S3_DEFAULT_AWS_SECRET_ACCESS_KEY }}", | |
| # "client_kwargs": { | |
| # "region_name": "us-west-2" | |
| # }, | |
| # "default_cache_type": "none", | |
| # "default_fill_cache": false, | |
| # "use_listings_cache": false | |
| # }, | |
| # "root_path": "${{ secrets.S3_DEFAULT_BUCKET }}/{job_name}" | |
| # }, | |
| # "InputCacheStorage": { | |
| # "fsspec_class": "s3fs.S3FileSystem", | |
| # "root_path": "${{ secrets.S3_DEFAULT_BUCKET }}" | |
| # } | |
| # } |