feat: add new DuckDB backend for reading ndjson directly #384
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| pull_request: | |
| paths-ignore: | |
| - 'docs/**' | |
| push: | |
| branches: | |
| - main | |
| paths-ignore: | |
| - 'docs/**' | |
| # The goal here is to cancel older workflows when a PR is updated (because it's pointless work) | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }} | |
| cancel-in-progress: true | |
| jobs: | |
| unittest: | |
| name: unit tests | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: 3.9 | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install ".[test]" | |
| - name: Test with pytest | |
| run: | | |
| python -m pytest | |
| lint: | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Install linters | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install ".[dev]" | |
| - name: Run sqlfluff on jinja templates | |
| run: | | |
| sqlfluff lint | |
| - name: Run black | |
| if: success() || failure() # still run black if above checks fails | |
| run: | | |
| black --check --verbose . | |
| regression: | |
| runs-on: ubuntu-22.04 | |
| permissions: | |
| id-token: write # This is required for requesting the JWT | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install . | |
| # See https://github.com/aws-actions/configure-aws-credentials for configuring | |
| # the aws side of this - the below action is just a light wrapper | |
| - name: Configure AWS Credentials | |
| uses: mcblair/[email protected] | |
| with: | |
| role-arn: arn:aws:iam::${{secrets.AWS_ACCOUNT}}:role/cumulus-library-ci | |
| region: us-east-1 | |
| profile-name: cumulus-library-ci | |
| - name: Rebuild tables | |
| env: | |
| PROFILE: cumulus-library-ci | |
| WG: cumulus | |
| DB: cumulus_library_regression_db | |
| run: | | |
| cumulus-library build -t core --profile $PROFILE --workgroup $WG --database $DB | |
| cumulus-library export -t core ./tests/regression/data_export/ --profile $PROFILE --workgroup $WG --database $DB | |
| - name: Compare vs known data | |
| run: python ./tests/regression/run_regression.py | |