diff --git a/.github/workflows/handle-required-checks.yml b/.github/workflows/handle-required-checks.yml deleted file mode 100644 index 40677fd4bc..0000000000 --- a/.github/workflows/handle-required-checks.yml +++ /dev/null @@ -1,22 +0,0 @@ -## "lint" and "tests" are required checks, but they run conditionally. -## This handles the cases where they don't run because no Python or JS file has been changed. -## See https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/defining-the-mergeability-of-pull-requests/troubleshooting-required-status-checks#handling-skipped-but-required-checks -name: handle-required-checks -on: - push: - paths-ignore: - - 'mathesar_ui/**' - - '**.py' - pull_request: - paths-ignore: - - 'mathesar_ui/**' - - '**.py' -jobs: - lint: - runs-on: ubuntu-latest - steps: - - run: 'echo "No lint required"' - tests: - runs-on: ubuntu-latest - steps: - - run: 'echo "No tests required"' diff --git a/.github/workflows/run-flake8.yml b/.github/workflows/run-flake8.yml deleted file mode 100644 index 98eba2706c..0000000000 --- a/.github/workflows/run-flake8.yml +++ /dev/null @@ -1,27 +0,0 @@ -name: Lint Python code - -on: - push: - paths: - - '**.py' - pull_request: - paths: - - '**.py' - -jobs: - lint: - runs-on: ubuntu-latest - # We only want to run on external PRs, since internal PRs are covered by "push" - # This prevents this from running twice on internal PRs - if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - - name: Run flake8 - uses: julianwachholz/flake8-action@main - with: - checkName: "flake8" - path: "." - plugins: flake8-no-types - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/run-lint-audit-tests-ui.yml b/.github/workflows/run-lint-audit-tests-ui.yml deleted file mode 100644 index 3e9aa41f06..0000000000 --- a/.github/workflows/run-lint-audit-tests-ui.yml +++ /dev/null @@ -1,129 +0,0 @@ -name: UI - Lint, Audit and Tests - -on: - push: - paths: - - 'mathesar_ui/**' - pull_request: - paths: - - 'mathesar_ui/**' - -jobs: - format: - runs-on: ubuntu-latest - defaults: - run: - working-directory: ./mathesar_ui - timeout-minutes: 5 - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-node@v2 - with: - node-version: 18 - - id: npm-cache-dir - run: echo "::set-output name=dir::$(npm config get cache)" - - uses: actions/cache@v2 - id: npm-cache - with: - path: ${{ steps.npm-cache-dir.outputs.dir }} - key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} - restore-keys: ${{ runner.os }}-node- - - run: npm ci --no-audit --prefer-offline - - run: npm run check-format - - lint: - runs-on: ubuntu-latest - defaults: - run: - working-directory: ./mathesar_ui - timeout-minutes: 5 - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-node@v2 - with: - node-version: 18 - - id: npm-cache-dir - run: echo "::set-output name=dir::$(npm config get cache)" - - uses: actions/cache@v2 - id: npm-cache - with: - path: ${{ steps.npm-cache-dir.outputs.dir }} - key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} - restore-keys: ${{ runner.os }}-node- - - run: npm ci --no-audit --prefer-offline - - run: npm run lint - - typecheck: - runs-on: ubuntu-latest - defaults: - run: - working-directory: ./mathesar_ui - timeout-minutes: 5 - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-node@v2 - with: - node-version: 18 - - id: npm-cache-dir - run: echo "::set-output name=dir::$(npm config get cache)" - - uses: actions/cache@v2 - id: npm-cache - with: - path: ${{ steps.npm-cache-dir.outputs.dir }} - key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} - restore-keys: ${{ runner.os }}-node- - - run: npm ci --no-audit --prefer-offline - - run: npm run typecheck - - audit: - runs-on: ubuntu-latest - defaults: - run: - working-directory: ./mathesar_ui - timeout-minutes: 5 - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-node@v2 - with: - node-version: 18 - - id: npm-cache-dir - run: echo "::set-output name=dir::$(npm config get cache)" - - uses: actions/cache@v2 - id: npm-cache - with: - path: ${{ steps.npm-cache-dir.outputs.dir }} - key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} - restore-keys: ${{ runner.os }}-node- - - run: npm ci - - uses: oke-py/npm-audit-action@v1.8.2 - with: - audit_level: moderate - github_token: ${{ secrets.GITHUB_TOKEN }} - create_pr_comments: false - dedupe_issues: true - working_directory: './mathesar_ui' - issue_labels: 'restricted: maintainers,type: bug,work: frontend,status: triage' - production_flag: true - continue-on-error: true - - tests: - runs-on: ubuntu-latest - defaults: - run: - working-directory: ./mathesar_ui - timeout-minutes: 15 - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-node@v2 - with: - node-version: 18 - - id: npm-cache-dir - run: echo "::set-output name=dir::$(npm config get cache)" - - uses: actions/cache@v2 - id: npm-cache - with: - path: ${{ steps.npm-cache-dir.outputs.dir }} - key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} - restore-keys: ${{ runner.os }}-node- - - run: npm ci --no-audit --prefer-offline - - run: npm test diff --git a/.github/workflows/run-pytest.yml b/.github/workflows/run-pytest.yml deleted file mode 100644 index d20f5a1d31..0000000000 --- a/.github/workflows/run-pytest.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: Run Python tests -on: - push: - paths: - - '*.py' - - 'mathesar/**' - - 'db/**' - pull_request: - paths: - - '*.py' - - 'mathesar/**' - - 'db/**' - - -jobs: - tests: - runs-on: ubuntu-latest - # We only want to run on external PRs, since internal PRs are covered by "push" - # This prevents this from running twice on internal PRs - if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository - strategy: - matrix: - pg-version: [13, 14, 15] - steps: - - uses: actions/checkout@v2 - - - name: Copy env file - run: cp .env.example .env - - # The code is checked out under uid 1001 - reset this to 1000 for the - # container to run tests successfully - - name: Fix permissions - run: sudo chown -R 1000:1000 . - - - name: Build the stack - run: docker build -t mathesar_service_test --build-arg BUILD_PG_MAJOR=${{ matrix.pg-version }} --build-arg PYTHON_REQUIREMENTS=requirements-dev.txt . - env: - PG_MAJOR: ${{ matrix.pg-version }} - - name: Run docker container - run: docker run -d --name mathesar_service_test mathesar_service_test - # Override setup.cfg docker-compose db service credentials with in built docker postgres server - - name: Run tests with pytest - run: docker exec -e DJANGO_DATABASE_URL="postgres://postgres:mathesar@localhost:5432/mathesar_django" -e MATHESAR_DATABASES="(mathesar_db_test|postgres://postgres:mathesar@localhost:5432/mathesar_db_test)" mathesar_service_test ./run_pytest.sh diff --git a/.github/workflows/run-sql-tests.yml b/.github/workflows/run-sql-tests.yml deleted file mode 100644 index d5761bc3ab..0000000000 --- a/.github/workflows/run-sql-tests.yml +++ /dev/null @@ -1,37 +0,0 @@ -name: Run SQL tests -on: - push: - paths: - - '**.sql' - pull_request: - paths: - - '**.sql' - - -jobs: - tests: - runs-on: ubuntu-latest - # We only want to run on external PRs, since internal PRs are covered by "push" - # This prevents this from running twice on internal PRs - if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository - strategy: - matrix: - pg-version: [13, 14, 15] - steps: - - uses: actions/checkout@v2 - - - name: Copy env file - run: cp .env.example .env - - # The code is checked out under uid 1001 - reset this to 1000 for the - # container to run tests successfully - - name: Fix permissions - run: sudo chown -R 1000:1000 . - - - name: Build the test DB - run: docker compose -f docker-compose.yml -f docker-compose.dev.yml up --build -d dev-db - env: - PG_VERSION: ${{ matrix.pg-version }} - - - name: Run tests with pg_prove - run: docker exec mathesar_dev_db /bin/bash /sql/run_tests.sh diff --git a/.github/workflows/run-vulture.yml b/.github/workflows/run-vulture.yml deleted file mode 100644 index b4533aedb0..0000000000 --- a/.github/workflows/run-vulture.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: Find unused code - -on: - push: - paths: - - '**.py' - pull_request: - paths: - - '**.py' - -jobs: - build: - runs-on: ubuntu-latest - # We only want to run on external PRs, since internal PRs are covered by "push" - # This prevents this from running twice on internal PRs - if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - - name: Install Vulture - run: pip3 install vulture - - name: Run Vulture - run: vulture . diff --git a/.github/workflows/test-and-lint-code.yml b/.github/workflows/test-and-lint-code.yml new file mode 100644 index 0000000000..b59fbe713c --- /dev/null +++ b/.github/workflows/test-and-lint-code.yml @@ -0,0 +1,329 @@ +## This workflow runs all tests, audits, and linters for Mathesar's source code. +## It's set up to run for every pull request change, as well as when a pull +## request is added to the merge queue. This, combined with our branch +## protections requiring pull requests and a merge queue for `develop` and +## `master` ensures that no code is merged into those branches without +## appropriate tests having run. + +name: Test and Lint Code +on: [pull_request, merge_group] +jobs: + +################################################################################ +## FILE CHANGE CHECKERS ## +## ## +## These jobs check which files have changed so that we can call appropriate ## +## testing, auditing, and linting jobs. Jobs in this section should check for ## +## file changes that would indicate whether we need to run some particular ## +## test suite, then they should output 'true' if such file changes are found. ## +## ## +################################################################################ + + python_tests_required: + name: Check for file changes requiring python tests + runs-on: ubuntu-latest + outputs: + tests_should_run: ${{ steps.changed_files.outputs.any_changed }} + steps: + - uses: actions/checkout@v4 + - name: Get changed files + id: changed_files + uses: tj-actions/changed-files@v39 + with: + files: | + *.py + mathesar/** + db/** + + python_lint_required: + name: Check for file changes requiring python linter + runs-on: ubuntu-latest + outputs: + lint_should_run: ${{ steps.changed_files.outputs.any_changed }} + steps: + - uses: actions/checkout@v4 + - name: Get changed files + id: changed_files + uses: tj-actions/changed-files@v39 + with: + files: '**.py' + + sql_tests_required: + name: Check for file changes requiring SQL tests + runs-on: ubuntu-latest + outputs: + tests_should_run: ${{ steps.changed_files.outputs.any_changed }} + steps: + - uses: actions/checkout@v4 + - name: Get changed files + id: changed_files + uses: tj-actions/changed-files@v39 + with: + files: '**.sql' + + front_end_checks_required: + name: Check for file changes requiring front end checks + runs-on: ubuntu-latest + outputs: + checks_should_run: ${{ steps.changed_files.outputs.any_changed }} + steps: + - uses: actions/checkout@v4 + - name: Get changed files + id: changed_files + uses: tj-actions/changed-files@v39 + with: + files: 'mathesar_ui/**' + +################################################################################ +## BACK END TEST/LINT RUNNERS ## +## ## +## These jobs run tests and linters. Each job in this section should be ## +## dependent on one of the FILE CHANGE CHECKERS above, and should only run if ## +## appropriate files have changed. You can see this by using a `needs:` block ## +## to make the job dependent on the relevant file checker, and an `if:` block ## +## to ensure that the file checker returned 'true' before running the actual ## +## job. Job IDs in this section must be namespaced (so `python_tests` ## +## instead of just `tests`). ## +## ## +################################################################################ + + python_tests: + name: Run Python tests + runs-on: ubuntu-latest + needs: python_tests_required + if: needs.python_tests_required.outputs.tests_should_run == 'true' + strategy: + matrix: + pg-version: [13, 14, 15] + steps: + - uses: actions/checkout@v4 + - name: Copy env file + run: cp .env.example .env + # The code is checked out under uid 1001 - reset this to 1000 for the + # container to run tests successfully + - name: Fix permissions + run: sudo chown -R 1000:1000 . + - name: Build the stack + run: docker compose -f docker-compose.yml -f docker-compose.dev.yml up --build -d test-service + env: + PG_VERSION: ${{ matrix.pg-version }} + - name: Run tests with pytest + run: docker exec mathesar_service_test ./run_pytest.sh + + sql_tests: + name: Run SQL tests + runs-on: ubuntu-latest + needs: sql_tests_required + if: needs.sql_tests_required.outputs.tests_should_run == 'true' + strategy: + matrix: + pg-version: [13, 14, 15] + steps: + - uses: actions/checkout@v4 + - name: Copy env file + run: cp .env.example .env + # The code is checked out under uid 1001 - reset this to 1000 for the + # container to run tests successfully + - name: Fix permissions + run: sudo chown -R 1000:1000 . + - name: Build the test DB + run: docker compose -f docker-compose.yml -f docker-compose.dev.yml up --build -d dev-db + env: + PG_VERSION: ${{ matrix.pg-version }} + - name: Run tests with pg_prove + run: docker exec mathesar_dev_db /bin/bash /sql/run_tests.sh + + python_lint: + name: Run Python linter + runs-on: ubuntu-latest + needs: python_lint_required + if: needs.python_lint_required.outputs.lint_should_run == 'true' + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v4 + - name: Run flake8 + uses: julianwachholz/flake8-action@main + with: + checkName: "flake8" + path: "." + plugins: flake8-no-types + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + vulture: + name: Find unused code + runs-on: ubuntu-latest + needs: python_lint_required + if: needs.python_lint_required.outputs.lint_should_run == 'true' + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v4 + - name: Install Vulture + run: pip3 install vulture + - name: Run Vulture + run: vulture . + +################################################################################ +## FRONT END CHECK RUNNERS ## +## ## +## These jobs run front end checks. Each job in this section should be ## +## dependent on one of the FILE CHANGE CHECKERS above (currently only ## +## front_end_checks_required), and should only run if appropriate files have ## +## changed. You can see this by using a `needs:` block to make the job ## +## dependent on the relevant file checker, and an `if:` block to ensure that ## +## the file checker returned 'true' before running the actual job. `lint` and ## +## `tests` Job IDs in this section must be namespaced (`front_end_tests` ## +## rather than `tests`). ## +## ## +################################################################################ + + front_end_format: + name: Check front end code format + runs-on: ubuntu-latest + needs: front_end_checks_required + if: needs.front_end_checks_required.outputs.checks_should_run == 'true' + defaults: + run: + working-directory: ./mathesar_ui + timeout-minutes: 5 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v3 + with: + node-version: 18 + - id: npm-cache-dir + run: echo "::set-output name=dir::$(npm config get cache)" + - uses: actions/cache@v3 + id: npm-cache + with: + path: ${{ steps.npm-cache-dir.outputs.dir }} + key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} + restore-keys: ${{ runner.os }}-node- + - run: npm ci --no-audit --prefer-offline + - run: npm run check-format + + front_end_lint: + name: Run front end linter + runs-on: ubuntu-latest + needs: front_end_checks_required + if: needs.front_end_checks_required.outputs.checks_should_run == 'true' + defaults: + run: + working-directory: ./mathesar_ui + timeout-minutes: 5 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v3 + with: + node-version: 18 + - id: npm-cache-dir + run: echo "::set-output name=dir::$(npm config get cache)" + - uses: actions/cache@v3 + id: npm-cache + with: + path: ${{ steps.npm-cache-dir.outputs.dir }} + key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} + restore-keys: ${{ runner.os }}-node- + - run: npm ci --no-audit --prefer-offline + - run: npm run lint + + front_end_typecheck: + name: Check front end types + runs-on: ubuntu-latest + needs: front_end_checks_required + if: needs.front_end_checks_required.outputs.checks_should_run == 'true' + defaults: + run: + working-directory: ./mathesar_ui + timeout-minutes: 5 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v3 + with: + node-version: 18 + - id: npm-cache-dir + run: echo "::set-output name=dir::$(npm config get cache)" + - uses: actions/cache@v3 + id: npm-cache + with: + path: ${{ steps.npm-cache-dir.outputs.dir }} + key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} + restore-keys: ${{ runner.os }}-node- + - run: npm ci --no-audit --prefer-offline + - run: npm run typecheck + + front_end_audit: + name: Audit front end code + runs-on: ubuntu-latest + needs: front_end_checks_required + if: needs.front_end_checks_required.outputs.checks_should_run == 'true' + defaults: + run: + working-directory: ./mathesar_ui + timeout-minutes: 5 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v3 + with: + node-version: 18 + - id: npm-cache-dir + run: echo "::set-output name=dir::$(npm config get cache)" + - uses: actions/cache@v3 + id: npm-cache + with: + path: ${{ steps.npm-cache-dir.outputs.dir }} + key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} + restore-keys: ${{ runner.os }}-node- + - run: npm ci + - uses: oke-py/npm-audit-action@v1.8.2 + with: + audit_level: moderate + github_token: ${{ secrets.GITHUB_TOKEN }} + create_pr_comments: false + dedupe_issues: true + working_directory: './mathesar_ui' + issue_labels: 'restricted: maintainers,type: bug,work: frontend,status: triage' + production_flag: true + continue-on-error: true + + front_end_tests: + name: Run front end tests + runs-on: ubuntu-latest + needs: front_end_checks_required + if: needs.front_end_checks_required.outputs.checks_should_run == 'true' + defaults: + run: + working-directory: ./mathesar_ui + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v3 + with: + node-version: 18 + - id: npm-cache-dir + run: echo "::set-output name=dir::$(npm config get cache)" + - uses: actions/cache@v3 + id: npm-cache + with: + path: ${{ steps.npm-cache-dir.outputs.dir }} + key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} + restore-keys: ${{ runner.os }}-node- + - run: npm ci --no-audit --prefer-offline + - run: npm test + +################################################################################ +## REQUIRED TEST/LINT COLLECTORS ## +## Jobs in this section collect outputs from matrix-strategy testing jobs, ## +## since these are otherwise impossible to capture for branch protection ## +## purposes. At the moment, they only need to have each required check as a ## +## dependency. Required checks should skip themselves if no relevant files ## +## have changed. ## +## ## +################################################################################ + + matrix_tests: + runs-on: ubuntu-latest + needs: [python_tests, sql_tests] + steps: + - name: Report success + run: echo "All tests succeeded or skipped!"