tests

gha: split lint & unit tests stages #4021

Workflow file for this run

	---
	name: tests

	on:
	workflow_dispatch:

	push:
	branches-ignore:
	- 'development/**'
	- 'q//*'

	env:
	# Secrets
	azurebackend_AZURE_STORAGE_ACCESS_KEY: >-
	${{ secrets.AZURE_STORAGE_ACCESS_KEY }}
	azurebackend_AZURE_STORAGE_ACCOUNT_NAME: >-
	${{ secrets.AZURE_STORAGE_ACCOUNT_NAME }}
	azurebackend_AZURE_STORAGE_ENDPOINT: >-
	${{ secrets.AZURE_STORAGE_ENDPOINT }}
	azurebackend2_AZURE_STORAGE_ACCESS_KEY: >-
	${{ secrets.AZURE_STORAGE_ACCESS_KEY_2 }}
	azurebackend2_AZURE_STORAGE_ACCOUNT_NAME: >-
	${{ secrets.AZURE_STORAGE_ACCOUNT_NAME_2 }}
	azurebackend2_AZURE_STORAGE_ENDPOINT: >-
	${{ secrets.AZURE_STORAGE_ENDPOINT_2 }}
	azurebackendmismatch_AZURE_STORAGE_ACCESS_KEY: >-
	${{ secrets.AZURE_STORAGE_ACCESS_KEY }}
	azurebackendmismatch_AZURE_STORAGE_ACCOUNT_NAME: >-
	${{ secrets.AZURE_STORAGE_ACCOUNT_NAME }}
	azurebackendmismatch_AZURE_STORAGE_ENDPOINT: >-
	${{ secrets.AZURE_STORAGE_ENDPOINT }}
	azurenonexistcontainer_AZURE_STORAGE_ACCESS_KEY: >-
	${{ secrets.AZURE_STORAGE_ACCESS_KEY }}
	azurenonexistcontainer_AZURE_STORAGE_ACCOUNT_NAME: >-
	${{ secrets.AZURE_STORAGE_ACCOUNT_NAME }}
	azurenonexistcontainer_AZURE_STORAGE_ENDPOINT: >-
	${{ secrets.AZURE_STORAGE_ENDPOINT }}
	azuretest_AZURE_BLOB_ENDPOINT: "${{ secrets.AZURE_STORAGE_ENDPOINT }}"
	b2backend_B2_ACCOUNT_ID: "${{ secrets.B2BACKEND_B2_ACCOUNT_ID }}"
	b2backend_B2_STORAGE_ACCESS_KEY: >-
	${{ secrets.B2BACKEND_B2_STORAGE_ACCESS_KEY }}
	GOOGLE_SERVICE_EMAIL: "${{ secrets.GCP_SERVICE_EMAIL }}"
	GOOGLE_SERVICE_KEY: "${{ secrets.GCP_SERVICE_KEY }}"
	AWS_S3_BACKEND_ACCESS_KEY: "${{ secrets.AWS_S3_BACKEND_ACCESS_KEY }}"
	AWS_S3_BACKEND_SECRET_KEY: "${{ secrets.AWS_S3_BACKEND_SECRET_KEY }}"
	AWS_S3_BACKEND_ACCESS_KEY_2: "${{ secrets.AWS_S3_BACKEND_ACCESS_KEY_2 }}"
	AWS_S3_BACKEND_SECRET_KEY_2: "${{ secrets.AWS_S3_BACKEND_SECRET_KEY_2 }}"
	AWS_GCP_BACKEND_ACCESS_KEY: "${{ secrets.AWS_GCP_BACKEND_ACCESS_KEY }}"
	AWS_GCP_BACKEND_SECRET_KEY: "${{ secrets.AWS_GCP_BACKEND_SECRET_KEY }}"
	AWS_GCP_BACKEND_ACCESS_KEY_2: "${{ secrets.AWS_GCP_BACKEND_ACCESS_KEY_2 }}"
	AWS_GCP_BACKEND_SECRET_KEY_2: "${{ secrets.AWS_GCP_BACKEND_SECRET_KEY_2 }}"
	b2backend_B2_STORAGE_ENDPOINT: "${{ secrets.B2BACKEND_B2_STORAGE_ENDPOINT }}"
	gcpbackend2_GCP_SERVICE_EMAIL: "${{ secrets.GCP2_SERVICE_EMAIL }}"
	gcpbackend2_GCP_SERVICE_KEY: "${{ secrets.GCP2_SERVICE_KEY }}"
	gcpbackend2_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
	gcpbackend_GCP_SERVICE_EMAIL: "${{ secrets.GCP_SERVICE_EMAIL }}"
	gcpbackend_GCP_SERVICE_KEY: "${{ secrets.GCP_SERVICE_KEY }}"
	gcpbackendmismatch_GCP_SERVICE_EMAIL: >-
	${{ secrets.GCPBACKENDMISMATCH_GCP_SERVICE_EMAIL }}
	gcpbackendmismatch_GCP_SERVICE_KEY: >-
	${{ secrets.GCPBACKENDMISMATCH_GCP_SERVICE_KEY }}
	gcpbackend_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
	gcpbackendmismatch_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
	gcpbackendnoproxy_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
	gcpbackendproxy_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
	# Configs
	ENABLE_LOCAL_CACHE: "true"
	REPORT_TOKEN: "report-token-1"
	REMOTE_MANAGEMENT_DISABLE: "1"
	# https://github.com/git-lfs/git-lfs/issues/5749
	GIT_CLONE_PROTECTION_ACTIVE: 'false'
	jobs:
	lint:
	runs-on: ubuntu-latest
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	- uses: actions/setup-node@v4
	with:
	node-version: '16'
	cache: yarn
	- name: install typescript
	shell: bash
	run: yarn global add [email protected]
	- name: install dependencies
	run: yarn install --frozen-lockfile --network-concurrency 1
	- uses: actions/setup-python@v5
	with:
	python-version: '3.9'
	cache: pip
	- name: Install python deps
	run: pip install flake8
	- name: Lint Javascript
	run: yarn run --silent lint -- --max-warnings 0
	- name: Lint Markdown
	run: yarn run --silent lint_md
	- name: Lint python
	run: flake8 $(git ls-files "*.py")
	- name: Lint Yaml
	run: yamllint -c yamllint.yml $(git ls-files "*.yml")

	unit-tests:
	runs-on: ubuntu-latest
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	- uses: actions/setup-node@v4
	with:
	node-version: '16'
	cache: yarn
	- name: install typescript
	shell: bash
	run: yarn global add [email protected]
	- name: install dependencies
	run: yarn install --frozen-lockfile --network-concurrency 1
	- name: Unit Coverage
	run: \|
	set -ex
	yarn run cover test
	yarn run cover test_legacy_location
	env:
	S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
	- name: Upload test results to Codecov
	uses: codecov/test-results-action@v1
	with:
	token: ${{ secrets.CODECOV_TOKEN }}
	files: '*/junit/junit*.xml'
	flags: unit
	- name: Upload coverage reports to Codecov
	uses: codecov/codecov-action@v5
	with:
	token: ${{ secrets.CODECOV_TOKEN }}
	directory: ./coverage/test
	flags: unit
	if: ${{ !cancelled() }}
	- name: preparing junit files for upload
	run: \|
	mkdir -p artifacts/junit
	find . -name "junit.xml" -exec cp {} artifacts/junit/ ";"
	if: always()
	- name: Upload files to artifacts
	uses: scality/action-artifacts@v4
	with:
	method: upload
	url: https://artifacts.scality.net
	user: ${{ secrets.ARTIFACTS_USER }}
	password: ${{ secrets.ARTIFACTS_PASSWORD }}
	source: artifacts
	if: always()

	build:
	runs-on: ubuntu-20.04
	permissions:
	contents: read
	packages: write
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	- name: Set up Docker Buildx
	uses: docker/setup-buildx-action@v3
	- name: Login to GitHub Registry
	uses: docker/login-action@v3
	with:
	registry: ghcr.io
	username: ${{ github.repository_owner }}
	password: ${{ github.token }}
	- name: Build and push cloudserver image
	uses: docker/build-push-action@v5
	with:
	push: true
	context: .
	provenance: false
	tags: \|
	ghcr.io/${{ github.repository }}:${{ github.sha }}
	labels: \|
	git.repository=${{ github.repository }}
	git.commit-sha=${{ github.sha }}
	cache-from: type=gha,scope=cloudserver
	cache-to: type=gha,mode=max,scope=cloudserver
	- name: Build and push pykmip image
	uses: docker/build-push-action@v5
	with:
	push: true
	context: .github/pykmip
	tags: \|
	ghcr.io/${{ github.repository }}/pykmip:${{ github.sha }}
	labels: \|
	git.repository=${{ github.repository }}
	git.commit-sha=${{ github.sha }}
	cache-from: type=gha,scope=pykmip
	cache-to: type=gha,mode=max,scope=pykmip
	- name: Build and push MongoDB
	uses: docker/build-push-action@v5
	with:
	push: true
	context: .github/docker/mongodb
	tags: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
	cache-from: type=gha,scope=mongodb
	cache-to: type=gha,mode=max,scope=mongodb

	multiple-backend:
	runs-on: ubuntu-latest
	needs: build
	env:
	CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
	MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
	S3BACKEND: mem
	S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigTests.json
	S3DATA: multiple
	JOB_NAME: ${{ github.job }}
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	- name: Login to Registry
	uses: docker/login-action@v3
	with:
	registry: ghcr.io
	username: ${{ github.repository_owner }}
	password: ${{ github.token }}
	- name: Setup CI environment
	uses: ./.github/actions/setup-ci
	- name: Setup CI services
	run: docker compose --profile sproxyd up -d
	working-directory: .github/docker
	- name: Run multiple backend test
	run: \|-
	set -o pipefail;
	bash wait_for_local_port.bash 8000 40
	bash wait_for_local_port.bash 81 40
	yarn run multiple_backend_test \| tee /tmp/artifacts/${{ github.job }}/tests.log
	env:
	S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
	- name: Upload logs to artifacts
	uses: scality/action-artifacts@v4
	with:
	method: upload
	url: https://artifacts.scality.net
	user: ${{ secrets.ARTIFACTS_USER }}
	password: ${{ secrets.ARTIFACTS_PASSWORD }}
	source: /tmp/artifacts
	if: always()

	mongo-v0-ft-tests:
	runs-on: ubuntu-latest
	needs: build
	env:
	S3BACKEND: mem
	MPU_TESTING: "yes"
	S3METADATA: mongodb
	S3KMS: file
	S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigTests.json
	DEFAULT_BUCKET_KEY_FORMAT: v0
	MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
	CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
	JOB_NAME: ${{ github.job }}
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	- name: Setup CI environment
	uses: ./.github/actions/setup-ci
	- name: Setup CI services
	run: docker compose --profile mongo up -d
	working-directory: .github/docker
	- name: Run functional tests
	run: \|-
	set -o pipefail;
	bash wait_for_local_port.bash 8000 40
	yarn run ft_test \| tee /tmp/artifacts/${{ github.job }}/tests.log
	env:
	S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
	- name: Upload logs to artifacts
	uses: scality/action-artifacts@v4
	with:
	method: upload
	url: https://artifacts.scality.net
	user: ${{ secrets.ARTIFACTS_USER }}
	password: ${{ secrets.ARTIFACTS_PASSWORD }}
	source: /tmp/artifacts
	if: always()

	mongo-v1-ft-tests:
	runs-on: ubuntu-latest
	needs: build
	env:
	S3BACKEND: mem
	MPU_TESTING: "yes"
	S3METADATA: mongodb
	S3KMS: file
	S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigTests.json
	DEFAULT_BUCKET_KEY_FORMAT: v1
	METADATA_MAX_CACHED_BUCKETS: 1
	MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
	CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
	JOB_NAME: ${{ github.job }}
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	- name: Setup CI environment
	uses: ./.github/actions/setup-ci
	- name: Setup CI services
	run: docker compose --profile mongo up -d
	working-directory: .github/docker
	- name: Run functional tests
	run: \|-
	set -o pipefail;
	bash wait_for_local_port.bash 8000 40
	yarn run ft_test \| tee /tmp/artifacts/${{ github.job }}/tests.log
	yarn run ft_mixed_bucket_format_version \| tee /tmp/artifacts/${{ github.job }}/mixed-tests.log
	env:
	S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
	- name: Upload logs to artifacts
	uses: scality/action-artifacts@v4
	with:
	method: upload
	url: https://artifacts.scality.net
	user: ${{ secrets.ARTIFACTS_USER }}
	password: ${{ secrets.ARTIFACTS_PASSWORD }}
	source: /tmp/artifacts
	if: always()

	file-ft-tests:
	strategy:
	matrix:
	include:
	- job-name: file-ft-tests
	name: ${{ matrix.job-name }}
	runs-on: ubuntu-latest
	needs: build
	env:
	S3BACKEND: file
	S3VAULT: mem
	CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
	MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
	MPU_TESTING: "yes"
	JOB_NAME: ${{ matrix.job-name }}
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	- name: Setup CI environment
	uses: ./.github/actions/setup-ci
	- name: Setup matrix job artifacts directory
	shell: bash
	run: \|
	set -exu
	mkdir -p /tmp/artifacts/${{ matrix.job-name }}/
	- name: Setup CI services
	run: docker compose up -d
	working-directory: .github/docker
	- name: Run file ft tests
	run: \|-
	set -o pipefail;
	bash wait_for_local_port.bash 8000 40
	yarn run ft_test \| tee /tmp/artifacts/${{ matrix.job-name }}/tests.log
	- name: Upload logs to artifacts
	uses: scality/action-artifacts@v4
	with:
	method: upload
	url: https://artifacts.scality.net
	user: ${{ secrets.ARTIFACTS_USER }}
	password: ${{ secrets.ARTIFACTS_PASSWORD }}
	source: /tmp/artifacts
	if: always()

	utapi-v2-tests:
	runs-on: ubuntu-latest
	needs: build
	env:
	ENABLE_UTAPI_V2: t
	S3BACKEND: mem
	BUCKET_DENY_FILTER: utapi-event-filter-deny-bucket
	CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
	MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
	JOB_NAME: ${{ github.job }}
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	- name: Setup CI environment
	uses: ./.github/actions/setup-ci
	- name: Setup CI services
	run: docker compose up -d
	working-directory: .github/docker
	- name: Run file utapi v2 tests
	run: \|-
	set -ex -o pipefail;
	bash wait_for_local_port.bash 8000 40
	yarn run test_utapi_v2 \| tee /tmp/artifacts/${{ github.job }}/tests.log
	- name: Upload logs to artifacts
	uses: scality/action-artifacts@v4
	with:
	method: upload
	url: https://artifacts.scality.net
	user: ${{ secrets.ARTIFACTS_USER }}
	password: ${{ secrets.ARTIFACTS_PASSWORD }}
	source: /tmp/artifacts
	if: always()

	quota-tests:
	runs-on: ubuntu-latest
	needs: build
	strategy:
	matrix:
	inflights:
	- name: "With Inflights"
	value: "true"
	- name: "Without Inflights"
	value: "false"
	env:
	S3METADATA: mongodb
	S3BACKEND: mem
	S3QUOTA: scuba
	QUOTA_ENABLE_INFLIGHTS: ${{ matrix.inflights.value }}
	SCUBA_HOST: localhost
	SCUBA_PORT: 8100
	SCUBA_HEALTHCHECK_FREQUENCY: 100
	CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
	MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
	JOB_NAME: ${{ github.job }}
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	- name: Setup CI environment
	uses: ./.github/actions/setup-ci
	- name: Setup CI services
	run: docker compose --profile mongo up -d
	working-directory: .github/docker
	- name: Run quota tests
	run: \|-
	set -ex -o pipefail;
	bash wait_for_local_port.bash 8000 40
	yarn run test_quota \| tee /tmp/artifacts/${{ github.job }}/tests.log
	- name: Upload logs to artifacts
	uses: scality/action-artifacts@v4
	with:
	method: upload
	url: https://artifacts.scality.net
	user: ${{ secrets.ARTIFACTS_USER }}
	password: ${{ secrets.ARTIFACTS_PASSWORD }}
	source: /tmp/artifacts
	if: always()

	kmip-ft-tests:
	runs-on: ubuntu-latest
	needs: build
	env:
	S3BACKEND: file
	S3VAULT: mem
	MPU_TESTING: "yes"
	CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
	PYKMIP_IMAGE: ghcr.io/${{ github.repository }}/pykmip:${{ github.sha }}
	MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
	JOB_NAME: ${{ github.job }}
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	- name: Setup CI environment
	uses: ./.github/actions/setup-ci
	- name: Copy KMIP certs
	run: cp -r ./certs /tmp/ssl-kmip
	working-directory: .github/pykmip
	- name: Setup CI services
	run: docker compose --profile pykmip up -d
	working-directory: .github/docker
	- name: Run file KMIP tests
	run: \|-
	set -ex -o pipefail;
	bash wait_for_local_port.bash 8000 40
	bash wait_for_local_port.bash 5696 40
	yarn run ft_kmip \| tee /tmp/artifacts/${{ github.job }}/tests.log
	- name: Upload logs to artifacts
	uses: scality/action-artifacts@v4
	with:
	method: upload
	url: https://artifacts.scality.net
	user: ${{ secrets.ARTIFACTS_USER }}
	password: ${{ secrets.ARTIFACTS_PASSWORD }}
	source: /tmp/artifacts
	if: always()

	ceph-backend-test:
	runs-on: ubuntu-latest
	needs: build
	env:
	S3BACKEND: mem
	S3DATA: multiple
	S3KMS: file
	CI_CEPH: 'true'
	MPU_TESTING: "yes"
	S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigCeph.json
	MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
	CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
	JOB_NAME: ${{ github.job }}
	steps:
	- name: Checkout
	uses: actions/checkout@v4
	- name: Login to GitHub Registry
	uses: docker/login-action@v3
	with:
	registry: ghcr.io
	username: ${{ github.repository_owner }}
	password: ${{ github.token }}
	- name: Setup CI environment
	uses: ./.github/actions/setup-ci
	- uses: ruby/setup-ruby@v1
	with:
	ruby-version: '2.5.9'
	- name: Install Ruby dependencies
	run: \|
	gem install nokogiri:1.12.5 excon:0.109.0 fog-aws:1.3.0 json:2.7.6 mime-types:3.1 rspec:3.5
	- name: Install Java dependencies
	run: \|
	sudo apt-get update && sudo apt-get install -y --fix-missing default-jdk maven
	- name: Setup CI services
	run: docker compose --profile ceph up -d
	working-directory: .github/docker
	env:
	S3METADATA: mongodb
	- name: Run Ceph multiple backend tests
	run: \|-
	set -ex -o pipefail;
	bash .github/ceph/wait_for_ceph.sh
	bash wait_for_local_port.bash 27018 40
	bash wait_for_local_port.bash 8000 40
	yarn run multiple_backend_test \| tee /tmp/artifacts/${{ github.job }}/multibackend-tests.log
	env:
	S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
	S3METADATA: mem
	- name: Run Java tests
	run: \|-
	set -ex -o pipefail;
	mvn test \| tee /tmp/artifacts/${{ github.job }}/java-tests.log
	working-directory: tests/functional/jaws
	- name: Run Ruby tests
	run: \|-
	set -ex -o pipefail;
	rspec -fd --backtrace tests.rb \| tee /tmp/artifacts/${{ github.job }}/ruby-tests.log
	working-directory: tests/functional/fog
	- name: Run Javascript AWS SDK tests
	run: \|-
	set -ex -o pipefail;
	yarn run ft_awssdk \| tee /tmp/artifacts/${{ github.job }}/js-awssdk-tests.log;
	yarn run ft_s3cmd \| tee /tmp/artifacts/${{ github.job }}/js-s3cmd-tests.log;
	env:
	S3_LOCATION_FILE: tests/locationConfig/locationConfigCeph.json
	S3BACKEND: file
	S3VAULT: mem
	S3METADATA: mongodb
	- name: Upload logs to artifacts
	uses: scality/action-artifacts@v4
	with:
	method: upload
	url: https://artifacts.scality.net
	user: ${{ secrets.ARTIFACTS_USER }}
	password: ${{ secrets.ARTIFACTS_PASSWORD }}
	source: /tmp/artifacts
	if: always()

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

gha: split lint & unit tests stages #4021

Workflow file

gha: split lint & unit tests stages #4021

Jobs

Run details

Workflow file for this run