Skip to content
Build

add connection to gcs and use different context for upload incase it … #644

Sign in to view logs

add connection to gcs and use different context for upload incase it …

add connection to gcs and use different context for upload incase it … #644

Workflow file for this run

.github/workflows/build.yaml at 23c7b7f
name: Build
on:
pull_request:
branches:
- master
push:
branches:
- master
jobs:
build:
name: Build
runs-on: ubuntu-22.04
strategy:
matrix:
golang-version:
- "1.20"
steps:
- name: Checkout project
uses: actions/checkout@v3
- name: Setup golang
id: setup-go
uses: actions/setup-go@v3
with:
go-version: '^${{ matrix.golang-version }}'
- name: Setup musl
id: setup-musl
run: |
sudo apt-get install -y musl-tools musl-dev
- name: Cache golang
id: cache-golang
uses: actions/cache@v3
with:
path: |
~/go/pkg/mod
~/.cache/go-build
key: clickhouse-backup-golang-${{ matrix.golang-version }}-${{ hashFiles('go.mod', '.github/workflows/*.yaml') }}
- name: Install golang dependencies
run: go mod download -x
if: |
steps.cache-golang.outputs.cache-hit != 'true'
- name: Build clickhouse-backup binary
id: make-race
env:
GOROOT: ${{ env.GOROOT_1_20_X64 }}
run: |
make build/linux/amd64/clickhouse-backup build/linux/arm64/clickhouse-backup
make build/linux/amd64/clickhouse-backup-fips build/linux/arm64/clickhouse-backup-fips
make build-race build-race-fips config test
- name: Report unittest coverage
uses: coverallsapp/github-action@v2
with:
file: _coverage_/coverage.out
parallel: true
format: golang
flag-name: unittest-${{ matrix.clickhouse }}
# be careful with encrypt with old OpenSSL - https://habr.com/ru/post/535140/
# openssl enc -base64 -aes-256-cbc -e -in test/integration/credentials.json -out test/integration/credentials.json.enc -md md5 -k ${VAULT_PASSWORD}
- name: Decrypting credentials for Google Cloud Storage
id: secrets
env:
VAULT_PASSWORD: ${{ secrets.VAULT_PASSWORD }}
run: |
if [[ "" != "${VAULT_PASSWORD}" ]]; then
openssl version
openssl enc -base64 -aes-256-cbc -d -in test/integration/credentials.json.enc -out test/integration/credentials.json -md md5 -k ${VAULT_PASSWORD}
fi
echo "GCS_TESTS=$(if [ -z "${{ secrets.VAULT_PASSWORD }}" ]; then echo "false"; else echo "true"; fi)" >> $GITHUB_OUTPUT
- uses: actions/upload-artifact@v3
with:
name: build-gcp-credentials
path: |
test/integration/credentials.json
if-no-files-found: error
retention-days: 1
if: |
steps.secrets.outputs.GCS_TESTS == 'true'
- uses: actions/upload-artifact@v3
with:
name: build-artifacts
path: |
build/linux/amd64/clickhouse-backup
build/linux/arm64/clickhouse-backup
build/linux/amd64/clickhouse-backup-fips
build/linux/arm64/clickhouse-backup-fips
if-no-files-found: error
retention-days: 1
- uses: actions/upload-artifact@v3
with:
name: build-test-artifacts
path: |
clickhouse-backup/clickhouse-backup-race
clickhouse-backup/clickhouse-backup-race-fips
if-no-files-found: error
retention-days: 1
outputs:
GCS_TESTS: ${{ steps.secrets.outputs.GCS_TESTS }}
testflows:
needs: build
name: Testflows
runs-on: ubuntu-22.04
strategy:
matrix:
clickhouse:
- '22.3'
- '22.8'
- '23.3'
steps:
- name: Checkout project
uses: actions/checkout@v3
- uses: actions/download-artifact@v3
with:
name: build-test-artifacts
path: ./clickhouse-backup/
- name: Install python venv
run: |
set -x
(dpkg -l | grep venv) || (apt-get update && apt-get install -y python3-venv)
python3 -m venv ~/venv/qa
- name: Cache python
uses: actions/cache@v3
id: cache-python
with:
path: ~/venv/qa
key: clickhouse-backup-python-${{ hashFiles('test/testflows/requirements.txt','.github/workflows/*.yaml') }}
- name: Install python dependencies
run: |
set -x
~/venv/qa/bin/pip3 install -U -r ./test/testflows/requirements.txt
if: |
steps.cache-python.outputs.cache-hit != 'true'
- name: Running TestFlows tests
env:
CLICKHOUSE_VERSION: ${{ matrix.clickhouse }}
QA_AWS_ACCESS_KEY: ${{ secrets.QA_AWS_ACCESS_KEY }}
QA_AWS_ENDPOINT: ${{ secrets.QA_AWS_ENDPOINT }}
QA_AWS_SECRET_KEY: ${{ secrets.QA_AWS_SECRET_KEY }}
QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
QA_AWS_BUCKET: ${{ secrets.QA_AWS_BUCKET }}
QA_GCS_CRED_JSON: ${{ secrets.QA_GCS_CRED_JSON }}
# don't change it to avoid not working CI/CD
RUN_TESTS: "*"
run: |
set -x
export CLICKHOUSE_TESTS_DIR=$(pwd)/test/testflows/clickhouse_backup
command -v docker-compose || (apt-get update && apt-get install -y python3-pip && pip3 install -U docker-compose)
docker-compose -f ${CLICKHOUSE_TESTS_DIR}/docker-compose/docker-compose.yml pull
chmod +x $(pwd)/clickhouse-backup/clickhouse-backup*
source ~/venv/qa/bin/activate
set +e
~/venv/qa/bin/python3 ./test/testflows/clickhouse_backup/regression.py --debug --only="${RUN_TESTS:-*}" --log ./test/testflows/raw.log
if [[ "0" != "$?" ]]; then
docker-compose -f ${CLICKHOUSE_TESTS_DIR}/docker-compose/docker-compose.yml logs clickhouse_backup
exit 1
fi
set -e
tfs --debug --no-colors transform compact ./test/testflows/raw.log ./test/testflows/compact.log
tfs --debug --no-colors transform nice ./test/testflows/raw.log ./test/testflows/nice.log.txt
tfs --debug --no-colors transform short ./test/testflows/raw.log ./test/testflows/short.log.txt
tfs --debug --no-colors report results -a "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/" ./test/testflows/raw.log - --confidential --copyright "Altinity LTD" --logo ./test/testflows/altinity.png | ~/venv/qa/bin/tfs --debug --no-colors document convert > ./test/testflows/report.html
sudo chmod -Rv +rx test/testflows/clickhouse_backup/_instances
- name: Format testflows coverage
run: |
sudo chmod -Rv a+rw test/testflows/_coverage_/
ls -la test/testflows/_coverage_
go tool covdata textfmt -i test/testflows/_coverage_/ -o test/testflows/_coverage_/coverage.out
- name: Report testflows coverage
uses: coverallsapp/github-action@v2
with:
file: test/testflows/_coverage_/coverage.out
parallel: true
format: golang
flag-name: testflows-${{ matrix.clickhouse }}
# todo wait when resolve https://github.com/actions/upload-artifact/issues/270 and uncomment
# - name: Upload testflows logs
# uses: actions/upload-artifact@v3
# with:
# name: testflows-logs-and-reports-${{ matrix.clickhouse }}-${{ github.run_id }}
# path: |
# test/testflows/*.log
# test/testflows/*.log.txt
# test/testflows/clickhouse_backup/_instances/**/*.log
# test/testflows/*.html
# retention-days: 7
test:
needs: build
name: Test
runs-on: ubuntu-20.04
strategy:
matrix:
golang-version:
- "1.20"
clickhouse:
- '1.1.54394'
- '19.17'
- '20.3'
- '20.8'
- '21.3'
- '21.8'
- '22.3'
- '22.8'
- '23.3'
steps:
- name: Checkout project
uses: actions/checkout@v3
- name: Setup golang
id: setup-go
uses: actions/setup-go@v3
with:
go-version: '^${{ matrix.golang-version }}'
- name: Cache golang
id: cache-golang
uses: actions/cache@v3
with:
path: |
~/go/pkg/mod
~/.cache/go-build
key: clickhouse-backup-golang-${{ matrix.golang-version }}-${{ hashFiles('go.mod', '.github/workflows/*.yaml') }}
- uses: actions/download-artifact@v3
with:
name: build-test-artifacts
path: ./clickhouse-backup/
- uses: actions/download-artifact@v3
with:
name: build-gcp-credentials
path: ./test/integration/
if: |
needs.build.outputs.GCS_TESTS == 'true'
- name: Running integration tests
env:
CLICKHOUSE_VERSION: ${{ matrix.clickhouse }}
# don't change it to avoid broken CI/CD!!!
# RUN_TESTS: "TestFIPS"
# LOG_LEVEL: "debug"
# FTP_DEBUG: "true"
# S3_DEBUG: "true"
CGO_ENABLED: 0
GCS_TESTS: ${{ needs.build.outputs.GCS_TESTS }}
RUN_ADVANCED_TESTS: 1
AZURE_TESTS: 1
# need for FIPS
QA_AWS_ACCESS_KEY: ${{ secrets.QA_AWS_ACCESS_KEY }}
QA_AWS_SECRET_KEY: ${{ secrets.QA_AWS_SECRET_KEY }}
QA_AWS_BUCKET: ${{ secrets.QA_AWS_BUCKET }}
QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
# need for GCP over S3
QA_GCS_OVER_S3_ACCESS_KEY: ${{ secrets.QA_GCS_OVER_S3_ACCESS_KEY }}
QA_GCS_OVER_S3_SECRET_KEY: ${{ secrets.QA_GCS_OVER_S3_SECRET_KEY }}
QA_GCS_OVER_S3_BUCKET: ${{ secrets.QA_GCS_OVER_S3_BUCKET }}
run: |
set -x
echo "CLICKHOUSE_VERSION=${CLICKHOUSE_VERSION}"
echo "GCS_TESTS=${GCS_TESTS}"
chmod +x $(pwd)/clickhouse-backup/clickhouse-backup*
if [[ "${CLICKHOUSE_VERSION}" =~ 2[2-9]+ ]]; then
export CLICKHOUSE_IMAGE=clickhouse/clickhouse-server
else
export CLICKHOUSE_IMAGE=yandex/clickhouse-server
fi
if [[ "${CLICKHOUSE_VERSION}" == 2* ]]; then
export COMPOSE_FILE=docker-compose_advanced.yml
else
export COMPOSE_FILE=docker-compose.yml
fi
command -v docker-compose || (apt-get update && apt-get install -y python3-pip && pip3 install -U docker-compose)
export CLICKHOUSE_BACKUP_BIN="$(pwd)/clickhouse-backup/clickhouse-backup-race"
docker-compose -f test/integration/${COMPOSE_FILE} up -d || ( docker-compose -f test/integration/${COMPOSE_FILE} ps -a && docker-compose -f test/integration/${COMPOSE_FILE} logs clickhouse && exit 1 )
docker-compose -f test/integration/${COMPOSE_FILE} ps -a
go test -timeout 60m -failfast -tags=integration -run "${RUN_TESTS:-.+}" -v test/integration/integration_test.go
- name: Format integration coverage
run: |
sudo chmod -Rv a+rw test/integration/_coverage_/
ls -la test/integration/_coverage_
go tool covdata textfmt -i test/integration/_coverage_/ -o test/integration/_coverage_/coverage.out
- name: Report integration coverage
uses: coverallsapp/github-action@v2
with:
file: test/integration/_coverage_/coverage.out
parallel: true
format: golang
flag-name: integration-${{ matrix.clickhouse }}
coverage:
needs:
- test
- testflows
name: coverage
runs-on: ubuntu-latest
steps:
- name: Coveralls Finished
uses: coverallsapp/github-action@v2
with:
parallel-finished: true
docker:
needs:
- test
- testflows
name: Docker
runs-on: ubuntu-22.04
steps:
- name: Checkout project
uses: actions/checkout@v3
- uses: actions/download-artifact@v3
with:
name: build-artifacts
path: ./build/linux/
- name: Extract DOCKER_TAG version
id: docker_tag
run: |
DOCKER_TAG=${GITHUB_REF##*/}
export DOCKER_TAG=${DOCKER_TAG##*\\}
echo "docker_tag=${DOCKER_TAG:-dev}" >> $GITHUB_OUTPUT
- name: Set up QEMU
uses: docker/setup-qemu-action@v2
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
- name: Building docker image
env:
DOCKER_REPO: ${{ secrets.DOCKER_REPO }}
DOCKER_IMAGE: ${{ secrets.DOCKER_IMAGE }}
DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}
DOCKER_USER: ${{ secrets.DOCKER_USER }}
DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }}
DOCKER_TAG: ${{ steps.docker_tag.outputs.docker_tag }}
run: |
if [[ "${DOCKER_TOKEN}" != "" ]]; then
export DOCKER_REGISTRY=${DOCKER_REGISTRY:-docker.io}
echo ${DOCKER_TOKEN} | docker login -u ${DOCKER_USER} --password-stdin ${DOCKER_REGISTRY}
docker buildx build --progress=plain --platform=linux/amd64,linux/arm64 --tag=${DOCKER_REGISTRY}/${DOCKER_REPO}/${DOCKER_IMAGE}:${DOCKER_TAG} --target=image_short --pull --push .
docker buildx build --progress=plain --platform=linux/amd64,linux/arm64 --tag=${DOCKER_REGISTRY}/${DOCKER_REPO}/${DOCKER_IMAGE}:${DOCKER_TAG}-fips --target=image_fips --pull --push .
docker buildx build --progress=plain --platform=linux/amd64,linux/arm64 --tag=${DOCKER_REGISTRY}/${DOCKER_REPO}/${DOCKER_IMAGE}-full:${DOCKER_TAG} --target=image_full --pull --push .
fi
cleanup:
name: Cleanup
needs:
- docker
- coverage
runs-on: ubuntu-22.04
if: always()
steps:
- name: delete build-artifacts
uses: geekyeggo/delete-artifact@v2
with:
name: build-artifacts
failOnError: false
- name: delete build-gcp-credentials
uses: geekyeggo/delete-artifact@v2
with:
name: build-gcp-credentials
failOnError: false