From d490a65433724969b8d8d6a36b34ffa4942900ab Mon Sep 17 00:00:00 2001 From: "Wruck_Schneider,Dr.,Hugo (BI X) BIX-DE-I" Date: Fri, 10 Jan 2020 11:24:55 +0100 Subject: [PATCH] Adding test infrastructure --- .../continuous-integration-workflow.yml | 72 +++++++ .travis.yml | 24 +-- create-projects/create-cd-jenkins.sh | 6 + create-projects/create-projects.sh | 2 +- docs/antora.yml | 1 + docs/modules/keycloak/nav.adoc | 1 + docs/modules/keycloak/pages/index.adoc | 59 +++++ infrastructure-setup/ansible/requirements.yml | 2 +- .../roles/geerlingguy.git/.ansible-lint | 2 + .../ansible/roles/geerlingguy.git/.gitignore | 3 +- .../ansible/roles/geerlingguy.git/.travis.yml | 70 +++--- .../ansible/roles/geerlingguy.git/README.md | 2 +- .../roles/geerlingguy.git/defaults/main.yml | 2 +- .../geerlingguy.git/meta/.galaxy_install_info | 2 +- .../roles/geerlingguy.git/meta/main.yml | 30 +-- .../molecule/default/molecule.yml | 29 +++ .../molecule/default/playbook-source.yml | 18 ++ .../molecule/default/playbook.yml | 17 ++ .../molecule/default/yaml-lint.yml | 6 + .../tasks/install-from-source.yml | 71 +++--- .../roles/geerlingguy.git/tasks/main.yml | 27 +-- .../roles/geerlingguy.git/vars/Debian.yml | 9 + .../roles/geerlingguy.git/vars/Fedora.yml | 12 ++ .../roles/geerlingguy.git/vars/RedHat.yml | 11 + .../scripts/json/createBlobStores.json | 19 +- .../scripts/json/createRepos.json | 53 ++--- .../scripts/json/deactivateAnonymous.json | 2 +- infrastructure-setup/scripts/keycloak.sh | 63 +++++- jenkins/master/import_certs.sh | 4 +- jenkins/ocp-config/Tailorfile | 1 - jenkins/ocp-config/bc.yml | 10 +- jenkins/slave-base/Dockerfile.centos7 | 3 +- jenkins/slave-base/Dockerfile.rhel7 | 3 +- ocp-scripts/clone-project.sh | 29 ++- ods-setup/ocp-config/cd-user/Tailorfile | 5 + ods-setup/ocp-config/cd-user/secret.yml | 20 ++ ods-setup/setup-jenkins-images.sh | 70 ++++++ ods-setup/setup-ods-project.sh | 56 +++++ sonarqube/Dockerfile | 1 + tests/Makefile | 14 ++ .../create-projects/create-cd-jenkins_test.go | 106 +++++++++ tests/create-projects/create-projects_test.go | 138 ++++++++++++ tests/create-projects/jenkinsfile_test.go | 169 +++++++++++++++ tests/go.mod | 25 +++ tests/go.sum | 71 ++++++ tests/ods-setup/setup-ods-project_test.go | 87 ++++++++ tests/scripts/apply-docker-settings.sh | 21 ++ .../scripts/create-env-from-local-cluster.sh | 202 ++++++++++++++++++ tests/scripts/deploy-mocks.sh | 56 +++++ tests/scripts/json/create-cluster-role.json | 27 +++ tests/scripts/json/daemon.json | 12 ++ tests/scripts/recreate-test-infrastructure.sh | 65 ++++++ tests/scripts/setup-mocked-ods-repo.sh | 82 +++++++ tests/scripts/utils/print-jenkins-log.sh | 4 + tests/utils/clean-up.go | 54 +++++ tests/utils/command-utils.go | 37 ++++ tests/utils/constants.go | 7 + tests/utils/images.go | 20 ++ tests/utils/ods-env.go | 33 +++ tests/utils/openshift-client.go | 21 ++ tests/utils/projects.go | 15 ++ tests/utils/role-bindings.go | 28 +++ tests/utils/types.go | 24 +++ 63 files changed, 1957 insertions(+), 178 deletions(-) create mode 100644 .github/workflows/continuous-integration-workflow.yml mode change 100644 => 100755 create-projects/create-cd-jenkins.sh mode change 100644 => 100755 create-projects/create-projects.sh create mode 100644 docs/modules/keycloak/nav.adoc create mode 100644 docs/modules/keycloak/pages/index.adoc create mode 100644 infrastructure-setup/ansible/roles/geerlingguy.git/.ansible-lint create mode 100644 infrastructure-setup/ansible/roles/geerlingguy.git/molecule/default/molecule.yml create mode 100644 infrastructure-setup/ansible/roles/geerlingguy.git/molecule/default/playbook-source.yml create mode 100644 infrastructure-setup/ansible/roles/geerlingguy.git/molecule/default/playbook.yml create mode 100644 infrastructure-setup/ansible/roles/geerlingguy.git/molecule/default/yaml-lint.yml create mode 100644 infrastructure-setup/ansible/roles/geerlingguy.git/vars/Debian.yml create mode 100644 infrastructure-setup/ansible/roles/geerlingguy.git/vars/Fedora.yml create mode 100644 infrastructure-setup/ansible/roles/geerlingguy.git/vars/RedHat.yml create mode 100644 ods-setup/ocp-config/cd-user/Tailorfile create mode 100644 ods-setup/ocp-config/cd-user/secret.yml create mode 100755 ods-setup/setup-jenkins-images.sh create mode 100755 ods-setup/setup-ods-project.sh create mode 100644 tests/Makefile create mode 100644 tests/create-projects/create-cd-jenkins_test.go create mode 100644 tests/create-projects/create-projects_test.go create mode 100644 tests/create-projects/jenkinsfile_test.go create mode 100644 tests/go.mod create mode 100644 tests/go.sum create mode 100644 tests/ods-setup/setup-ods-project_test.go create mode 100755 tests/scripts/apply-docker-settings.sh create mode 100755 tests/scripts/create-env-from-local-cluster.sh create mode 100755 tests/scripts/deploy-mocks.sh create mode 100644 tests/scripts/json/create-cluster-role.json create mode 100644 tests/scripts/json/daemon.json create mode 100755 tests/scripts/recreate-test-infrastructure.sh create mode 100755 tests/scripts/setup-mocked-ods-repo.sh create mode 100755 tests/scripts/utils/print-jenkins-log.sh create mode 100644 tests/utils/clean-up.go create mode 100644 tests/utils/command-utils.go create mode 100644 tests/utils/constants.go create mode 100644 tests/utils/images.go create mode 100644 tests/utils/ods-env.go create mode 100644 tests/utils/openshift-client.go create mode 100644 tests/utils/projects.go create mode 100644 tests/utils/role-bindings.go create mode 100644 tests/utils/types.go diff --git a/.github/workflows/continuous-integration-workflow.yml b/.github/workflows/continuous-integration-workflow.yml new file mode 100644 index 000000000..b7a9c0344 --- /dev/null +++ b/.github/workflows/continuous-integration-workflow.yml @@ -0,0 +1,72 @@ +name: Continous Integration Tests +on: + pull_request: + branches: + - master +jobs: + test: + name: ODS resources setup and provisioning tests + runs-on: ubuntu-16.04 + steps: + - + name: GitHub context + env: + GITHUB_CONTEXT: ${{ toJson(github) }} + run: jq . <<< "${GITHUB_CONTEXT}" + - + name: Checkout repository + uses: actions/checkout@v2.0.0 + with: + fetch-depth: 0 + - + name: Setup Go 1.13 + uses: actions/setup-go@v1.0.0 + with: + version: 1.13 + - + name: Download OpenShift Client + run: | + wget https://github.com/openshift/origin/releases/download/v3.11.0/openshift-origin-client-tools-v3.11.0-0cbc58b-linux-64bit.tar.gz + tar -xzvf openshift-origin-client-tools-v3.11.0-0cbc58b-linux-64bit.tar.gz + sudo mv openshift-origin-client-tools-v3.11.0-0cbc58b-linux-64bit/oc /usr/local/bin/oc + - + name: Download Tailor + run: | + curl -LO "https://github.com/opendevstack/tailor/releases/download/v0.11.0/tailor-linux-amd64" + chmod +x tailor-linux-amd64 + sudo mv tailor-linux-amd64 /usr/local/bin/tailor + - + name: Tailor version + run: tailor version + - + name: OpenShift client version + run: oc version + - + name: jq version + run: jq --version + - + name: golang version + run: go version + - + name: Docker version + run: docker --version + - + name: Network before changes + run: ifconfig + - + name: Configure docker network and insecure registries + run: ./tests/scripts/apply-docker-settings.sh + - + name: Start OC cluster + run: oc cluster up --base-dir=${HOME}/openshift.local.clusterup --routing-suffix 172.17.0.1.nip.io --public-hostname 172.17.0.1 --enable=centos-imagestreams --enable=persistent-volumes --enable=registry --enable=router + - + name: Login into the cluster + run: oc login -u system:admin + - + name: Create test infrastructure + run: | + mkdir -p ods-config + ./tests/scripts/recreate-test-infrastructure.sh + - + name: Run tests + run: make -C tests test diff --git a/.travis.yml b/.travis.yml index d1c148228..f7ee08b53 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,16 +2,16 @@ # https://stackoverflow.com/questions/27644586/how-to-set-up-travis-ci-with-multiple-languages matrix: - include: - - language: go - go: - - "1.11.x" - before_install: - - go get golang.org/x/tools/cmd/goimports - - curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(go env GOPATH)/bin v1.17.1 + include: + - language: go + go: + - "1.11.x" + before_install: + - go get golang.org/x/tools/cmd/goimports + - curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(go env GOPATH)/bin v1.17.1 - before_script: - - cd jenkins/webhook-proxy - script: - - env GO111MODULE=on make lint - - env GO111MODULE=on make test + before_script: + - cd jenkins/webhook-proxy + script: + - env GO111MODULE=on make lint + - env GO111MODULE=on make test diff --git a/create-projects/create-cd-jenkins.sh b/create-projects/create-cd-jenkins.sh old mode 100644 new mode 100755 index 7a9a065a2..20f0e7bad --- a/create-projects/create-cd-jenkins.sh +++ b/create-projects/create-cd-jenkins.sh @@ -17,9 +17,11 @@ function usage { printf "\t-h|--help\tPrints the usage\n" printf "\t-v|--verbose\tVerbose output\n" printf "\t-t|--tailor\tChanges the executable of tailor. Default: tailor\n" + printf "\tods-namespace\tThe namespace where all ODS resources reside. Default: cd\n" } +ODS_NAMESPACE="cd" while [[ "$#" -gt 0 ]]; do case $1 in --status) STATUS=true;; @@ -33,6 +35,9 @@ while [[ "$#" -gt 0 ]]; do case $1 in -t=*|--tailor=*) TAILOR="${1#*=}";; -t|--tailor) TAILOR="$2"; shift;; + --ods-namespace=*) ODS_NAMESPACE="${1#*=}";; + --ods-namespace) ODS_NAMESPACE="$2"; shift;; + *) echo "Unknown parameter passed: $1"; usage; exit 1;; esac; shift; done @@ -82,5 +87,6 @@ tailor_update_in_dir "${SCRIPT_DIR}/ocp-config/cd-jenkins" \ "--param=PROXY_TRIGGER_SECRET_B64=${PIPELINE_TRIGGER_SECRET}" \ "--param=PROJECT=${PROJECT_ID}" \ "--param=CD_USER_ID_B64=${CD_USER_ID_B64}" \ + "--param=NAMESPACE=${ODS_NAMESPACE}" \ $cdUserPwdParam \ --selector "template=cd-jenkins-template" diff --git a/create-projects/create-projects.sh b/create-projects/create-projects.sh old mode 100644 new mode 100755 index f01c09f5a..227ea8598 --- a/create-projects/create-projects.sh +++ b/create-projects/create-projects.sh @@ -1,4 +1,4 @@ -#!/bin/#!/usr/bin/env bash +#!/usr/bin/env bash set -ex # check required parameters diff --git a/docs/antora.yml b/docs/antora.yml index 945c4a7b0..f5fd76168 100644 --- a/docs/antora.yml +++ b/docs/antora.yml @@ -7,3 +7,4 @@ nav: - modules/infrastructure-setup/nav.adoc - modules/jenkins/nav.adoc - modules/sonarqube/nav.adoc +- modules/keycloak/nav.adoc \ No newline at end of file diff --git a/docs/modules/keycloak/nav.adoc b/docs/modules/keycloak/nav.adoc new file mode 100644 index 000000000..c92e35745 --- /dev/null +++ b/docs/modules/keycloak/nav.adoc @@ -0,0 +1 @@ +* xref:index.adoc[Keycloak] diff --git a/docs/modules/keycloak/pages/index.adoc b/docs/modules/keycloak/pages/index.adoc new file mode 100644 index 000000000..da9ca8f9c --- /dev/null +++ b/docs/modules/keycloak/pages/index.adoc @@ -0,0 +1,59 @@ += Keycloak +:experimental: +:page-layout: documentation +:toc: + +Keycloak is an open source identity and access managment tool. + +OpenDevStack uses eigther _Atlassian Crowd_ or _OpenID Connect / OAuth 2.0_ protocol for authentication users of provisioning app. + +IMPORTANT: The usage of Keycloak is optional. Instead of keycloak, _Atlassian Crowd_ can still be used for +authentication. See documentation under _ODS-Components_ -> _Provisioning App_ -> _Configuration Guide_ regarding +the configuration of provisioning app. + + +Setup of keycloak for local-installation is all done without further manual setup actions +by starting the vagrant-box _idmanager_. + +== Realm _master_ + +=== Users + + +|=== +| User | Password | Roles + +|admin | admin | keycloak admin + +|=== + +== Realm _opendevstack_ +There is (beside the master realm) one single OpenDevStack-specific realm: _opendevstack_. + +=== Groups +There are two OpenDevStack-specific groups inside the _opendevstack_ realm: + +- opendevstack-administrators +- opendevstack-users + + +=== Users + +|=== +| User | Password | Groups + +|admin1 +|admin1 +a|- opendevstack-administrators +- opendevstack-users + +|user1 +| user1 +a|- opendevstack-users + +|=== + +=== Client _ods-provisioning-app_ +There is one OpenDevStack-specific client inside the _opendevstack_ realm: _ods-provisioning-app_. + +There is a protocoll mapper called _Group Mapper_ that maps the user's group membership to a token claim with name _roles_. \ No newline at end of file diff --git a/infrastructure-setup/ansible/requirements.yml b/infrastructure-setup/ansible/requirements.yml index 44c0ba5ee..7e56d6409 100755 --- a/infrastructure-setup/ansible/requirements.yml +++ b/infrastructure-setup/ansible/requirements.yml @@ -6,7 +6,7 @@ - src: geerlingguy.docker version: 2.0.4 - src: geerlingguy.git - version: 2.0.4 + version: 2.1.0 # modified -> modified neel.rundeck #- src: neel.rundeck diff --git a/infrastructure-setup/ansible/roles/geerlingguy.git/.ansible-lint b/infrastructure-setup/ansible/roles/geerlingguy.git/.ansible-lint new file mode 100644 index 000000000..1ae5e6ccf --- /dev/null +++ b/infrastructure-setup/ansible/roles/geerlingguy.git/.ansible-lint @@ -0,0 +1,2 @@ +skip_list: + - '204' diff --git a/infrastructure-setup/ansible/roles/geerlingguy.git/.gitignore b/infrastructure-setup/ansible/roles/geerlingguy.git/.gitignore index c9b2377e3..f56f5b578 100644 --- a/infrastructure-setup/ansible/roles/geerlingguy.git/.gitignore +++ b/infrastructure-setup/ansible/roles/geerlingguy.git/.gitignore @@ -1,2 +1,3 @@ *.retry -tests/test.sh +*/__pycache__ +*.pyc diff --git a/infrastructure-setup/ansible/roles/geerlingguy.git/.travis.yml b/infrastructure-setup/ansible/roles/geerlingguy.git/.travis.yml index bebd98101..742ed3b71 100644 --- a/infrastructure-setup/ansible/roles/geerlingguy.git/.travis.yml +++ b/infrastructure-setup/ansible/roles/geerlingguy.git/.travis.yml @@ -1,56 +1,34 @@ --- +language: python services: docker env: - # Test source install on latest supported OSes. - - distro: centos7 - playbook: test-source.yml - GIT_VERSION: 2.9.3 - - distro: ubuntu1604 - playbook: test-source.yml - GIT_VERSION: 2.9.3 - - # Test package install on all supported OSes. - - distro: centos7 - playbook: test.yml - GIT_VERSION: 1.8.3.1 - - distro: centos6 - playbook: test.yml - GIT_VERSION: 1.7.1 - - distro: fedora24 - playbook: test.yml - GIT_VERSION: 2.7.4 - - distro: ubuntu1604 - playbook: test.yml - GIT_VERSION: 2.7.4 - - distro: ubuntu1404 - playbook: test.yml - GIT_VERSION: 1.9.1 - - distro: ubuntu1204 - playbook: test.yml - GIT_VERSION: 1.7.9.5 - - distro: debian8 - playbook: test.yml - GIT_VERSION: 2.1.4 + global: + - ROLE_NAME: git + matrix: + - MOLECULE_DISTRO: centos7 + MOLECULE_PLAYBOOK: playbook-source.yml + - MOLECULE_DISTRO: ubuntu1804 + MOLECULE_PLAYBOOK: playbook-source.yml + - MOLECULE_DISTRO: centos7 + - MOLECULE_DISTRO: centos6 + - MOLECULE_DISTRO: ubuntu1804 + - MOLECULE_DISTRO: ubuntu1604 + - MOLECULE_DISTRO: debian9 + +install: + # Install test dependencies. + - pip install molecule docker + +before_script: + # Use actual Ansible Galaxy role name for the project directory. + - cd ../ + - mv ansible-role-$ROLE_NAME geerlingguy.$ROLE_NAME + - cd geerlingguy.$ROLE_NAME script: - # Configure test script so we can run extra tests after playbook is run. - - export container_id=$(date +%s) - - export cleanup=false - - # Download test shim. - - wget -O ${PWD}/tests/test.sh https://gist.githubusercontent.com/geerlingguy/73ef1e5ee45d8694570f334be385e181/raw/ - - chmod +x ${PWD}/tests/test.sh - # Run tests. - - ${PWD}/tests/test.sh - - # Ensure Git is installed and at the right version. - - 'docker exec --tty ${container_id} env TERM=xterm which git' - - 'docker exec --tty ${container_id} env TERM=xterm test -x /usr/bin/git' - - - 'docker exec --tty ${container_id} env TERM=xterm git --version' - - 'docker exec --tty ${container_id} env TERM=xterm /usr/bin/git --version | grep -qF "$GIT_VERSION"' + - molecule test notifications: webhooks: https://galaxy.ansible.com/api/v1/notifications/ diff --git a/infrastructure-setup/ansible/roles/geerlingguy.git/README.md b/infrastructure-setup/ansible/roles/geerlingguy.git/README.md index 02733dd32..d1eb74df3 100644 --- a/infrastructure-setup/ansible/roles/geerlingguy.git/README.md +++ b/infrastructure-setup/ansible/roles/geerlingguy.git/README.md @@ -28,7 +28,7 @@ The specific Git packages that will be installed. By default, `git-svn` is inclu git_install_from_source: false git_install_path: "/usr" - git_version: "2.1.0" + git_version: "2.16.2" Whether to install Git from source; if set to `true`, `git_version` is required and will be used to install a particular version of git (see all available versions here: https://www.kernel.org/pub/software/scm/git/), and `git_install_path` defines where git should be installed. diff --git a/infrastructure-setup/ansible/roles/geerlingguy.git/defaults/main.yml b/infrastructure-setup/ansible/roles/geerlingguy.git/defaults/main.yml index d08bf8c35..d01f33254 100644 --- a/infrastructure-setup/ansible/roles/geerlingguy.git/defaults/main.yml +++ b/infrastructure-setup/ansible/roles/geerlingguy.git/defaults/main.yml @@ -13,7 +13,7 @@ git_packages: # the 'git_version' variable instead of using a package. git_install_from_source: false git_install_path: "/usr" -git_version: "2.9.3" +git_version: "2.16.2" # If git is already installed at and older version, force a new source build. # Only applies if git_install_from_source is `true`. diff --git a/infrastructure-setup/ansible/roles/geerlingguy.git/meta/.galaxy_install_info b/infrastructure-setup/ansible/roles/geerlingguy.git/meta/.galaxy_install_info index 0373c7d7f..c40830888 100644 --- a/infrastructure-setup/ansible/roles/geerlingguy.git/meta/.galaxy_install_info +++ b/infrastructure-setup/ansible/roles/geerlingguy.git/meta/.galaxy_install_info @@ -1 +1 @@ -{install_date: 'Thu Apr 27 15:29:48 2017', version: 1.3.0} +{install_date: 'Thu Jan 9 08:54:43 2020', version: 2.1.0} diff --git a/infrastructure-setup/ansible/roles/geerlingguy.git/meta/main.yml b/infrastructure-setup/ansible/roles/geerlingguy.git/meta/main.yml index 0c0174032..e06b1c528 100644 --- a/infrastructure-setup/ansible/roles/geerlingguy.git/meta/main.yml +++ b/infrastructure-setup/ansible/roles/geerlingguy.git/meta/main.yml @@ -6,20 +6,24 @@ galaxy_info: description: Git version control software company: "Midwestern Mac, LLC" license: "license (BSD, MIT)" - min_ansible_version: 2.2 + min_ansible_version: 2.5 platforms: - - name: EL - versions: - - all - - name: Fedora - versions: - - all - - name: Debian - versions: - - all - - name: Ubuntu - versions: - - all + - name: EL + versions: + - all + - name: Fedora + versions: + - all + - name: Debian + versions: + - all + - name: Ubuntu + versions: + - all galaxy_tags: - development - system + - git + - vcs + - source + - code diff --git a/infrastructure-setup/ansible/roles/geerlingguy.git/molecule/default/molecule.yml b/infrastructure-setup/ansible/roles/geerlingguy.git/molecule/default/molecule.yml new file mode 100644 index 000000000..2ca6feaf9 --- /dev/null +++ b/infrastructure-setup/ansible/roles/geerlingguy.git/molecule/default/molecule.yml @@ -0,0 +1,29 @@ +--- +dependency: + name: galaxy +driver: + name: docker +lint: + name: yamllint + options: + config-file: molecule/default/yaml-lint.yml +platforms: + - name: instance + image: "geerlingguy/docker-${MOLECULE_DISTRO:-centos7}-ansible:latest" + command: ${MOLECULE_DOCKER_COMMAND:-""} + volumes: + - /sys/fs/cgroup:/sys/fs/cgroup:ro + privileged: true + pre_build_image: true +provisioner: + name: ansible + lint: + name: ansible-lint + playbooks: + converge: ${MOLECULE_PLAYBOOK:-playbook.yml} +scenario: + name: default +verifier: + name: testinfra + lint: + name: flake8 diff --git a/infrastructure-setup/ansible/roles/geerlingguy.git/molecule/default/playbook-source.yml b/infrastructure-setup/ansible/roles/geerlingguy.git/molecule/default/playbook-source.yml new file mode 100644 index 000000000..59743635c --- /dev/null +++ b/infrastructure-setup/ansible/roles/geerlingguy.git/molecule/default/playbook-source.yml @@ -0,0 +1,18 @@ +--- +- name: Converge + hosts: all + become: true + + vars: + git_install_from_source: true + git_install_from_source_force_update: true + git_version: "2.16.2" + + pre_tasks: + - name: Update apt cache. + apt: update_cache=true cache_valid_time=600 + when: ansible_os_family == 'Debian' + changed_when: false + + roles: + - role: geerlingguy.git diff --git a/infrastructure-setup/ansible/roles/geerlingguy.git/molecule/default/playbook.yml b/infrastructure-setup/ansible/roles/geerlingguy.git/molecule/default/playbook.yml new file mode 100644 index 000000000..2b5787499 --- /dev/null +++ b/infrastructure-setup/ansible/roles/geerlingguy.git/molecule/default/playbook.yml @@ -0,0 +1,17 @@ +--- +- name: Converge + hosts: all + become: true + + vars: + git_install_from_source: false + git_install_path: /usr/local + + pre_tasks: + - name: Update apt cache. + apt: update_cache=true cache_valid_time=600 + when: ansible_os_family == 'Debian' + changed_when: false + + roles: + - role: geerlingguy.git diff --git a/infrastructure-setup/ansible/roles/geerlingguy.git/molecule/default/yaml-lint.yml b/infrastructure-setup/ansible/roles/geerlingguy.git/molecule/default/yaml-lint.yml new file mode 100644 index 000000000..db22c42ea --- /dev/null +++ b/infrastructure-setup/ansible/roles/geerlingguy.git/molecule/default/yaml-lint.yml @@ -0,0 +1,6 @@ +--- +extends: default +rules: + line-length: + max: 160 + level: warning diff --git a/infrastructure-setup/ansible/roles/geerlingguy.git/tasks/install-from-source.yml b/infrastructure-setup/ansible/roles/geerlingguy.git/tasks/install-from-source.yml index 7720f81de..0d420f450 100644 --- a/infrastructure-setup/ansible/roles/geerlingguy.git/tasks/install-from-source.yml +++ b/infrastructure-setup/ansible/roles/geerlingguy.git/tasks/install-from-source.yml @@ -1,62 +1,57 @@ --- -- name: Ensure git's dependencies are installed (RedHat). - package: "name={{ item }} state=installed" - with_items: - - perl-ExtUtils-MakeMaker - - autoconf - - libcurl-devel - - expat-devel - - gettext-devel - - openssl-devel - - perl-devel - - zlib-devel - - gcc - - make -# subversion-perl is necessary for git-svn, does not exist on redhat el7 and not -# necessary for a server -# git svn will fail with "Can't locate SVN/Core.pm" -# enable, if available to add git svn support -# - subversion-perl - when: ansible_os_family == 'RedHat' +- name: Include OS-specific variables (RedHat). + include_vars: "{{ ansible_os_family }}.yml" + when: + - ansible_os_family == "RedHat" + - ansible_distribution != "Fedora" -- name: Ensure git's dependencies are installed (Debian). - apt: "name={{ item }} state=installed" - with_items: - - libcurl4-gnutls-dev - - libexpat1-dev - - gettext - - libssl-dev - - build-essential - - gcc - when: ansible_os_family == 'Debian' +- name: Include OS-specific variables (Fedora). + include_vars: "{{ ansible_distribution }}.yml" + when: ansible_distribution == "Fedora" + +- name: Include OS-specific variables (Debian). + include_vars: "{{ ansible_os_family }}.yml" + when: ansible_os_family == "Debian" + +- name: Define git_install_from_source_dependencies. + set_fact: + git_install_from_source_dependencies: "{{ __git_install_from_source_dependencies | list }}" + when: git_install_from_source_dependencies is not defined + +- name: Ensure git's dependencies are installed. + package: + name: "{{ git_install_from_source_dependencies }}" + state: present -- name: Get installed version +- name: Get installed version. command: > git --version warn=no changed_when: false failed_when: false - check_mode: no + check_mode: false register: git_installed_version -- name: Force git install if the version numbers do not match +- name: Force git install if the version numbers do not match. set_fact: git_reinstall_from_source: true - when: 'git_install_from_source_force_update and (git_installed_version|success and (git_installed_version.stdout | regex_replace("^.*?([0-9\.]+)$", "\\1") | version_compare(git_version, operator="!=")))' + when: + - git_install_from_source_force_update | bool + - (git_installed_version.rc == 0) and (git_installed_version.stdout | regex_replace("^.*?([0-9\.]+)$", "\\1") is version(git_version, operator="!=")) - name: Download git. get_url: url: "https://www.kernel.org/pub/software/scm/git/git-{{ git_version }}.tar.gz" dest: "{{ workspace }}/git-{{ git_version }}.tar.gz" - when: git_installed_version is failed or git_reinstall_from_source + when: (git_installed_version.rc != 0) or (git_reinstall_from_source | bool) - name: Expand git archive. unarchive: src: "{{ workspace }}/git-{{ git_version }}.tar.gz" dest: "{{ workspace }}" creates: "{{ workspace }}/git-{{ git_version }}/README" - copy: no - when: git_installed_version is failed or git_reinstall_from_source + copy: false + when: (git_installed_version.rc != 0) or (git_reinstall_from_source | bool) - name: Build git. command: > @@ -65,5 +60,5 @@ with_items: - all - install - when: git_installed_version is failed or git_reinstall_from_source - become: yes + when: (git_installed_version.rc != 0) or (git_reinstall_from_source | bool) + become: true diff --git a/infrastructure-setup/ansible/roles/geerlingguy.git/tasks/main.yml b/infrastructure-setup/ansible/roles/geerlingguy.git/tasks/main.yml index d829b1c75..d7cc5811b 100644 --- a/infrastructure-setup/ansible/roles/geerlingguy.git/tasks/main.yml +++ b/infrastructure-setup/ansible/roles/geerlingguy.git/tasks/main.yml @@ -1,23 +1,24 @@ --- - name: Ensure git is installed (RedHat). package: - name: "{{ item }}" - state: installed - enablerepo: "{{ git_enablerepo }}" - with_items: "{{ git_packages }}" - when: (git_install_from_source == false) and (ansible_os_family == 'RedHat') + name: "{{ git_packages }}" + state: present + enablerepo: "{{ git_enablerepo | default(omit, true) }}" + when: + - not git_install_from_source | bool + - ansible_os_family == 'RedHat' - name: Update apt cache (Debian). - apt: update_cache=yes cache_valid_time=86400 + apt: update_cache=true cache_valid_time=86400 when: ansible_os_family == 'Debian' - name: Ensure git is installed (Debian). apt: - name: "{{ item }}" - state: installed - with_items: "{{ git_packages }}" - when: (git_install_from_source == false) and (ansible_os_family == 'Debian') + name: "{{ git_packages }}" + state: present + when: + - not git_install_from_source | bool + - ansible_os_family == 'Debian' -# Install git from source when git_install_from_source is true. -- include: install-from-source.yml - when: git_install_from_source == true +- import_tasks: install-from-source.yml + when: git_install_from_source | bool diff --git a/infrastructure-setup/ansible/roles/geerlingguy.git/vars/Debian.yml b/infrastructure-setup/ansible/roles/geerlingguy.git/vars/Debian.yml new file mode 100644 index 000000000..230e6740b --- /dev/null +++ b/infrastructure-setup/ansible/roles/geerlingguy.git/vars/Debian.yml @@ -0,0 +1,9 @@ +--- +git_install_from_source_dependencies: + - libcurl4-gnutls-dev + - libexpat1-dev + - gettext + - libssl-dev + - zlib1g-dev + - build-essential + - gcc diff --git a/infrastructure-setup/ansible/roles/geerlingguy.git/vars/Fedora.yml b/infrastructure-setup/ansible/roles/geerlingguy.git/vars/Fedora.yml new file mode 100644 index 000000000..c0daee15d --- /dev/null +++ b/infrastructure-setup/ansible/roles/geerlingguy.git/vars/Fedora.yml @@ -0,0 +1,12 @@ +--- +git_install_from_source_dependencies: + - gettext-devel + - expat-devel + - curl-devel + - zlib-devel + - perl-devel + - openssl-devel + - subversion-perl + - make + - gcc + - tar diff --git a/infrastructure-setup/ansible/roles/geerlingguy.git/vars/RedHat.yml b/infrastructure-setup/ansible/roles/geerlingguy.git/vars/RedHat.yml new file mode 100644 index 000000000..d54dc5b2b --- /dev/null +++ b/infrastructure-setup/ansible/roles/geerlingguy.git/vars/RedHat.yml @@ -0,0 +1,11 @@ +--- +git_install_from_source_dependencies: + - gettext-devel + - expat-devel + - curl-devel + - zlib-devel + - perl-devel + - openssl-devel + - subversion-perl + - make + - gcc diff --git a/infrastructure-setup/scripts/json/createBlobStores.json b/infrastructure-setup/scripts/json/createBlobStores.json index 505076a38..ca535b488 100644 --- a/infrastructure-setup/scripts/json/createBlobStores.json +++ b/infrastructure-setup/scripts/json/createBlobStores.json @@ -1,8 +1,11 @@ -{ - "name":"createBlobStores", - "content":"import org.sonatype.nexus.blobstore.api.BlobStoreManager; - blobStore.createFileBlobStore('candidates','/nexus-data/blobs/candidates'); - blobStore.createFileBlobStore('releases','/nexus-data/blobs/releases'); - blobStore.createFileBlobStore('atlassian_public','/nexus-data/blobs/atlassian_public')", - "type":"groovy" -} \ No newline at end of file +{ + "name":"createBlobStores", + "content":"import org.sonatype.nexus.blobstore.api.BlobStoreManager; + blobStore.createFileBlobStore('candidates', '/nexus-data/blobs/candidates'); + blobStore.createFileBlobStore('releases', '/nexus-data/blobs/releases'); + blobStore.createFileBlobStore('atlassian_public', '/nexus-data/blobs/atlassian_public'); + blobStore.createFileBlobStore('npm-private', '/nexus-data/blobs/npm-private'); + blobStore.createFileBlobStore('pypi-private', '/nexus-data/blobs/pypi-private'); + blobStore.createFileBlobStore('leva-documentation', '/nexus-data/blobs/leva-documentation')", + "type":"groovy" +} diff --git a/infrastructure-setup/scripts/json/createRepos.json b/infrastructure-setup/scripts/json/createRepos.json index e775b92bc..7a7e3db62 100644 --- a/infrastructure-setup/scripts/json/createRepos.json +++ b/infrastructure-setup/scripts/json/createRepos.json @@ -1,27 +1,28 @@ -{ - "name":"createRepos", - "content":"import org.sonatype.nexus.blobstore.api.BlobStoreManager; - import org.sonatype.nexus.repository.storage.WritePolicy; - import org.sonatype.nexus.repository.maven.VersionPolicy; - import org.sonatype.nexus.repository.maven.LayoutPolicy; - repository.createMavenHosted('candidates', 'candidates', true, VersionPolicy.RELEASE, WritePolicy.ALLOW_ONCE, LayoutPolicy.STRICT); - repository.createMavenHosted('releases', 'releases', true, VersionPolicy.RELEASE, WritePolicy.ALLOW_ONCE, LayoutPolicy.STRICT); - repository.createNpmProxy('npmjs', 'https://registry.npmjs.org', 'default', true); - repository.createPyPiProxy('pypi-proxy', 'https://pypi.org/', 'default', false); - repository.createMavenProxy('atlassian_public', 'https://maven.atlassian.com/content/repositories/atlassian-public/','atlassian_public', true, VersionPolicy.RELEASE, LayoutPolicy.STRICT); - repository.createMavenProxy('jcenter', 'https://jcenter.bintray.com','default', true, VersionPolicy.RELEASE, LayoutPolicy.STRICT); - repository.createMavenProxy('sbt-plugins', 'http://dl.bintray.com/sbt/sbt-plugin-releases/','default', false, VersionPolicy.RELEASE, LayoutPolicy.PERMISSIVE); - repository.createMavenProxy('sbt-releases', 'https://repo.scala-sbt.org/scalasbt/sbt-plugin-releases','default', false, VersionPolicy.RELEASE, LayoutPolicy.PERMISSIVE); - repository.createMavenProxy('typesafe-ivy-releases', 'https://dl.bintray.com/typesafe/ivy-releases','default', false, VersionPolicy.RELEASE, LayoutPolicy.PERMISSIVE); - def ivy_members = ['sbt-plugins', 'sbt-releases', 'typesafe-ivy-releases']; - repository.createMavenGroup('ivy-releases',ivy_members,'default'); - def pypi_members = ['pypi-proxy']; - repository.createPyPiGroup('pypi-all',pypi_members,'default'); - repositoryManager = repository.repositoryManager; - repository = repositoryManager.get('maven-public'); - config = repository.configuration.copy(); - def maven_members = ['maven-releases','maven-snapshots','maven-central','jcenter']; - config.attributes['group']['memberNames'] = maven_members; - repositoryManager.update(config)", - "type":"groovy" +{ + "name":"createRepos", + "content":"import org.sonatype.nexus.blobstore.api.BlobStoreManager; + import org.sonatype.nexus.repository.storage.WritePolicy; + import org.sonatype.nexus.repository.maven.VersionPolicy; + import org.sonatype.nexus.repository.maven.LayoutPolicy; + repository.createMavenHosted('candidates', 'candidates', true, VersionPolicy.RELEASE, WritePolicy.ALLOW_ONCE, LayoutPolicy.STRICT); + repository.createMavenHosted('releases', 'releases', true, VersionPolicy.RELEASE, WritePolicy.ALLOW_ONCE, LayoutPolicy.STRICT); + repository.createMavenProxy('atlassian_public', 'https://maven.atlassian.com/content/repositories/atlassian-public/','atlassian_public', true, VersionPolicy.RELEASE, LayoutPolicy.STRICT); + repository.createMavenProxy('jcenter', 'https://jcenter.bintray.com', 'default', true, VersionPolicy.RELEASE, LayoutPolicy.STRICT); + repository.createMavenProxy('sbt-plugins', 'http://dl.bintray.com/sbt/sbt-plugin-releases/', 'default', false, VersionPolicy.RELEASE, LayoutPolicy.PERMISSIVE); + repository.createMavenProxy('sbt-releases', 'https://repo.scala-sbt.org/scalasbt/sbt-plugin-releases', 'default', false, VersionPolicy.RELEASE, LayoutPolicy.PERMISSIVE); + repository.createMavenProxy('typesafe-ivy-releases', 'https://dl.bintray.com/typesafe/ivy-releases', 'default', false, VersionPolicy.RELEASE, LayoutPolicy.PERMISSIVE); + repository.createMavenGroup('ivy-releases', ['sbt-plugins', 'sbt-releases', 'typesafe-ivy-releases'], 'default'); + repository.createNpmProxy('npm-registry', 'https://registry.npmjs.org', 'default', true); + repository.createNpmHosted('npm-private', 'npm-private', true, VersionPolicy.RELEASE, WritePolicy.ALLOW_ONCE, LayoutPolicy.STRICT); + repository.createNpmGroup('npm-all', ['npm-registry', 'npm-private'], 'default'); + repository.createPyPiProxy('pypi-registry', 'https://pypi.org/', 'default', false); + repository.createPyPiHosted('pypi-private', 'pypi-private', true, VersionPolicy.RELEASE, WritePolicy.ALLOW_ONCE, LayoutPolicy.STRICT); + repository.createPyPiGroup('pypi-all', ['pypi-registry', 'pypi-private'], 'default'); + repository.createRawHosted('leva-documentation', 'leva-documentation', false, WritePolicy.ALLOW); + repositoryManager = repository.repositoryManager; + repository = repositoryManager.get('maven-public'); + config = repository.configuration.copy(); + config.attributes['group']['memberNames'] = ['maven-releases', 'maven-snapshots', 'maven-central', 'jcenter']; + repositoryManager.update(config)", + "type":"groovy" } diff --git a/infrastructure-setup/scripts/json/deactivateAnonymous.json b/infrastructure-setup/scripts/json/deactivateAnonymous.json index 28b0d68f2..99ea1ca48 100644 --- a/infrastructure-setup/scripts/json/deactivateAnonymous.json +++ b/infrastructure-setup/scripts/json/deactivateAnonymous.json @@ -1,6 +1,6 @@ { "name":"deactivateAnonymous", "content":"security.setAnonymousAccess(false);", -"type":"groovy" + "type":"groovy" } diff --git a/infrastructure-setup/scripts/keycloak.sh b/infrastructure-setup/scripts/keycloak.sh index bd72eaf92..ac237b7d3 100644 --- a/infrastructure-setup/scripts/keycloak.sh +++ b/infrastructure-setup/scripts/keycloak.sh @@ -7,10 +7,16 @@ VERSION=4.8.3.Final DOWNLOAD_URL=http://downloads.jboss.org/keycloak/${VERSION}/keycloak-${VERSION}.tar.gz -# Install Java + echo "Installing Java 8 JDK ..." yum -y -q install java-1.8.0-openjdk-devel +echo "Installing epel-release repository ..." +yum -y install epel-release + +echo "Installing jq..." +yum -y install jq + # Install Keycloak if [ -f "/vagrant/downloads/keycloak-${VERSION}.tar.gz" ]; then @@ -55,6 +61,59 @@ echo "Starting Keycloak ..." /sbin/service keycloak restart +cd /opt/keycloak/bin +echo "Login to keycloak via admin user" +./kcadm.sh config credentials --server http://localhost:8080/auth --realm master --user admin --password admin + +REALM=opendevstack + +echo "create realm $REALM..." + +./kcadm.sh create realms -s realm=$REALM -s enabled=true + +echo "create group 'opendevstack-administrators' and 'opendevstack-user'" +./kcadm.sh create groups -r $REALM -s name=opendevstack-administrators +./kcadm.sh create groups -r $REALM -s name=opendevstack-users + +ODS_ADM_GROUP=$(./kcadm.sh get groups -r $REALM --fields 'id,name' | jq -r '.[] |select(.name =="opendevstack-administrators").id') +ODS_USER_GROUP=$(./kcadm.sh get groups -r $REALM --fields 'id,name' | jq -r '.[] |select(.name =="opendevstack-users").id') + +echo "create user 'user1'" +./kcadm.sh create users -r $REALM -s username=user1 -s enabled=true -s email=user1@idmanager.int +./kcadm.sh set-password -r $REALM --username user1 --new-password user1 +USER_ID=$(./kcadm.sh get users -r $REALM -q username=user1 | jq -r '.[0].id') +./kcadm.sh update users/$USER_ID/groups/$ODS_USER_GROUP -r $REALM -s realm=$REALM -s userId=$USER_ID -s groupId=$ODS_USER_GROUP -n + +echo "create user 'admin1'" +./kcadm.sh create users -r $REALM -s username=admin1 -s enabled=true -s email=admin1@idmanager.int +./kcadm.sh set-password -r $REALM --username admin1 --new-password admin1 +USER_ID=$(./kcadm.sh get users -r $REALM -q username=admin1 | jq -r '.[0].id') +./kcadm.sh update users/$USER_ID/groups/$ODS_ADM_GROUP -r $REALM -s realm=$REALM -s userId=$USER_ID -s groupId=$ODS_ADM_GROUP -n + +CLIENT_NAME=ods-provisioning-app + +echo "create client '$CLIENT_NAME'" +./kcadm.sh create clients -r $REALM -s clientId=$CLIENT_NAME -s 'redirectUris=["*"]' + +echo "create 'Group Membership' mapper in client $CLIENT_NAME" +CLIENT_ID=$(./kcadm.sh get clients -r $REALM -q clientId=ods-provisioning-app --fields id | jq -r '.[0].id') + +./kcadm.sh create -r $REALM clients/$CLIENT_ID/protocol-mappers/models -f - << 'EOF' +{ + "name": "Group Mapper", + "protocol": "openid-connect", + "protocolMapper": "oidc-group-membership-mapper", + "consentRequired": false, + "config": { + "full.path": "false", + "id.token.claim": "true", + "access.token.claim": "true", + "claim.name": "roles", + "userinfo.token.claim": "true" + } +} +EOF + echo "Opening port 8080 on iptables ..." iptables -I INPUT -p tcp -m state --state NEW -m tcp --dport 8080 -j ACCEPT -iptables-save > /etc/sysconfig/iptables +iptables-save > /etc/sysconfig/iptables \ No newline at end of file diff --git a/jenkins/master/import_certs.sh b/jenkins/master/import_certs.sh index 6967a6d85..248db9f66 100755 --- a/jenkins/master/import_certs.sh +++ b/jenkins/master/import_certs.sh @@ -4,6 +4,8 @@ oldIFS=$IFS IFS=';' KEYSTORE="$JAVA_HOME/lib/security/cacerts" +: "${APP_DNS_PORT:=443}" + if [ "${TARGET_HOSTS}x" == "x" ] ; then TARGET_HOSTS=${APP_DNS} fi @@ -11,7 +13,7 @@ fi echo "KEYSTORE=${KEYSTORE}" for dns in $TARGET_HOSTS; do cert_bundle_path="/etc/pki/ca-trust/source/anchors/${dns}.pem" - gnutls-cli --insecure --print-cert ${dns} /tmp/oc_app.crt + && gnutls-cli --insecure --print-cert ${APP_DNS} -p ${APP_DNS_PORT} /tmp/oc_app.crt # Add root ca into cert store so OC can pick it up. RUN cat /tmp/oc_app.crt >> /etc/ssl/certs/ca-bundle.trust.crt \ diff --git a/jenkins/slave-base/Dockerfile.rhel7 b/jenkins/slave-base/Dockerfile.rhel7 index f69fc7c94..6f7482915 100644 --- a/jenkins/slave-base/Dockerfile.rhel7 +++ b/jenkins/slave-base/Dockerfile.rhel7 @@ -13,6 +13,7 @@ ENV SONAR_SCANNER_VERSION=3.1.0.1141 \ OSTREE_VERSION=2018.5-1 ARG APP_DNS=192.168.99.100.nip.io +ARG APP_DNS_PORT=443 ARG SNYK_DISTRIBUTION_URL RUN yum -y install \ @@ -29,7 +30,7 @@ RUN yum -y install java-1.8.0-openjdk-devel.x86_64 \ # Fetch certificates and store them in tmp directory. RUN echo ${APP_DNS} \ - && gnutls-cli --insecure --print-cert ${APP_DNS} /tmp/oc_app.crt + && gnutls-cli --insecure --print-cert ${APP_DNS} -p ${APP_DNS_PORT} /tmp/oc_app.crt # Add root ca into cert store so OC can pick it up. RUN cat /tmp/oc_app.crt >> /etc/ssl/certs/ca-bundle.trust.crt \ diff --git a/ocp-scripts/clone-project.sh b/ocp-scripts/clone-project.sh index 6fe8a6d79..133848a5d 100755 --- a/ocp-scripts/clone-project.sh +++ b/ocp-scripts/clone-project.sh @@ -88,14 +88,28 @@ echo "Provided params: \ SOURCE_PROJECT="$PROJECT_ID-$SOURCE_ENV" TARGET_PROJECT="$PROJECT_ID-$TARGET_ENV" +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" echo "[INFO]: creating workplace: mkdir -p oc_migration_scripts/migration_config" mkdir -p oc_migration_scripts/migration_config cd oc_migration_scripts echo $(pwd) -export_url="https://$BITBUCKET_HOST/projects/opendevstack/repos/ods-core/raw/ocp-scripts/export-project.sh?at=refs%2Fheads%2Fproduction" -curl --fail -s --user $CREDENTIALS -G $export_url -d raw -o export.sh -import_url="https://$BITBUCKET_HOST/projects/opendevstack/repos/ods-core/raw/ocp-scripts/import-project.sh?at=refs%2Fheads%2Fproduction" -curl --fail -s --user $CREDENTIALS -G $import_url -d raw -o import.sh +# The curling of the export/import scripts is for backwards +# compatibility with older shared jenkins library +cleanup=() +if [ ! -f "$SCRIPT_DIR/export-project.sh" ]; then + export_url="https://$BITBUCKET_HOST/projects/opendevstack/repos/ods-core/raw/ocp-scripts/export-project.sh?at=refs%2Fheads%2Fproduction" + echo "Retrieving missing export-project.sh from $export_url" + file="$SCRIPT_DIR/export-project.sh" + curl --fail -s --user $CREDENTIALS -G $export_url -d raw -o "$file" + cleanup+=( "$file" ) +fi +if [ ! -f "$SCRIPT_DIR/import-project.sh" ]; then + import_url="https://$BITBUCKET_HOST/projects/opendevstack/repos/ods-core/raw/ocp-scripts/import-project.sh?at=refs%2Fheads%2Fproduction" + echo "Retrieving missing import-project.sh from $import_url" + file="$SCRIPT_DIR/import-project.sh" + curl --fail -s --user $CREDENTIALS -G $import_url -d raw -o "$file" + cleanup+=( "$file" ) +fi cd migration_config echo $(pwd) @@ -120,13 +134,16 @@ else fi echo "[INFO]: export resources from $SOURCE_ENV" -sh export.sh -p $PROJECT_ID -h $OPENSHIFT_HOST -e $SOURCE_ENV -g $git_url -gb $GIT_BRANCH -cpj $verbose +sh "$SCRIPT_DIR/export-project.sh" -p $PROJECT_ID -h $OPENSHIFT_HOST -e $SOURCE_ENV -g $git_url -gb $GIT_BRANCH -cpj $verbose echo "[INFO]: import resources into $TARGET_ENV" -sh import.sh -h $OPENSHIFT_HOST -p $PROJECT_ID -e $SOURCE_ENV -g $git_url -gb $GIT_BRANCH -n $TARGET_PROJECT $verbose --apply true +sh "$SCRIPT_DIR/import-project.sh" -h $OPENSHIFT_HOST -p $PROJECT_ID -e $SOURCE_ENV -g $git_url -gb $GIT_BRANCH -n $TARGET_PROJECT $verbose --apply true echo "[INFO]: cleanup workplace" cd .. rm -rf oc_migration_scripts +for file in "${cleanup[@]}"; do + rm "$file" +done if [[ ! -z "$SKIP_TAGS" ]]; then echo "[INFO] Skipping imagestream tagging" diff --git a/ods-setup/ocp-config/cd-user/Tailorfile b/ods-setup/ocp-config/cd-user/Tailorfile new file mode 100644 index 000000000..777683925 --- /dev/null +++ b/ods-setup/ocp-config/cd-user/Tailorfile @@ -0,0 +1,5 @@ +param-file ../../../ods-config/ods-core.env + +ignore-unknown-parameters true + +secret diff --git a/ods-setup/ocp-config/cd-user/secret.yml b/ods-setup/ocp-config/cd-user/secret.yml new file mode 100644 index 000000000..07d86612f --- /dev/null +++ b/ods-setup/ocp-config/cd-user/secret.yml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Template +labels: + template: secrets-template +objects: +- apiVersion: v1 + kind: Secret + metadata: + name: cd-user-token + type: kubernetes.io/basic-auth + data: + password: ${CD_USER_PWD_B64} + username: ${CD_USER_ID_B64} +parameters: +- name: CD_USER_PWD_B64 + description: cd_user password (base64 encoded) + required: true +- name: CD_USER_ID_B64 + description: cd_user name (base64 encoded) + required: true diff --git a/ods-setup/setup-jenkins-images.sh b/ods-setup/setup-jenkins-images.sh new file mode 100755 index 000000000..15108de37 --- /dev/null +++ b/ods-setup/setup-jenkins-images.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash + +#Builds all jenkins-master jenkins-slave-base and webhook-proxy + +set -ue + +function usage { + printf "usage: %s [options]\n" $0 + printf "\t--force\tIgnores warnings and error with tailor --force\n" + printf "\t-h|--help\tPrints the usage\n" + printf "\t-v|--verbose\tVerbose output\n" + printf "\t-t|--tailor\tChanges the executable of tailor. Default: tailor\n" + printf "\t-n|--namespace\tChanges the default OpenDevStack namespace where all resources will be created. Default: cd\n" + printf "\t-r|--ods-base-repository\tODS base repository. Overrides default in settings file\n" + printf "\t-b|--ods-ref\tODS reference in repository. Overrides default in settings file\n" + +} +TAILOR="tailor" +NAMESPACE="cd" +REPOSITORY="" +REF="" +while [[ "$#" -gt 0 ]]; do case $1 in + + -v|--verbose) set -x;; + + --force) FORCE="--force"; ;; + + -h|--help) usage; exit 0;; + + -t=*|--tailor=*) TAILOR="${1#*=}";; + -t|--tailor) TAILOR="$2"; shift;; + + -n=*|--namespace=*) NAMESPACE="${1#*=}";; + -n|--namespace) NAMESPACE="$2"; shift;; + + -r=*|--ods-base-repository=*) REPOSITORY="${1#*=}";; + -r|--ods-base-repository) REPOSITORY="$2"; shift;; + + -b=*|--ods-ref=*) REF="${1#*=}";; + -b|--ods-ref) REF="$2"; shift;; + + *) echo "Unknown parameter passed: $1"; usage; exit 1;; + esac; shift; done + +if ! oc whoami; then + echo "You should be logged into OpenShift to run the script" + exit 1 +fi + +if ! oc project ${NAMESPACE}; then + echo "The project '${NAMESPACE}' does not exist. Please setup the project using 'setup-ods-project.sh'" + exit 1 +fi + +echo "Applying Tailorfile to project '${NAMESPACE}'" + +if [ ! -z "${REF}" ]; then +REF="--param=ODS_GIT_REF=${REF}" +fi + +if [ ! -z "${REPOSITORY}" ]; then +REPOSITORY="--param=REPO_BASE=${REPOSITORY}" +fi + +${TAILOR} update ${FORCE} --context-dir=${BASH_SOURCE%/*}/../jenkins/ocp-config --non-interactive -n ${NAMESPACE} ${REF} ${REPOSITORY} + +echo "Start Jenkins Builds" +oc start-build -n ${NAMESPACE} jenkins-master --follow +oc start-build -n ${NAMESPACE} jenkins-slave-base --follow +oc start-build -n ${NAMESPACE} jenkins-webhook-proxy --follow diff --git a/ods-setup/setup-ods-project.sh b/ods-setup/setup-ods-project.sh new file mode 100755 index 000000000..5053fa7f2 --- /dev/null +++ b/ods-setup/setup-ods-project.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash + +# Create the namespace for holding all ODS resources + +set -ue + +function usage { + printf "usage: %s [options]\n" $0 + printf "\t--force\tIgnores warnings and error with tailor --force\n" + printf "\t-h|--help\tPrints the usage\n" + printf "\t-v|--verbose\tVerbose output\n" + printf "\t-t|--tailor\tChanges the executable of tailor. Default: tailor\n" + printf "\t-n|--namespace\tChanges the default OpenDevStack namespace where all resources will be created. Default: cd\n" + +} +TAILOR="tailor" +NAMESPACE="cd" + +while [[ "$#" -gt 0 ]]; do case $1 in + + -v|--verbose) set -x;; + + --force) FORCE="--force"; ;; + + -h|--help) usage; exit 0;; + + -t=*|--tailor=*) TAILOR="${1#*=}";; + -t|--tailor) TAILOR="$2"; shift;; + + -n=*|--namespace=*) NAMESPACE="${1#*=}";; + -n|--namespace) NAMESPACE="$2"; shift;; + + *) echo "Unknown parameter passed: $1"; usage; exit 1;; + esac; shift; done + +if ! oc whoami; then + echo "You should be logged to run the script" + exit 1 +fi + +if oc project ${NAMESPACE}; then + echo "The project '${NAMESPACE}' already exists" + exit 1 +fi + +oc new-project ${NAMESPACE} --description="Base project holding the templates and the Repositoy Manager" --display-name="OpenDevStack" + +# Allow system:authenticated group to pull images from CD namespace +oc adm policy add-cluster-role-to-group system:image-puller system:authenticated -n ${NAMESPACE} +oc adm policy add-role-to-group view system:authenticated -n ${NAMESPACE} + +oc create sa deployment -n ${NAMESPACE} +oc adm policy add-cluster-role-to-user cluster-admin system:serviceaccount:${NAMESPACE}:deployment + +# create secrets for global cd_user +${TAILOR} update ${FORCE} --context-dir=${BASH_SOURCE%/*}/ocp-config/cd-user --non-interactive diff --git a/sonarqube/Dockerfile b/sonarqube/Dockerfile index 2bf8c2a95..74b53c649 100644 --- a/sonarqube/Dockerfile +++ b/sonarqube/Dockerfile @@ -59,6 +59,7 @@ ADD https://github.com/deepy/sonar-crowd/releases/download/2.1.3/sonar-crowd-plu ADD https://github.com/vaulttec/sonar-auth-oidc/releases/download/v1.1.0/sonar-auth-oidc-plugin-1.1.0.jar /opt/configuration/sonarqube/plugins/ ADD https://github.com/dependency-check/dependency-check-sonar-plugin/releases/download/1.2.6/sonar-dependency-check-plugin-1.2.6.jar /opt/configuration/sonarqube/plugins/ ADD https://github.com/rht-labs/sonar-auth-openshift/releases/download/v1.1.1/sonar-auth-openshift-plugin.jar /opt/configuration/sonarqube/plugins/ +ADD https://binaries.sonarsource.com/Distribution/sonar-scm-git-plugin/sonar-scm-git-plugin-1.9.1.1834.jar /opt/configuration/sonarqube/plugins/ # Language plugins ADD https://binaries.sonarsource.com/Distribution/sonar-java-plugin/sonar-java-plugin-5.14.0.18788.jar /opt/configuration/sonarqube/plugins/ ADD https://binaries.sonarsource.com/Distribution/sonar-go-plugin/sonar-go-plugin-1.6.0.719.jar /opt/configuration/sonarqube/plugins/ diff --git a/tests/Makefile b/tests/Makefile new file mode 100644 index 000000000..1665dac79 --- /dev/null +++ b/tests/Makefile @@ -0,0 +1,14 @@ +.PHONY: test lint + +test: create-projects/create-projects_test.go create-projects/create-projects_test.go go.mod go.sum + go test -v github.com/opendevstack/ods-core/tests/create-projects -run "^(TestCreateJenkins)" + sleep 5 + go test -v github.com/opendevstack/ods-core/tests/create-projects -run "^(TestCreateProject)" + sleep 5 + go test -v github.com/opendevstack/ods-core/tests/create-projects -run "^(TestJenkinsFile)" + sleep 5 + go test -v github.com/opendevstack/ods-core/tests/ods-setup -run "^(TestCreateOdsProject)" + +lint: + echo "Checking code ..." + golangci-lint run diff --git a/tests/create-projects/create-cd-jenkins_test.go b/tests/create-projects/create-cd-jenkins_test.go new file mode 100644 index 000000000..55741a7dd --- /dev/null +++ b/tests/create-projects/create-cd-jenkins_test.go @@ -0,0 +1,106 @@ +package create_projects + +import ( + "fmt" + "github.com/opendevstack/ods-core/tests/utils" + "testing" +) + +func TestCreateJenkinsWithMissingEnvVars(t *testing.T) { + + values, err := utils.ReadConfiguration() + if err != nil { + t.Fatal( + "Could not read ods-core.env") + } + + user := values["CD_USER_ID_B64"] + secret := values["PIPELINE_TRIGGER_SECRET_B64"] + + var testCases = map[string]struct { + envVars []string + missingEnvVar string + }{ + "Create Jenkins without project id": { + envVars: []string{ + // utils.PROJECT_ENV_VAR, + "CD_USER_TYPE=general", + fmt.Sprintf("CD_USER_ID_B64=%s", user), + fmt.Sprintf("PIPELINE_TRIGGER_SECRET=%s", secret), + }, + missingEnvVar: "PROJECT_ID", + }, + "Create Jenkins without CD user type": { + envVars: []string{ + utils.PROJECT_ENV_VAR, + //"CD_USER_TYPE=general", + fmt.Sprintf("CD_USER_ID_B64=%s", user), + fmt.Sprintf("PIPELINE_TRIGGER_SECRET=%s", secret), + }, + missingEnvVar: "CD_USER_TYPE", + }, + "Create Jenkins without pipeline trigger secret": { + envVars: []string{ + utils.PROJECT_ENV_VAR, + "CD_USER_TYPE=general", + fmt.Sprintf("CD_USER_ID_B64=%s", user), + // fmt.Sprintf("PIPELINE_TRIGGER_SECRET=%s", secret), + }, + missingEnvVar: "PIPELINE_TRIGGER_SECRET", + }, + } + + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + stdout, stderr, err := utils.RunScriptFromBaseDir("create-projects/create-cd-jenkins.sh", []string{"--force", "--verbose"}, testCase.envVars) + if err == nil { + t.Fatalf( + "Execution of `create-cd-jenkins.sh` must fail if no %s is set: \nStdOut: %s\nStdErr: %s", + testCase.missingEnvVar, + stdout, + stderr, + ) + } + }) + } +} + +func TestCreateJenkinsSuccessfully(t *testing.T) { + err := utils.RemoveAllTestOCProjects() + if err != nil { + t.Fatal("Unable to remove test projects") + } + odsNamespace := "cd" + stdout, stderr, err := utils.RunScriptFromBaseDir("create-projects/create-projects.sh", []string{}, []string{utils.PROJECT_ENV_VAR}) + if err != nil { + t.Fatalf( + "Execution of `create-project.sh` failed: \nStdOut: %s\nStdErr: %s", + stdout, + stderr) + } + + values, err := utils.ReadConfiguration() + if err != nil { + t.Fatalf( + "Could not read ods-core.env") + } + + user := values["CD_USER_ID_B64"] + secret := values["PIPELINE_TRIGGER_SECRET_B64"] + + stdout, stderr, err = utils.RunScriptFromBaseDir("create-projects/create-cd-jenkins.sh", []string{"--force", "--verbose", "--ods-namespace", odsNamespace}, + []string{ + utils.PROJECT_ENV_VAR, + "CD_USER_TYPE=general", + fmt.Sprintf("CD_USER_ID_B64=%s", user), + fmt.Sprintf("PIPELINE_TRIGGER_SECRET=%s", secret), + }, + ) + if err != nil { + t.Fatalf( + "Execution of `create-cd-jenkins.sh` failed: \nStdOut: %s\nStdErr: %s", + stdout, + stderr) + } + CheckJenkinsWithTailor(values, utils.PROJECT_NAME_CD, utils.PROJECT_NAME, t) +} diff --git a/tests/create-projects/create-projects_test.go b/tests/create-projects/create-projects_test.go new file mode 100644 index 000000000..5bc7ed4ac --- /dev/null +++ b/tests/create-projects/create-projects_test.go @@ -0,0 +1,138 @@ +package create_projects + +import ( + "fmt" + "github.com/opendevstack/ods-core/tests/utils" + projectClientV1 "github.com/openshift/client-go/project/clientset/versioned/typed/project/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + rbacv1client "k8s.io/client-go/kubernetes/typed/rbac/v1" + "testing" +) + +func TestCreateProjectWithoutProjectId(t *testing.T) { + stdout, stderr, err := utils.RunScriptFromBaseDir("create-projects/create-projects.sh", []string{}, []string{}) + if err == nil { + t.Fatalf( + "Execution of `create-project.sh` must fail if no PROJECT_ID is set: \nStdOut: %s\nStdErr: %s", + stdout, + stderr) + } +} + +func TestCreateProject(t *testing.T) { + err := utils.RemoveAllTestOCProjects() + if err != nil { + t.Fatal("Unable to remove test projects") + } + + stdout, stderr, err := utils.RunScriptFromBaseDir("create-projects/create-projects.sh", []string{}, []string{utils.PROJECT_ENV_VAR}) + if err != nil { + t.Fatalf( + "Execution of `create-project.sh` failed: \nStdOut: %s\nStdErr: %s", + stdout, + stderr) + } + CheckProjectSetup(t) + +} + +func CheckProjectSetup(t *testing.T) { + config, err := utils.GetOCClient() + if err != nil { + t.Fatalf("Error creating OC config: %s", err) + } + client, err := projectClientV1.NewForConfig(config) + if err != nil { + t.Fatalf("Error creating Project client: %s", err) + } + projects, err := client.Projects().List(metav1.ListOptions{}) + if err != nil { + t.Fatalf("Cannot list projects: %s", err) + } + expectedProjects := []string{utils.PROJECT_NAME_CD, utils.PROJECT_NAME_TEST, utils.PROJECT_NAME_DEV} + for _, expectedProject := range expectedProjects { + if err = utils.FindProject(projects, expectedProject); err != nil { + t.Fatal(err) + } + } + rbacV1Client, err := rbacv1client.NewForConfig(config) + if err != nil { + t.Fatalf("Cannot initialize RBAC Client: %s", err) + } + roleBindings, _ := rbacV1Client.RoleBindings(utils.PROJECT_NAME_CD).List(metav1.ListOptions{}) + expectedRoleBindings := []utils.RoleBinding{ + { + SubjectName: "jenkins", + SubjectType: "ServiceAccount", + Namespace: utils.PROJECT_NAME_CD, + RoleName: "edit", + }, + { + SubjectName: "default", + SubjectType: "ServiceAccount", + Namespace: utils.PROJECT_NAME_CD, + RoleName: "edit", + }, + { + SubjectName: fmt.Sprintf("system:serviceaccounts:%s", utils.PROJECT_NAME_DEV), + SubjectType: "Group", + Namespace: "", + RoleName: "system:image-puller", + }, + { + SubjectName: fmt.Sprintf("system:serviceaccounts:%s", utils.PROJECT_NAME_TEST), + SubjectType: "Group", + Namespace: "", + RoleName: "system:image-puller", + }, + } + for _, expectedRoleBinding := range expectedRoleBindings { + if err = utils.FindRoleBinding(roleBindings, expectedRoleBinding.SubjectName, expectedRoleBinding.SubjectType, expectedRoleBinding.Namespace, expectedRoleBinding.RoleName); err != nil { + t.Error(err) + } + } + roleBindings, _ = rbacV1Client.RoleBindings(utils.PROJECT_NAME_DEV).List(metav1.ListOptions{}) + expectedRoleBindings = []utils.RoleBinding{ + { + SubjectName: "default", + SubjectType: "ServiceAccount", + Namespace: utils.PROJECT_NAME_DEV, + RoleName: "system:image-builder", + }, { + SubjectName: fmt.Sprintf("system:serviceaccounts:%s", utils.PROJECT_NAME_TEST), + SubjectType: "Group", + Namespace: "", + RoleName: "system:image-puller", + }, { + SubjectName: "jenkins", + SubjectType: "ServiceAccount", + Namespace: utils.PROJECT_NAME_CD, + RoleName: "admin", + }, + } + for _, expectedRoleBinding := range expectedRoleBindings { + if err = utils.FindRoleBinding(roleBindings, expectedRoleBinding.SubjectName, expectedRoleBinding.SubjectType, expectedRoleBinding.Namespace, expectedRoleBinding.RoleName); err != nil { + t.Error(err) + } + } + roleBindings, _ = rbacV1Client.RoleBindings(utils.PROJECT_NAME_TEST).List(metav1.ListOptions{}) + expectedRoleBindings = []utils.RoleBinding{ + { + SubjectName: "default", + SubjectType: "ServiceAccount", + Namespace: utils.PROJECT_NAME_TEST, + RoleName: "system:image-builder", + }, { + SubjectName: "jenkins", + SubjectType: "ServiceAccount", + Namespace: utils.PROJECT_NAME_CD, + RoleName: "admin", + }, + } + for _, expectedRoleBinding := range expectedRoleBindings { + if err = utils.FindRoleBinding(roleBindings, expectedRoleBinding.SubjectName, expectedRoleBinding.SubjectType, expectedRoleBinding.Namespace, expectedRoleBinding.RoleName); err != nil { + t.Error(err) + } + } + fmt.Printf("WARNING: Seeding special and default permission groups is not tested yet!") +} diff --git a/tests/create-projects/jenkinsfile_test.go b/tests/create-projects/jenkinsfile_test.go new file mode 100644 index 000000000..9111a363a --- /dev/null +++ b/tests/create-projects/jenkinsfile_test.go @@ -0,0 +1,169 @@ +package create_projects + +import ( + "bytes" + "crypto/tls" + "encoding/json" + "fmt" + "github.com/opendevstack/ods-core/tests/utils" + v1 "github.com/openshift/api/build/v1" + buildClientV1 "github.com/openshift/client-go/build/clientset/versioned/typed/build/v1" + "io/ioutil" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "net/http" + "path" + "runtime" + "testing" + "time" +) + +func TestJenkinsFile(t *testing.T) { + projectName := utils.PROJECT_NAME + projectNameCd := utils.PROJECT_NAME_CD + + err := utils.RemoveAllTestOCProjects() + if err != nil { + t.Fatal("Unable to remove test projects") + } + + values, err := utils.ReadConfiguration() + if err != nil { + t.Fatalf("Error reading ods-core.env: %s", err) + } + + request := utils.RequestBuild{ + Repository: "ods-core", + Branch: "cicdtests", + Project: "opendevstack", + Env: []utils.EnvPair{ + { + Name: "PROJECT_ID", + Value: projectName, + }, + { + Name: "CD_USER_TYPE", + Value: "general", + }, + { + Name: "CD_USER_ID_B64", + Value: values["CD_USER_ID_B64"], + }, + { + Name: "PIPELINE_TRIGGER_SECRET", + Value: values["PIPELINE_TRIGGER_SECRET_B64"], + }, + { + Name: "ODS_GIT_REF", + Value: "cicdtests", + }, + { + Name: "ODS_IMAGE_TAG", + Value: values["ODS_IMAGE_TAG"], + }, + }, + } + + body, err := json.Marshal(request) + if err != nil { + t.Fatalf("Could not marchal json: %s", err) + } + + http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + reponse, err := http.Post( + fmt.Sprintf("https://webhook-proxy-prov-cd.172.17.0.1.nip.io/build?trigger_secret=%s&jenkinsfile_path=create-projects/Jenkinsfile&component=ods-corejob-create-project-%s", + values["PIPELINE_TRIGGER_SECRET"], + projectName), + "application/json", + bytes.NewBuffer(body)) + + if err != nil { + t.Fatalf("Could not post request: %s", err) + } + + if reponse.StatusCode >= http.StatusAccepted { + bodyBytes, err := ioutil.ReadAll(reponse.Body) + if err != nil { + t.Fatal(err) + } + t.Fatalf("Could not post request: %s", string(bodyBytes)) + } + + if reponse.StatusCode >= http.StatusAccepted { + bodyBytes, err := ioutil.ReadAll(reponse.Body) + if err != nil { + t.Fatal(err) + } + t.Fatalf("Could not post request: %s", string(bodyBytes)) + } + + config, err := utils.GetOCClient() + if err != nil { + t.Fatalf("Error creating OC config: %s", err) + } + + buildClient, err := buildClientV1.NewForConfig(config) + if err != nil { + t.Fatalf("Error creating Build client: %s", err) + } + + time.Sleep(10 * time.Second) + build, err := buildClient.Builds("prov-cd").Get(fmt.Sprintf("ods-corejob-create-project-%s-cicdtests-1", projectName), metav1.GetOptions{}) + count := 0 + max := 240 + for (err != nil || build.Status.Phase == v1.BuildPhaseNew || build.Status.Phase == v1.BuildPhasePending || build.Status.Phase == v1.BuildPhaseRunning) && count < max { + build, err = buildClient.Builds("prov-cd").Get(fmt.Sprintf("ods-corejob-create-project-%s-cicdtests-1", projectName), metav1.GetOptions{}) + time.Sleep(2 * time.Second) + if err != nil { + fmt.Printf("Build is still not available") + } else { + fmt.Printf("Waiting for build. Current status: %s", build.Status.Phase) + } + count++ + } + + stdout, stderr, _ := utils.RunScriptFromBaseDir( + "tests/scripts/utils/print-jenkins-log.sh", + []string{ + fmt.Sprintf("ods-corejob-create-project-%s-cicdtests-1", projectName), + }, []string{}) + + if count >= max || build.Status.Phase != v1.BuildPhaseComplete { + + if count >= max { + t.Fatalf( + "Timeout during build: \nStdOut: %s\nStdErr: %s", + stdout, + stderr) + } else { + t.Fatalf( + "Error during build: \nStdOut: %s\nStdErr: %s", + stdout, + stderr) + } + + } + CheckProjectSetup(t) + CheckJenkinsWithTailor(values, projectNameCd, projectName, t) + +} + +func CheckJenkinsWithTailor(values map[string]string, projectNameCd string, projectName string, t *testing.T) { + _, filename, _, _ := runtime.Caller(0) + dir := path.Join(path.Dir(filename), "..", "..", "create-projects", "ocp-config", "cd-jenkins") + + user := values["CD_USER_ID_B64"] + secret := values["PIPELINE_TRIGGER_SECRET_B64"] + + stdout, stderr, err := utils.RunCommandWithWorkDir("tailor", []string{"status", "--force", "--reveal-secrets", "-n", projectNameCd, + fmt.Sprintf("--param=PROJECT=%s", projectName), + fmt.Sprintf("--param=CD_USER_ID_B64=%s", user), + "--selector", "template=cd-jenkins-template", + fmt.Sprintf("--param=%s", fmt.Sprintf("PROXY_TRIGGER_SECRET_B64=%s", secret))}, dir, []string{}) + if err != nil { + + t.Fatalf( + "Execution of tailor failed: \nStdOut: %s\nStdErr: %s", + stdout, + stderr) + } +} diff --git a/tests/go.mod b/tests/go.mod new file mode 100644 index 000000000..c7500112e --- /dev/null +++ b/tests/go.mod @@ -0,0 +1,25 @@ +module github.com/opendevstack/ods-core/tests + +go 1.13 + +require ( + github.com/gogo/protobuf v1.3.1 // indirect + github.com/google/go-cmp v0.3.1 // indirect + github.com/imdario/mergo v0.3.8 // indirect + github.com/json-iterator/go v1.1.8 // indirect + github.com/modern-go/reflect2 v1.0.1 // indirect + github.com/openshift/api v3.9.0+incompatible + github.com/openshift/client-go v3.9.0+incompatible + github.com/pkg/errors v0.8.1 // indirect + github.com/spf13/pflag v1.0.5 // indirect + golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 // indirect + golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6 // indirect + golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v2 v2.2.7 // indirect + k8s.io/api v0.0.0-20190222213804-5cb15d344471 + k8s.io/apimachinery v0.0.0-20190221213512-86fb29eff628 + k8s.io/client-go v0.0.0-20190228174230-b40b2a5939e4 + k8s.io/klog v1.0.0 // indirect + sigs.k8s.io/yaml v1.1.0 // indirect +) diff --git a/tests/go.sum b/tests/go.sum new file mode 100644 index 000000000..164879fcd --- /dev/null +++ b/tests/go.sum @@ -0,0 +1,71 @@ +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/imdario/mergo v0.3.8 h1:CGgOkSJeqMRmt0D9XLWExdT4m4F1vd3FV3VPt+0VxkQ= +github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/json-iterator/go v1.1.8 h1:QiWkFLKq0T7mpzwOTu6BzNDbfTE8OLrYhVKYMLF46Ok= +github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/opendevstack/ods-core v1.2.0 h1:sYaqnULBQ9frsXa+E2KL79g72uVIvzmFtwTr4zTNAPM= +github.com/openshift/api v3.9.0+incompatible h1:fJ/KsefYuZAjmrr3+5U9yZIZbTOpVkDDLDLFresAeYs= +github.com/openshift/api v3.9.0+incompatible/go.mod h1:dh9o4Fs58gpFXGSYfnVxGR9PnV53I8TW84pQaJDdGiY= +github.com/openshift/client-go v3.9.0+incompatible h1:13k3Ok0B7TA2hA3bQW2aFqn6y04JaJWdk7ITTyg+Ek0= +github.com/openshift/client-go v3.9.0+incompatible/go.mod h1:6rzn+JTr7+WYS2E1TExP4gByoABxMznR6y2SnUIkmxk= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 h1:ULYEB3JvPRE/IfO+9uO7vKV/xzVTO7XPAwm8xbf4w2g= +golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e h1:bRhVy7zSSasaqNksaRZiA5EEI+Ei4I1nO5Jh72wfHlg= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 h1:0GoQqolDA55aaLxZyTzK/Y2ePZzZTUrRacwib7cNsYQ= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6 h1:pE8b58s1HRDMi8RDc79m0HISf9D4TzseP40cEA6IGfs= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo= +gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= +gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +k8s.io/api v0.0.0-20190222213804-5cb15d344471 h1:MzQGt8qWQCR+39kbYRd0uQqsvSidpYqJLFeWiJ9l4OE= +k8s.io/api v0.0.0-20190222213804-5cb15d344471/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= +k8s.io/apimachinery v0.0.0-20190221213512-86fb29eff628 h1:UYfHH+KEF88OTg+GojQUwFTNxbxwmoktLwutUzR0GPg= +k8s.io/apimachinery v0.0.0-20190221213512-86fb29eff628/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= +k8s.io/client-go v0.0.0-20190228174230-b40b2a5939e4 h1:aE8wOCKuoRs2aU0OP/Rz8SXiAB0FTTku3VtGhhrkSmc= +k8s.io/client-go v0.0.0-20190228174230-b40b2a5939e4/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s= +k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= +k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= +sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= diff --git a/tests/ods-setup/setup-ods-project_test.go b/tests/ods-setup/setup-ods-project_test.go new file mode 100644 index 000000000..41b0d9a72 --- /dev/null +++ b/tests/ods-setup/setup-ods-project_test.go @@ -0,0 +1,87 @@ +package ods_setup + +import ( + "github.com/opendevstack/ods-core/tests/utils" + imageClientV1 "github.com/openshift/client-go/image/clientset/versioned/typed/image/v1" + projectClientV1 "github.com/openshift/client-go/project/clientset/versioned/typed/project/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "testing" +) + +func TestCreateOdsProject(t *testing.T) { + namespace := "ods" + _ = utils.RemoveProject(namespace) + stdout, stderr, err := utils.RunScriptFromBaseDir("ods-setup/setup-ods-project.sh", []string{ + "--force", + "--verbose", + "--namespace", + namespace, + }, []string{}) + if err != nil { + t.Fatalf( + "Execution of `setup-ods-project.sh` failed: \nStdOut: %s\nStdErr: %s", + stdout, + stderr) + } + + config, err := utils.GetOCClient() + if err != nil { + t.Fatalf("Error creating OC config: %s", err) + } + client, err := projectClientV1.NewForConfig(config) + if err != nil { + t.Fatalf("Error creating Project client: %s", err) + } + projects, err := client.Projects().List(metav1.ListOptions{}) + if err != nil { + t.Fatalf("Cannot list projects: %s", err) + } + + if err = utils.FindProject(projects, namespace); err != nil { + t.Fatal(err) + } + + gitReference := "cicdtests" + + stdout, stderr, err = utils.RunScriptFromBaseDir("ods-setup/setup-jenkins-images.sh", []string{ + "--verbose", + "--force", + "--ods-ref", gitReference, + "--namespace", namespace, + }, []string{}) + if err != nil { + t.Fatalf( + "Execution of `setup-jenkins-images.sh` failed: \nStdOut: %s\nStdErr: %s", + stdout, + stderr) + } + + imageClient, err := imageClientV1.NewForConfig(config) + if err != nil { + t.Fatalf("Error creating Image client: %s", err) + } + + expectedImageTags := []utils.ImageTag{ + { + ImageName: "jenkins-master", + ImageTag: gitReference, + }, + { + ImageName: "jenkins-slave-base", + ImageTag: gitReference, + }, + { + ImageName: "jenkins-webhook-proxy", + ImageTag: gitReference, + }, + } + + images, err := imageClient.ImageStreams(namespace).List(metav1.ListOptions{}) + + for _, imageTag := range expectedImageTags { + if err = utils.FindImageTag(images, imageTag.ImageName, imageTag.ImageTag); err != nil { + t.Fatalf("%s\nScript:\nStdOut: %s\nStdErr: %s", err, stdout, stderr) + } + } + +} diff --git a/tests/scripts/apply-docker-settings.sh b/tests/scripts/apply-docker-settings.sh new file mode 100755 index 000000000..f2d95134e --- /dev/null +++ b/tests/scripts/apply-docker-settings.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +set -uex + +DAEMON_JSON=$(cat ${BASH_SOURCE%/*}/json/daemon.json) +DAEMON_FILE_PATH="/etc/docker/daemon.json" + +if [ -f "${DAEMON_FILE_PATH}" ]; then + cat "${DAEMON_FILE_PATH}" +else + echo '{}' +fi | jq --argjson daemonJson "${DAEMON_JSON}" '. + $daemonJson' | sudo tee /etc/docker/daemon.json + +for i in $(seq 1 5); do + [ $i -gt 1 ] && sleep 2; + sudo service docker restart && EXIT_CODE=0 && break || EXIT_CODE=$?; + if [ ${EXIT_CODE} != 0 ]; then + sudo systemctl status docker.service + sudo journalctl -xe + fi +done; +exit $EXIT_CODE diff --git a/tests/scripts/create-env-from-local-cluster.sh b/tests/scripts/create-env-from-local-cluster.sh new file mode 100755 index 000000000..54a8e7087 --- /dev/null +++ b/tests/scripts/create-env-from-local-cluster.sh @@ -0,0 +1,202 @@ +#!/usr/bin/env bash +set -uex + +BASE_OC_DIR="${HOME}/openshift.local.clusterup" + +function usage() { + + exit +} + +URL=$(oc config view --minify -o jsonpath='{.clusters[*].cluster.server}') +if [ ${URL} != "https://127.0.0.1:8443" ]; then + echo "You are not in a local cluster. Stopping now!!!" +fi + +while [[ "$#" -gt 0 ]]; do case $1 in + -b=*|--base-oc-dir=*) BASE_OC_DIR="${1#*=}";; + -b|--base-oc-dir) BASE_OC_DIR="$2"; shift;; + + -o=*|--output=*) OUTPUT="${1#*=}";; + -o|--output) OUTPUT="$2"; shift;; + + -h|--help) usage; exit 0;; + + *) echo "Unknown parameter passed: $1"; usage; exit 1;; +esac; shift; done + + + +if [[ -z "${BASE_OC_DIR}" && ! -d "${BASE_OC_DIR}" ]]; then + echo "--base-oc-dir must be provided and pointing to the oc cluster base diretory." + usage +fi + +SUBDOMAIN=$(grep -A 1 routingConfig "${BASE_OC_DIR}/openshift-apiserver/master-config.yaml" | tail -n1 | awk '{print $2}') +REGISTRY_IP=172.30.1.1 +echo "ODS_IMAGE_TAG=cicdtests" > ${OUTPUT} +echo "" >> ${OUTPUT} +echo "#####-#####-#####-#####-#####" >> ${OUTPUT} +echo "##### NEXUS #####" >> ${OUTPUT} +echo "#####-#####-#####-#####-#####" >> ${OUTPUT} +echo "" >> ${OUTPUT} +NEXUS_HOST="nexus-cd.${SUBDOMAIN}" +NEXUS_URL="https://${NEXUS_HOST}" +NEXUS_USERNAME=developer +NEXUS_PASSWORD=$(dd if=/dev/urandom bs=1 count=32 2>/dev/null | base64 | rev | cut -b 2- | rev | tr -cd '[:alnum:]') +NEXUS_PASSWORD_B64=$(echo -n "${NEXUS_PASSWORD}" | base64) +NEXUS_AUTH="${NEXUS_USERNAME}:${NEXUS_PASSWORD}" + +echo "NEXUS_URL=${NEXUS_URL}" >> ${OUTPUT} +echo "NEXUS_HOST=${NEXUS_HOST}" >> ${OUTPUT} +echo "NEXUS_USERNAME=${NEXUS_USERNAME}" >> ${OUTPUT} +echo "NEXUS_PASSWORD=${NEXUS_PASSWORD}" >> ${OUTPUT} +echo "NEXUS_PASSWORD_B64=${NEXUS_PASSWORD_B64}" >> ${OUTPUT} +echo "NEXUS_AUTH=${NEXUS_AUTH}" >> ${OUTPUT} + +echo "" >> ${OUTPUT} +echo "#####-#####-#####-#####-#####" >> ${OUTPUT} +echo "##### SonarQube #####" >> ${OUTPUT} +echo "#####-#####-#####-#####-#####" >> ${OUTPUT} +echo "" >> ${OUTPUT} + +SONARQUBE_HOST="sonarqube-cd.${SUBDOMAIN}" + +# SonarQube URL exposed by the SonarQube route +SONARQUBE_URL="https://${SONARQUBE_HOST}" + +# Username and password for SonarQube +SONAR_ADMIN_USERNAME="admin" +SONAR_PASSWORD=$(dd if=/dev/urandom bs=1 count=32 2>/dev/null | base64 | rev | cut -b 2- | rev | tr -cd '[:alnum:]') +SONAR_PASSWORD_B64=$(echo -n "${SONAR_PASSWORD}" | base64) +SONAR_AUTH_TOKEN="${SONAR_ADMIN_USERNAME}:${SONAR_PASSWORD}" +SONAR_AUTH_TOKEN_B64=$(echo -n "${SONAR_AUTH_TOKEN}" | base64) + +# Application in Crowd used for authentication +SONAR_CROWD_APPLICATION=sonarqube +SONAR_CROWD_PASSWORD=$(dd if=/dev/urandom bs=1 count=32 2>/dev/null | base64 | rev | cut -b 2- | rev | tr -cd '[:alnum:]') +SONAR_CROWD_PASSWORD_B64=$(echo -n "${SONAR_CROWD_PASSWORD}" | base64) + +# Postgres DB for SonarQube +SONAR_DATABASE_JDBC_URL=jdbc:postgresql://sonarqube-postgresql.svc:5432/sonarqube +SONAR_DATABASE_NAME=sonarqube +SONAR_DATABASE_USER=sonarqube +SONAR_DATABASE_PASSWORD=$(dd if=/dev/urandom bs=1 count=32 2>/dev/null | base64 | rev | cut -b 2- | rev | tr -cd '[:alnum:]') +SONAR_DATABASE_PASSWORD_B64=$(echo -n "${SONAR_DATABASE_PASSWORD}" | base64) + +#SonarQube Download +SONAR_VERSION=7.9 +SONAR_DISTRIBUTION_URL="https://binaries.sonarsource.com/Distribution/sonarqube/sonarqube-${SONAR_VERSION}.zip" +SONAR_AUTH_CROWD=false + + +echo "SONARQUBE_HOST=${SONARQUBE_HOST}" >> ${OUTPUT} +echo "SONARQUBE_URL=${SONARQUBE_URL}" >> ${OUTPUT} +echo "SONAR_ADMIN_USERNAME=${SONAR_ADMIN_USERNAME}" >> ${OUTPUT} +echo "SONAR_PASSWORD=${SONAR_PASSWORD}" >> ${OUTPUT} +echo "SONAR_PASSWORD_B64=${SONAR_PASSWORD_B64}" >> ${OUTPUT} +echo "SONAR_AUTH_TOKEN=${SONAR_AUTH_TOKEN}" >> ${OUTPUT} +echo "SONAR_AUTH_TOKEN_B64=${SONAR_AUTH_TOKEN_B64}" >> ${OUTPUT} +echo "SONAR_CROWD_APPLICATION=${SONAR_CROWD_APPLICATION}" >> ${OUTPUT} +echo "SONAR_CROWD_PASSWORD=${SONAR_CROWD_PASSWORD}" >> ${OUTPUT} +echo "SONAR_CROWD_PASSWORD_B64=${SONAR_CROWD_PASSWORD_B64}" >> ${OUTPUT} +echo "SONAR_DATABASE_JDBC_URL=${SONAR_DATABASE_JDBC_URL}" >> ${OUTPUT} +echo "SONAR_DATABASE_NAME=${SONAR_DATABASE_NAME}" >> ${OUTPUT} +echo "SONAR_DATABASE_USER=${SONAR_DATABASE_USER}" >> ${OUTPUT} +echo "SONAR_DATABASE_PASSWORD=${SONAR_DATABASE_PASSWORD}" >> ${OUTPUT} +echo "SONAR_DATABASE_PASSWORD_B64=${SONAR_DATABASE_PASSWORD_B64}" >> ${OUTPUT} +echo "SONAR_VERSION=${SONAR_VERSION}" >> ${OUTPUT} +echo "SONAR_DISTRIBUTION_URL=${SONAR_DISTRIBUTION_URL}" >> ${OUTPUT} +echo "SONAR_AUTH_CROWD=${SONAR_AUTH_CROWD}" >> ${OUTPUT} + +echo "" >> ${OUTPUT} +echo "#####-#####-#####-#####-#####" >> ${OUTPUT} +echo "##### Jira #####" >> ${OUTPUT} +echo "#####-#####-#####-#####-#####" >> ${OUTPUT} +echo "" >> ${OUTPUT} + +echo "JIRA_URL=https://jira-cd.${SUBDOMAIN}" >> ${OUTPUT} + +echo "" >> ${OUTPUT} +echo "#####-#####-#####-#####-#####" >> ${OUTPUT} +echo "##### Crowd #####" >> ${OUTPUT} +echo "#####-#####-#####-#####-#####" >> ${OUTPUT} +echo "" >> ${OUTPUT} + +echo "CROWD_URL=https://crowd-cd.${SUBDOMAIN}" >> ${OUTPUT} + +echo "" >> ${OUTPUT} +echo "#####-#####-#####-#####-#####" >> ${OUTPUT} +echo "##### BitBucket #####" >> ${OUTPUT} +echo "#####-#####-#####-#####-#####" >> ${OUTPUT} +echo "" >> ${OUTPUT} + +BITBUCKET_HOST=172.17.0.1:8080 +REPO_BASE=http://${BITBUCKET_HOST}/scm + +CD_USER_ID=cd_user +CD_USER_ID_B64=$(echo -n "${CD_USER_ID}" | base64) +CD_USER_PWD=$(dd if=/dev/urandom bs=1 count=32 2>/dev/null | base64 | rev | cut -b 2- | rev | tr -cd '[:alnum:]') +CD_USER_PWD_B64=$(echo -n "${CD_USER_PWD}" | base64) + +echo "BITBUCKET_HOST=${BITBUCKET_HOST}" >> ${OUTPUT} +echo "REPO_BASE=${REPO_BASE}" >> ${OUTPUT} +echo "CD_USER_ID=${CD_USER_ID}" >> ${OUTPUT} +echo "CD_USER_ID_B64=${CD_USER_ID_B64}" >> ${OUTPUT} +echo "CD_USER_PWD=${CD_USER_PWD}" >> ${OUTPUT} +echo "CD_USER_PWD_B64=${CD_USER_PWD_B64}" >> ${OUTPUT} + +echo "" >> ${OUTPUT} +echo "#####-#####-#####-#####-#####" >> ${OUTPUT} +echo "##### Jenkins #####" >> ${OUTPUT} +echo "#####-#####-#####-#####-#####" >> ${OUTPUT} +echo "" >> ${OUTPUT} + +echo "JENKINS_AGENT_BASE_IMAGE=Dockerfile.centos7" >> ${OUTPUT} +echo "JENKINS_AGENT_BASE_FROM_IMAGE=openshift/jenkins-slave-base-centos7" >> ${OUTPUT} +echo "JENKINS_AGENT_BASE_SNYK_DISTRIBUTION_URL=https://github.com/snyk/snyk/releases/download/v1.180.1/snyk-linux" >> ${OUTPUT} + + +echo "" >> ${OUTPUT} +echo "#####-#####-#####-#####-#####" >> ${OUTPUT} +echo "##### OpenShift #####" >> ${OUTPUT} +echo "#####-#####-#####-#####-#####" >> ${OUTPUT} +echo "" >> ${OUTPUT} + +DOCKER_REGISTRY=${REGISTRY_IP}:5000 +APP_DNS="${SUBDOMAIN}" +APP_DNS_PORT=8443 +TARGET_HOSTS= +OPENSHIFT_API_HOST=https://127.0.0.1:8443 +OPENSHIFT_CONSOLE_HOST=https://127.0.0.1:8443 +OPENSHIFT_APPS_BASEDOMAIN=".${SUBDOMAIN}" +PIPELINE_TRIGGER_SECRET=$(dd if=/dev/urandom bs=1 count=32 2>/dev/null | base64 | rev | cut -b 2- | rev | tr -cd '[:alnum:]') +PIPELINE_TRIGGER_SECRET_B64=$(echo -n "${PIPELINE_TRIGGER_SECRET}" | base64) + +echo "DOCKER_REGISTRY=${DOCKER_REGISTRY}" >> ${OUTPUT} +echo "APP_DNS=${APP_DNS}" >> ${OUTPUT} +echo "APP_DNS_PORT=${APP_DNS_PORT}" >> ${OUTPUT} +echo "TARGET_HOSTS=${TARGET_HOSTS}" >> ${OUTPUT} +echo "OPENSHIFT_API_HOST=${OPENSHIFT_API_HOST}" >> ${OUTPUT} +echo "OPENSHIFT_CONSOLE_HOST=${OPENSHIFT_CONSOLE_HOST}" >> ${OUTPUT} +echo "OPENSHIFT_APPS_BASEDOMAIN=${OPENSHIFT_APPS_BASEDOMAIN}" >> ${OUTPUT} +echo "PIPELINE_TRIGGER_SECRET=${PIPELINE_TRIGGER_SECRET}" >> ${OUTPUT} +echo "PIPELINE_TRIGGER_SECRET_B64=${PIPELINE_TRIGGER_SECRET_B64}" >> ${OUTPUT} + +echo "" >> ${OUTPUT} +echo "#####-#####-#####-#####-#####" >> ${OUTPUT} +echo "##### Identity Provider #####" >> ${OUTPUT} +echo "#####-#####-#####-#####-#####" >> ${OUTPUT} +echo "" >> ${OUTPUT} + +echo "IDP_DNS=${SUBDOMAIN}" >> ${OUTPUT} + +echo "" >> ${OUTPUT} +echo "#####-#####-#####-#####-#####" >> ${OUTPUT} +echo "##### Storage #####" >> ${OUTPUT} +echo "#####-#####-#####-#####-#####" >> ${OUTPUT} +echo "" >> ${OUTPUT} + +echo "STORAGE_PROVISIONER=" >> ${OUTPUT} +echo "STORAGE_CLASS_DATA=" >> ${OUTPUT} +echo "STORAGE_CLASS_BACKUP=" >> ${OUTPUT} diff --git a/tests/scripts/deploy-mocks.sh b/tests/scripts/deploy-mocks.sh new file mode 100755 index 000000000..c6a500afb --- /dev/null +++ b/tests/scripts/deploy-mocks.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# (Re)deploys the mocked services for ODS using docker + +set -eu + +function usage { + printf "usage: %s [options]\n", $0 + printf "\t-h|--help\tPrints the usage\n" + printf "\t-v|--verbose\tVerbose output\n" + printf "\t-w|--wait\tWaits for the service(s). Depends on netcat \n" + +} +WAIT=false +while [[ "$#" -gt 0 ]]; do case $1 in + + -v|--verbose) set -x;; + + -h|--help) usage; exit 0;; + + -w|--wait) WAIT=true;; + + *) echo "Unknown parameter passed: $1"; usage; exit 1;; + esac; shift; done + +URL=$(oc config view --minify -o jsonpath='{.clusters[*].cluster.server}') +if [ ${URL} != "https://127.0.0.1:8443" ]; then + echo "You are not in a local cluster. Stopping now!!!" +fi + +if docker ps -a --format "{{.Names}}" | grep mockbucket; then + docker rm mockbucket --force +fi + +source ${BASH_SOURCE%/*}/../../ods-config/ods-core.env +docker run -d -p "8080:8080" \ + --env="BASIC_USERNAME=${CD_USER_ID}" \ + --env="BASIC_PASSWORD=${CD_USER_PWD}" \ + --env="REPOS=opendevstack/ods-core.git;opendevstack/ods-configuration.git;opendevstack/ods-quickstarters.git;opendevstack/ods-jenkins-shared-library.git" \ + --name mockbucket \ + hugowschneider/mockbucket:latest + +if [ ${WAIT} == "true" ]; then + echo "Waiting for mockbucket to launch on 8080..." + + while ! nc -z 172.17.0.1 8080; do + echo "Port 8080 is not responding..." + sleep 1 + done + + while [ "$(curl http://172.17.0.1:8080 -u $CD_USER_ID:$CD_USER_PWD -o /dev/null -w '%{http_code}' -s)" != "200" ]; do + echo "Service is still not running..." + sleep 1 + done + + echo "Mockbucket is running" +fi diff --git a/tests/scripts/json/create-cluster-role.json b/tests/scripts/json/create-cluster-role.json new file mode 100644 index 000000000..0985ea0ec --- /dev/null +++ b/tests/scripts/json/create-cluster-role.json @@ -0,0 +1,27 @@ +{ + "apiVersion": "v1", + "kind": "ClusterRole", + "metadata": { + "name": "request_role" + }, + "rules": [ + { + "verbs": [ + "create" + ], + "resources": [ + "projectrequests" + ] + }, + { + "verbs": [ + "get", + "list", + "watch" + ], + "resources": [ + "*" + ] + } + ] +} diff --git a/tests/scripts/json/daemon.json b/tests/scripts/json/daemon.json new file mode 100644 index 000000000..23bc0e192 --- /dev/null +++ b/tests/scripts/json/daemon.json @@ -0,0 +1,12 @@ +{ + "bip": "172.17.0.1/16", + "default-address-pools": [ + { + "base": "172.17.0.0/16", + "size": 16 + } + ], + "insecure-registries": [ + "172.30.0.0/16" + ] +} diff --git a/tests/scripts/recreate-test-infrastructure.sh b/tests/scripts/recreate-test-infrastructure.sh new file mode 100755 index 000000000..c00518932 --- /dev/null +++ b/tests/scripts/recreate-test-infrastructure.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +set -uxe + + +URL=$(oc config view --minify -o jsonpath='{.clusters[*].cluster.server}') + +if [ ${URL} != "https://172.17.0.1:8443" ]; then + echo "You are not in a local cluster. Stopping now!!!" + exit 1 +fi + +if ! oc get clusterroles | grep request_role; then + oc create -f ${BASH_SOURCE%/*}/json/create-cluster-role.json +fi + +if [ ! -d ${BASH_SOURCE%/*}/../../ods-config ]; then + mkdir -p ${BASH_SOURCE%/*}/../../ods-config +fi + +${BASH_SOURCE%/*}/create-env-from-local-cluster.sh --base-oc-dir=${HOME}/openshift.local.clusterup --output ${BASH_SOURCE%/*}/../../ods-config/ods-core.env + +if [ -d "${BASH_SOURCE%/*}/../../../ods-configuration" ]; then + rm -rf "${BASH_SOURCE%/*}/../../../ods-configuration" +fi + +NAMSPACE="cd" +REF="cicdtests" + +if ! oc whoami; then + echo "You must be logged in to the OC Cluster" +fi + +if oc project ${NAMSPACE}; then + oc delete project ${NAMSPACE} +fi + +${BASH_SOURCE%/*}/deploy-mocks.sh --verbose --wait +${BASH_SOURCE%/*}/setup-mocked-ods-repo.sh --ods-ref ${REF} --verbose + +${BASH_SOURCE%/*}/../../ods-setup/setup-ods-project.sh --verbose --force --namespace ${NAMSPACE} + +${BASH_SOURCE%/*}/../../ods-setup/setup-jenkins-images.sh --namespace ${NAMSPACE} --force --verbose --ods-ref ${REF} + +source ${BASH_SOURCE%/*}/../../ods-config/ods-core.env +PROJECT_ID=prov +if oc project "${PROJECT_ID}-test" > /dev/null; then + oc delete project "${PROJECT_ID}-test" +fi +if oc project "${PROJECT_ID}-dev" > /dev/null; then + oc delete project "${PROJECT_ID}-dev" +fi +if oc project "${PROJECT_ID}-cd" > /dev/null; then + oc delete project "${PROJECT_ID}-cd" +fi + +PROJECT_ID=${PROJECT_ID} \ + ${BASH_SOURCE%/*}/../../create-projects/create-projects.sh --verbose + +PROJECT_ID=${PROJECT_ID} \ +CD_USER_TYPE=general \ +CD_USER_ID_B64=${CD_USER_ID_B64} \ +PIPELINE_TRIGGER_SECRET=${PIPELINE_TRIGGER_SECRET_B64} \ + ${BASH_SOURCE%/*}/../../create-projects/create-cd-jenkins.sh --ods-namespace ${NAMSPACE} --force --verbose + +oc adm policy add-cluster-role-to-user self-provisioner system:serviceaccount:prov-cd:jenkins diff --git a/tests/scripts/setup-mocked-ods-repo.sh b/tests/scripts/setup-mocked-ods-repo.sh new file mode 100755 index 000000000..f929475cf --- /dev/null +++ b/tests/scripts/setup-mocked-ods-repo.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +set -eu + +function usage { + printf "usage: %s [options]\n", $0 + printf "\t-h|--help\tPrints the usage\n" + printf "\t-v|--verbose\tVerbose output\n" + printf "\t-b|--ods-ref\tReference to be created in the mocked git repo.\n" + +} + +urlencode() { + # urlencode + + local length="${#1}" + for (( i = 0; i < length; i++ )); do + local c="${1:i:1}" + case $c in + [a-zA-Z0-9.~_-]) printf "$c" ;; + *) printf '%%%02X' "'$c" ;; + esac + done + + +} + +REF="" + +URL=$(oc config view --minify -o jsonpath='{.clusters[*].cluster.server}') +if [ ${URL} != "https://172.17.0.1:8443" ]; then + echo "You are not in a local cluster. Stopping now!!!" + exit 1 +fi + +while [[ "$#" -gt 0 ]]; do case $1 in + + -v|--verbose) set -x;; + + -h|--help) usage; exit 0;; + + -b=*|--ods-ref=*) REF="${1#*=}";; + -b|--ods-ref) REF="$2"; shift;; + + *) echo "Unknown parameter passed: $1"; usage; exit 1;; + esac; shift; done + +if git remote -v | grep mockbucket; then + git remote remove mockbucket +fi + + +if [ -z "${REF}" ]; then + echo "Reference --ods-ref must be provided" + exit 1 +fi + +source ${BASH_SOURCE%/*}/../../ods-config/ods-core.env + +docker ps | grep mockbucket + +# git checkout -b "${REF}" +HEAD=$(git rev-parse --abbrev-ref HEAD) +if [ "${HEAD}" = "HEAD" ]; then + HEAD="cicdtests" + git checkout -b ${HEAD} +fi +git remote add mockbucket "http://$(urlencode ${CD_USER_ID}):$(urlencode ${CD_USER_PWD})@${BITBUCKET_HOST}/scm/opendevstack/ods-core.git" +git -c http.sslVerify=false push mockbucket --set-upstream "${HEAD}:${REF}" +git remote remove mockbucket + +mkdir -p "${BASH_SOURCE%/*}/../../../ods-configuration" +cp ${BASH_SOURCE%/*}/../../ods-config/ods-core.env ${BASH_SOURCE%/*}/../../../ods-configuration + +cd "${BASH_SOURCE%/*}/../../../ods-configuration" +git init +git config user.email "test@suite.nip.io" +git config user.name "Test Suite" +git add ods-core.env +git commit -m "Initial Commit" +git remote add mockbucket "http://$(urlencode ${CD_USER_ID}):$(urlencode ${CD_USER_PWD})@${BITBUCKET_HOST}/scm/opendevstack/ods-configuration.git" +git -c http.sslVerify=false push mockbucket --set-upstream "$(git rev-parse --abbrev-ref HEAD):${REF}" +cd - diff --git a/tests/scripts/utils/print-jenkins-log.sh b/tests/scripts/utils/print-jenkins-log.sh new file mode 100755 index 000000000..3e16ef71d --- /dev/null +++ b/tests/scripts/utils/print-jenkins-log.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +LOG_URL=$(oc get build $1 -o json | jq '.metadata.annotations."openshift.io/jenkins-log-url"' | sed -e 's/"//g' ) +curl ${LOG_URL} --header "Authorization: Bearer $(oc get sa/builder --template='{{range .secrets}}{{ .name }} {{end}}' | xargs -n 1 oc get secret --template='{{ if .data.token }}{{ .data.token }}{{end}}' | head -n 1 | base64 -d -)" -k diff --git a/tests/utils/clean-up.go b/tests/utils/clean-up.go new file mode 100644 index 000000000..785407e21 --- /dev/null +++ b/tests/utils/clean-up.go @@ -0,0 +1,54 @@ +package utils + +import ( + projectClientV1 "github.com/openshift/client-go/project/clientset/versioned/typed/project/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "time" +) + +func RemoveProject(projectName string) error { + config, err := GetOCClient() + if err != nil { + return err + } + client, err := projectClientV1.NewForConfig(config) + if err != nil { + return err + } + project, err := client.Projects().Get(projectName, metav1.GetOptions{}) + if err != nil || project == nil { + return nil + } + + err = client.Projects().Delete(projectName, &metav1.DeleteOptions{}) + if err != nil { + return err + } + for { + time.Sleep(500 * time.Millisecond) + project, err = client.Projects().Get(projectName, metav1.GetOptions{}) + if err != nil || project == nil { + break + } + } + + return nil +} + +func RemoveAllTestOCProjects() error { + + err := RemoveProject(PROJECT_NAME_TEST) + if err != nil { + return err + } + err = RemoveProject(PROJECT_NAME_DEV) + if err != nil { + return err + } + err = RemoveProject(PROJECT_NAME_CD) + if err != nil { + return err + } + + return nil +} diff --git a/tests/utils/command-utils.go b/tests/utils/command-utils.go new file mode 100644 index 000000000..c6c358de8 --- /dev/null +++ b/tests/utils/command-utils.go @@ -0,0 +1,37 @@ +package utils + +import ( + "bytes" + "fmt" + "os" + "os/exec" + "path" + "runtime" +) + +func RunScriptFromBaseDir(command string, args []string, envVars []string) (string, string, error) { + _, filename, _, _ := runtime.Caller(0) + dir := path.Join(path.Dir(filename), "..", "..") + return RunCommand(fmt.Sprintf("%s/%s", dir, command), args, envVars) +} + +func RunCommand(command string, args []string, envVars []string) (string, string, error) { + cmd := exec.Command(command, args...) + cmd.Env = append(os.Environ(), envVars...) + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + err := cmd.Run() + return string(stdout.Bytes()), string(stderr.Bytes()), err +} + +func RunCommandWithWorkDir(command string, args []string, workDir string, envVars []string) (string, string, error) { + cmd := exec.Command(command, args...) + cmd.Env = append(os.Environ(), envVars...) + cmd.Dir = workDir + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + err := cmd.Run() + return string(stdout.Bytes()), string(stderr.Bytes()), err +} diff --git a/tests/utils/constants.go b/tests/utils/constants.go new file mode 100644 index 000000000..53b4be03d --- /dev/null +++ b/tests/utils/constants.go @@ -0,0 +1,7 @@ +package utils + +const PROJECT_NAME = "unitt" +const PROJECT_ENV_VAR = "PROJECT_ID=" + PROJECT_NAME +const PROJECT_NAME_CD = "unitt-cd" +const PROJECT_NAME_DEV = "unitt-dev" +const PROJECT_NAME_TEST = "unitt-test" diff --git a/tests/utils/images.go b/tests/utils/images.go new file mode 100644 index 000000000..1bde09bb4 --- /dev/null +++ b/tests/utils/images.go @@ -0,0 +1,20 @@ +package utils + +import ( + "fmt" + v1 "github.com/openshift/api/image/v1" +) + +func FindImageTag(imageStreamList *v1.ImageStreamList, imageName string, tagName string) error { + for _, imageStream := range imageStreamList.Items { + if imageStream.Name != imageName { + continue + } + for _, tag := range imageStream.Status.Tags { + if tag.Tag == tagName { + return nil + } + } + } + return fmt.Errorf("Image tag '%s:%s' not found.", imageName, tagName) +} diff --git a/tests/utils/ods-env.go b/tests/utils/ods-env.go new file mode 100644 index 000000000..81850d401 --- /dev/null +++ b/tests/utils/ods-env.go @@ -0,0 +1,33 @@ +package utils + +import ( + "bufio" + "os" + "path" + "runtime" + "strings" +) + +func ReadConfiguration() (map[string]string, error) { + + _, filename, _, _ := runtime.Caller(0) + dir := path.Join(path.Dir(filename), "..", "..", "ods-config", "ods-core.env") + + file, err := os.Open(dir) + if err != nil { + return nil, err + } + defer file.Close() + values := map[string]string{} + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + parts := strings.SplitN(line, "=", 2) + if len(parts) == 2 { + values[parts[0]] = parts[1] + } + } + + return values, nil +} diff --git a/tests/utils/openshift-client.go b/tests/utils/openshift-client.go new file mode 100644 index 000000000..dbe02e49e --- /dev/null +++ b/tests/utils/openshift-client.go @@ -0,0 +1,21 @@ +package utils + +import ( + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "os" + "path/filepath" +) + +func GetOCClient() (*rest.Config, error) { + home, err := os.UserHomeDir() + if err != nil { + return nil, err + } + config, err := clientcmd.BuildConfigFromFlags("", filepath.Join(home, ".kube", "config")) + if err != nil { + return nil, err + } + + return config, nil +} diff --git a/tests/utils/projects.go b/tests/utils/projects.go new file mode 100644 index 000000000..8c0f9ef57 --- /dev/null +++ b/tests/utils/projects.go @@ -0,0 +1,15 @@ +package utils + +import ( + "fmt" + v1 "github.com/openshift/api/project/v1" +) + +func FindProject(projects *v1.ProjectList, projectName string) error { + for _, project := range projects.Items { + if project.Name == projectName { + return nil + } + } + return fmt.Errorf("Project '%s' not found", projectName) +} diff --git a/tests/utils/role-bindings.go b/tests/utils/role-bindings.go new file mode 100644 index 000000000..d94392c87 --- /dev/null +++ b/tests/utils/role-bindings.go @@ -0,0 +1,28 @@ +package utils + +import ( + "fmt" + v1 "k8s.io/api/rbac/v1" +) + +func FindRoleBinding(roleBindings *v1.RoleBindingList, subjectName string, subjectType string, subjectNamespace string, roleName string) error { + for _, roleBinding := range roleBindings.Items { + for _, subject := range roleBinding.Subjects { + if subject.Name == subjectName && subject.Namespace == subjectNamespace && roleBinding.RoleRef.Name == roleName && subject.Kind == subjectType { + return nil + } + } + } + return fmt.Errorf("Subject '%s' of kind '%s' in namespace '%s' does not have the role '%s'", subjectName, subjectType, subjectNamespace, roleName) +} + +func FindClusterRoleBinding(roleBindings *v1.ClusterRoleBindingList, subjectName string, subjectType string, subjectNamespace string, roleName string) error { + for _, roleBinding := range roleBindings.Items { + for _, subject := range roleBinding.Subjects { + if subject.Name == subjectName && subject.Namespace == subjectNamespace && roleBinding.RoleRef.Name == roleName && subject.Kind == subjectType { + return nil + } + } + } + return fmt.Errorf("Subject '%s' of kind '%s' in namespace '%s' does not have the cluster role '%s'", subjectName, subjectType, subjectNamespace, roleName) +} diff --git a/tests/utils/types.go b/tests/utils/types.go new file mode 100644 index 000000000..6af2ed0c6 --- /dev/null +++ b/tests/utils/types.go @@ -0,0 +1,24 @@ +package utils + +type EnvPair struct { + Name string `json:"name"` + Value string `json:"value"` +} + +type RequestBuild struct { + Branch string `json:"branch"` + Repository string `json:"repository"` + Env []EnvPair `json:"env"` + Project string `json:"project"` +} +type RoleBinding struct { + SubjectName string + SubjectType string + Namespace string + RoleName string +} + +type ImageTag struct { + ImageName string + ImageTag string +}