Skip to content

Commit

Permalink
test: Testing Report (#122)
Browse files Browse the repository at this point in the history
* Initial Testing update for UAT Sanity tests

* update ibm uat test and add testing report code to github workflow

* Add 3rd job to uat suite to handle test report generation and attachment to output

* Update dependency management for each job

* Remove incorrect reference to mod path in test report creation job

* Remove redundant dependency install and upload html file without compression for faster viewing

* Add testing reporting capability to all workflows

* Move IBM2 UAT user data to example without provider

* revert back to single file for UAT sanity suite

* Tidy Go module

* Point UAT IBM2 test to example without external providers

* remove unnecessary file

* remove unnecessary .tfvars.example file
  • Loading branch information
thogarty authored Aug 28, 2024
1 parent e72d397 commit 1aae259
Show file tree
Hide file tree
Showing 13 changed files with 310 additions and 61 deletions.
25 changes: 18 additions & 7 deletions .github/workflows/terratests-fcr-prod-suite.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,16 @@ jobs:
go-version-file: './go.mod'
id: go

- name: Set up Python3
uses: actions/setup-python@v5
with:
python-version: '3.10'

- name: Get dependencies
run: |
go mod download
go mod download
go install github.com/thogarty/go-junit-report/v2@latest
pip3 install junit2html
- uses: hashicorp/setup-terraform@v3
with:
Expand Down Expand Up @@ -61,11 +68,15 @@ jobs:
jq --arg timestamp "$TIMESTAMP" '.connection_name = ($timestamp + "_" + .connection_name)' ./examples/cloud-router-2-virtual-device-connection/terraform.tfvars.json > ./examples/cloud-router-2-virtual-device-connection/tmp.test.json && mv ./examples/cloud-router-2-virtual-device-connection/tmp.test.json ./examples/cloud-router-2-virtual-device-connection/terraform.tfvars.json
- name: Run Go Tests
run:
go test ./tests/prod/cloud-router -v -coverprofile coverage_prod_modules.txt -covermode=atomic -count 1 -parallel 8 -run "(DIGP)" -timeout 180m
go test ./tests/prod/cloud-router -json -v -count 1 -parallel 8 -run "(DIGP)" -timeout 180m | tee fcr_prod_suite.log

- name: Create HTML Testing Report
run: |
go-junit-report -parser gojson -in fcr_prod_suite.log > fcr_prod_suite.xml && python3 -m junit2htmlreport fcr_prod_suite.xml fcr_prod_suite.html
- name: Upload test coverage to Codecov
if: ${{ always() }}
uses: codecov/codecov-action@v4
- name: Upload HTML Testing Report
uses: actions/upload-artifact@v4
with:
token: ${{ secrets.CODECOV_TOKEN }}
files: ./coverage_prod_modules.txt
name: FCR Prod Suite Terraform Modules Testing Report
path: fcr_prod_suite.html
compression-level: 0
23 changes: 17 additions & 6 deletions .github/workflows/terratests-metal-nimf-suite.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,16 @@ jobs:
go-version-file: './go.mod'
id: go

- name: Set up Python3
uses: actions/setup-python@v5
with:
python-version: '3.10'

- name: Get dependencies
run: |
go mod download
go install github.com/thogarty/go-junit-report/v2@latest
pip3 install junit2html
- uses: hashicorp/setup-terraform@v3
with:
Expand Down Expand Up @@ -68,11 +75,15 @@ jobs:
- name: Run Go Tests
run:
go test ./tests/prod/metal-nimf -v -coverprofile coverage_metal_nimf_modules.txt -covermode=atomic -count 1 -parallel 8 -timeout 180m
go test ./tests/prod/metal-nimf -v -json -count 1 -parallel 8 -timeout 180m | tee metal_prod_suite.log

- name: Create HTML Testing Report
run: |
go-junit-report -parser gojson -in metal_prod_suite.log > metal_prod_suite.xml && python3 -m junit2htmlreport metal_prod_suite.xml metal_prod_suite.html
- name: Upload test coverage to Codecov
if: ${{ always() }}
uses: codecov/codecov-action@v4
- name: Upload HTML Testing Report
uses: actions/upload-artifact@v4
with:
token: ${{ secrets.CODECOV_TOKEN }}
files: ./coverage_metal_nimf_modules.txt
name: Metal NIMF Prod Suite Terraform Modules Testing Report
path: metal_prod_suite.html
compression-level: 0
25 changes: 18 additions & 7 deletions .github/workflows/terratests-port-prod-suite.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,16 @@ jobs:
go-version-file: './go.mod'
id: go

- name: Set up Python3
uses: actions/setup-python@v5
with:
python-version: '3.10'

- name: Get dependencies
run: |
go mod download
go mod download
go install github.com/thogarty/go-junit-report/v2@latest
pip3 install junit2html
- uses: hashicorp/setup-terraform@v3
with:
Expand Down Expand Up @@ -67,11 +74,15 @@ jobs:
- name: Run Go Tests
run:
go test ./tests/prod/port -v -coverprofile coverage_prod_modules.txt -covermode=atomic -count 1 -parallel 8 -run "(DIGP)" -timeout 180m
go test ./tests/prod/port -v -json -count 1 -parallel 8 -run "(DIGP)" -timeout 180m | tee port_prod_suite.log

- name: Create HTML Testing Report
run: |
go-junit-report -parser gojson -in port_prod_suite.log > port_prod_suite.xml && python3 -m junit2htmlreport port_prod_suite.xml port_prod_suite.html
- name: Upload test coverage to Codecov
if: ${{ always() }}
uses: codecov/codecov-action@v4
- name: Upload HTML Testing Report
uses: actions/upload-artifact@v4
with:
token: ${{ secrets.CODECOV_TOKEN }}
files: ./coverage_prod_modules.txt
name: Port Prod Suite Terraform Modules Testing Report
path: port_prod_suite.html
compression-level: 0
23 changes: 17 additions & 6 deletions .github/workflows/terratests-prod-suite-all.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,16 @@ jobs:
go-version-file: './go.mod'
id: go

- name: Set up Python3
uses: actions/setup-python@v5
with:
python-version: '3.10'

- name: Get dependencies
run: |
go mod download
go install github.com/thogarty/go-junit-report/v2@latest
pip3 install junit2html
- uses: hashicorp/setup-terraform@v3
with:
Expand Down Expand Up @@ -103,11 +110,15 @@ jobs:
- name: Run Go Tests
run:
go test ./tests/prod -v -coverprofile coverage_prod_modules.txt -covermode=atomic -count 1 -parallel 8 -run "(DIGP)" -timeout 180m
go test ./tests/prod -json -v -count 1 -parallel 8 -run "(DIGP)" -timeout 180m | tee prod_suite_all.log

- name: Create HTML Testing Report
run: |
go-junit-report -parser gojson -in prod_suite_all.log > prod_suite_all.xml && python3 -m junit2htmlreport prod_suite_all.xml prod_suite_all.html
- name: Upload test coverage to Codecov
if: ${{ always() }}
uses: codecov/codecov-action@v4
- name: Upload HTML Testing Report
uses: actions/upload-artifact@v4
with:
token: ${{ secrets.CODECOV_TOKEN }}
files: ./coverage_prod_modules.txt
name: Prod Suite ALL Terraform Modules Testing Report
path: prod_suite_all.html
compression-level: 0
66 changes: 53 additions & 13 deletions .github/workflows/terratests-uat-suite.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -45,14 +45,13 @@ jobs:
- name: Run Go Tests
run:
go test ./tests/uat -v -coverprofile coverage_uat_modules.txt -covermode=atomic -count 1 -parallel 8 -run "(PNFV)" -timeout 180m
go test ./tests/uat -json -v -coverprofile coverage_uat_modules.txt -covermode=atomic -count 1 -parallel 8 -run "(PNFV)" -timeout 180m | tee pnfv_test_output.log

- name: Upload test coverage to Codecov
if: ${{ always() }}
uses: codecov/codecov-action@v4
- name: Upload PNFV Testing Log
uses: actions/upload-artifact@v4
with:
token: ${{ secrets.CODECOV_TOKEN }}
files: ./coverage_uat_modules.txt
name: pnfv_test_logs
path: pnfv_test_output.log

test-PFCR:
name: Setup UAT PFCR Tests
Expand Down Expand Up @@ -100,7 +99,7 @@ jobs:
run: |
echo $TEST_DATA_UAT_PORT_2_AWS_CONNECTION >> "./tests/examples-without-external-providers/port-2-aws-connection/terraform.tfvars.json"
echo $TEST_DATA_UAT_PORT_2_AZURE_CONNECTION >> "./tests/examples-without-external-providers/port-2-azure-connection/terraform.tfvars.json"
echo $TEST_DATA_UAT_PORT_2_IBM2_CONNECTION >> "./examples/port-2-ibm2-connection/terraform.tfvars.json"
echo $TEST_DATA_UAT_PORT_2_IBM2_CONNECTION >> "./tests/examples-without-external-providers/port-2-ibm2-connection/terraform.tfvars.json"
echo $TEST_DATA_UAT_PORT_2_PORT_CONNECTION >> "./examples/port-2-port-connection/terraform.tfvars.json"
echo $TEST_DATA_UAT_PORT_2_PRIVATE_SERVICE_PROFILE_CONNECTION >> "./examples/port-2-private-service-profile-connection/terraform.tfvars.json"
echo $TEST_DATA_UAT_PORT_2_PUBLIC_SERVICE_PROFILE_CONNECTION >> "./examples/port-2-public-service-profile-connection/terraform.tfvars.json"
Expand All @@ -115,11 +114,52 @@ jobs:
- name: Run Go Tests
run:
go test ./tests/uat -v -coverprofile coverage_uat_modules.txt -covermode=atomic -count 1 -parallel 8 -run "(PFCR)" -timeout 180m
go test ./tests/uat -json -v -count 1 -parallel 8 -run "(PFCR)" -timeout 180m | tee pfcr_test_output.log

- name: Upload test coverage to Codecov
if: ${{ always() }}
uses: codecov/codecov-action@v4
- name: Upload PFCR Testing Log
uses: actions/upload-artifact@v4
with:
token: ${{ secrets.CODECOV_TOKEN }}
files: ./coverage_uat_modules.txt
name: pfcr_test_logs
path: pfcr_test_output.log

upload-test-report:
name: Upload Testing Report
needs: [test-PNFV, test-PFCR]
runs-on: ubuntu-latest

steps:
- name: Set up Go
uses: actions/setup-go@v5
id: go

- name: Set up Python3
uses: actions/setup-python@v5
with:
python-version: '3.10'

- name: Get dependencies
run: |
go install github.com/thogarty/go-junit-report/v2@latest
pip3 install junit2html
- name: Download PNFV Test Logs
uses: actions/download-artifact@v4
with:
name: pnfv_test_logs

- name: Download PFCR Test Logs
uses: actions/download-artifact@v4
with:
name: pfcr_test_logs

- name: Create HTML Testing Report
run: |
cat pfcr_test_output.log pnfv_test_output.log > uat_sanity_report.log
go-junit-report -parser gojson -in uat_sanity_report.log > uat_sanity_report.xml && python3 -m junit2htmlreport uat_sanity_report.xml uat_sanity_report.html
- name: Upload HTML Testing Report
uses: actions/upload-artifact@v4
with:
name: UAT Terraform Modules Testing Report
path: uat_sanity_report.html
compression-level: 0
25 changes: 18 additions & 7 deletions .github/workflows/terratests-virtualdevice-prod-suite.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,16 @@ jobs:
go-version-file: './go.mod'
id: go

- name: Set up Python3
uses: actions/setup-python@v5
with:
python-version: '3.10'

- name: Get dependencies
run: |
go mod download
go mod download
go install github.com/thogarty/go-junit-report/v2@latest
pip3 install junit2html
- uses: hashicorp/setup-terraform@v3
with:
Expand All @@ -53,11 +60,15 @@ jobs:
- name: Run Go Tests
run:
go test ./tests/prod/virtual-device -v -coverprofile coverage_prod_modules.txt -covermode=atomic -count 1 -parallel 8 -run "(DIGP)" -timeout 180m
go test ./tests/prod/virtual-device -json -v -count 1 -parallel 8 -run "(DIGP)" -timeout 180m | tee vd_prod_suite.log

- name: Create HTML Testing Report
run: |
go-junit-report -parser gojson -in vd_prod_suite.log > vd_prod_suite.xml && python3 -m junit2htmlreport vd_prod_suite.xml vd_prod_suite.html
- name: Upload test coverage to Codecov
if: ${{ always() }}
uses: codecov/codecov-action@v4
- name: Upload HTML Testing Report
uses: actions/upload-artifact@v4
with:
token: ${{ secrets.CODECOV_TOKEN }}
files: ./coverage_prod_modules.txt
name: Virtual Device Prod Suite Terraform Modules Testing Report
path: vd_prod_suite.html
compression-level: 0
8 changes: 8 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -38,3 +38,11 @@ terraform.rc

# Ignore IDE Files
.idea*

# Ignore OSX Files
.DS_Store

# Ignore log/xml/html files generated from test output
*.log
*.xml
*.html
14 changes: 0 additions & 14 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -338,22 +338,12 @@ github.com/googleapis/gax-go/v2 v2.7.1 h1:gF4c0zjUP2H/s/hEGyLA3I0fA2ZWjzYiONAD6c
github.com/googleapis/gax-go/v2 v2.7.1/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI=
github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4=
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
github.com/gruntwork-io/terratest v0.43.0 h1:/ewxlYwd/HBralCt7P2bAhYM8hdkOiFJHrMz5XdlsSY=
github.com/gruntwork-io/terratest v0.43.0/go.mod h1:vZO4J6UW023NDvl1vI+twZk9r//+QEaNT6MfkheH7z0=
github.com/gruntwork-io/terratest v0.46.15 h1:qfqjTFveymaqe7aAWn3LjlK0SwVGpRfoOut5ggNyfQ8=
github.com/gruntwork-io/terratest v0.46.15/go.mod h1:9bd22zAojjBBiYdsp+AR1iyl2iB6bRUVm2Yf1AFhfrA=
github.com/gruntwork-io/terratest v0.46.16 h1:l+HHuU7lNLwoAl2sP8zkYJy0uoE2Mwha2nw+rim+OhQ=
github.com/gruntwork-io/terratest v0.46.16/go.mod h1:oywHw1cFKXSYvKPm27U7quZVzDUlA22H2xUrKCe26xM=
github.com/gruntwork-io/terratest v0.47.0 h1:xIy1pT7NbGVlMLDZEHl3+3iSnvffh8tN2pL6idn448c=
github.com/gruntwork-io/terratest v0.47.0/go.mod h1:oywHw1cFKXSYvKPm27U7quZVzDUlA22H2xUrKCe26xM=
github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
github.com/hashicorp/go-getter v1.7.1 h1:SWiSWN/42qdpR0MdhaOc/bLR48PLuP1ZQtYLRlM69uY=
github.com/hashicorp/go-getter v1.7.1/go.mod h1:W7TalhMmbPmsSMdNjD0ZskARur/9GJ17cfHTRtXV744=
github.com/hashicorp/go-getter v1.7.4 h1:3yQjWuxICvSpYwqSayAdKRFcvBl1y/vogCxczWSmix0=
github.com/hashicorp/go-getter v1.7.4/go.mod h1:W7TalhMmbPmsSMdNjD0ZskARur/9GJ17cfHTRtXV744=
github.com/hashicorp/go-getter v1.7.5 h1:dT58k9hQ/vbxNMwoI5+xFYAJuv6152UNvdHokfI5wE4=
github.com/hashicorp/go-getter v1.7.5/go.mod h1:W7TalhMmbPmsSMdNjD0ZskARur/9GJ17cfHTRtXV744=
github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI=
Expand Down Expand Up @@ -428,8 +418,6 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/tmccombs/hcl2json v0.3.3 h1:+DLNYqpWE0CsOQiEZu+OZm5ZBImake3wtITYxQ8uLFQ=
Expand Down Expand Up @@ -581,8 +569,6 @@ golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri
golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg=
golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg=
golang.org/x/oauth2 v0.1.0/go.mod h1:G9FE4dLTsbXUu90h/Pf85g4w1D+SSAgR+q46nJZ8M4A=
golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g=
golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4=
golang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8=
golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
provider "equinix" {
client_id = var.equinix_client_id
client_secret = var.equinix_client_secret
}

provider "ibm" {
ibmcloud_api_key = var.ibm_cloud_api_key
iaas_classic_username = var.ibm_classic_username
iaas_classic_api_key = var.ibm_classic_api_key
}

module "create_port_2_ibm2_connection" {
source = "../../../modules/port-connection"

connection_name = var.connection_name
connection_type = var.connection_type
notifications_type = var.notifications_type
notifications_emails = var.notifications_emails
project_id = var.project_id
bandwidth = var.bandwidth
purchase_order_number = var.purchase_order_number

# A-side
aside_port_name = var.aside_port_name
aside_vlan_tag = var.aside_vlan_tag
aside_vlan_inner_tag = var.aside_vlan_inner_tag

# Z-side
zside_ap_type = var.zside_ap_type
zside_ap_authentication_key = var.zside_ap_authentication_key
zside_ap_profile_type = var.zside_ap_profile_type
zside_location = var.zside_location
zside_seller_region = var.zside_seller_region
zside_sp_name = var.zside_sp_name
additional_info = var.additional_info
}

resource "time_sleep" "wait_dl_connection" {
create_duration = "1m"
}

data "ibm_dl_gateway" "test_ibm_dl_gateway" {
name = var.connection_name
depends_on = [time_sleep.wait_dl_connection]
}

data "ibm_resource_group" "rg" {
name = var.ibm_resource_group_name
}

resource "ibm_dl_gateway_action" "test_dl_gateway_action" {
gateway = data.ibm_dl_gateway.test_ibm_dl_gateway.id
action = var.ibm_gateway_action
global = var.ibm_gateway_global
metered = var.ibm_gateway_metered
resource_group = data.ibm_resource_group.rg.id
}
Loading

0 comments on commit 1aae259

Please sign in to comment.