diff --git a/.github/workflows/release-discover-branches.yaml b/.github/workflows/release-discover-branches.yaml new file mode 100644 index 00000000..bbeb26a1 --- /dev/null +++ b/.github/workflows/release-discover-branches.yaml @@ -0,0 +1,54 @@ +--- +name: Discover branches + +on: + pull_request: + branches: + - '**' + push: + branches: + - 'main' + schedule: + - cron: "0 5 * * *" # Daily at 05:00. + workflow_dispatch: # Manual workflow trigger + +jobs: + discover-branches: + name: generate-ci + runs-on: ubuntu-latest + env: + GOPATH: ${{ github.workspace }} + steps: + + - name: Checkout openshift-knative/hack + uses: actions/checkout@v3 + with: + path: ./src/github.com/openshift-knative/hack + + - name: Setup Golang + uses: actions/setup-go@v5 + with: + go-version-file: ./src/github.com/openshift-knative/hack/go.mod + + - name: Configure Git user + run: | + git config --global user.email "serverless-support@redhat.com" + git config --global user.name "OpenShift Serverless" + + - name: Discover branches + working-directory: ./src/github.com/openshift-knative/hack + run: make discover-branches ARGS="--config config/" + + - name: Create Discovery Pull Request + if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.ref_name == 'main' + uses: peter-evans/create-pull-request@v5 + with: + token: ${{ secrets.SERVERLESS_QE_ROBOT }} + path: ./src/github.com/openshift-knative/hack + base: main + branch: discover-serverless-ci + title: "Add new midstream branches" + commit-message: "Discover new branches" + delete-branch: true + body: | + Discover new branches using openshift-knative/hack. diff --git a/Makefile b/Makefile index 5fcc922c..9c85a7ac 100644 --- a/Makefile +++ b/Makefile @@ -25,6 +25,10 @@ generate-serverless-operator-ci: go run github.com/openshift-knative/hack/cmd/prowgen --config config/serverless-operator.yaml $(ARGS) .PHONY: generate-serverless-operator-ci +discover-branches: + go run github.com/openshift-knative/hack/cmd/discover $(ARGS) +.PHONY: discover-branches + unit-tests: go test ./pkg/... diff --git a/cmd/discover/discover.go b/cmd/discover/discover.go new file mode 100644 index 00000000..b865dc03 --- /dev/null +++ b/cmd/discover/discover.go @@ -0,0 +1,9 @@ +package main + +import ( + "github.com/openshift-knative/hack/pkg/discover" +) + +func main() { + discover.Main() +} diff --git a/config/eventing-hyperfoil-benchmark.yaml b/config/eventing-hyperfoil-benchmark.yaml index 4b7dabfa..7a453532 100644 --- a/config/eventing-hyperfoil-benchmark.yaml +++ b/config/eventing-hyperfoil-benchmark.yaml @@ -1,18 +1,17 @@ -# Full struct in cmd/prowgen/prowgen.go#Config - config: branches: - "main": + main: openShiftVersions: - - version: "4.14" - + - version: "4.14" repositories: - - org: openshift-knative - repo: eventing-hyperfoil-benchmark - imagePrefix: knative-eventing-hyperfoil-benchmark - slackChannel: "#knative-eventing-ci" - e2e: - - match: ".*test-kafka-broker-upstream-.*" - - match: ".*test-kafka-broker-namespaced-midstream-.*" - onDemand: true - +- dockerfiles: {} + e2e: + - match: .*test-kafka-broker-upstream-.* + - match: .*test-kafka-broker-namespaced-midstream-.* + onDemand: true + ignoreConfigs: {} + imagePrefix: knative-eventing-hyperfoil-benchmark + org: openshift-knative + promotion: {} + repo: eventing-hyperfoil-benchmark + slackChannel: '#knative-eventing-ci' diff --git a/config/eventing-istio.yaml b/config/eventing-istio.yaml index 63eb382d..ce675690 100644 --- a/config/eventing-istio.yaml +++ b/config/eventing-istio.yaml @@ -1,34 +1,33 @@ -# Full struct in cmd/prowgen/prowgen.go#Config - config: branches: - "release-v1.10": + release-next: openShiftVersions: - - version: "4.14" - - version: "4.11" - "release-v1.11": + - version: "4.14" + - version: "4.11" + release-v1.10: openShiftVersions: - - version: "4.14" - - version: "4.11" - "release-v1.12": + - version: "4.14" + - version: "4.11" + release-v1.11: openShiftVersions: - - version: "4.14" - - version: "4.11" - "release-v1.13": + - version: "4.14" + - version: "4.11" + release-v1.12: openShiftVersions: - - version: "4.14" - - version: "4.11" - "release-next": + - version: "4.14" + - version: "4.11" + release-v1.13: openShiftVersions: - - version: "4.14" - - version: "4.11" - + - version: "4.14" + - version: "4.11" repositories: - - org: openshift-knative - repo: eventing-istio - promotion: - namespace: openshift - imagePrefix: knative-eventing-istio - slackChannel: "#knative-eventing-ci" - e2e: - - match: ".*e2e-tests$" +- dockerfiles: {} + e2e: + - match: .*e2e-tests$ + ignoreConfigs: {} + imagePrefix: knative-eventing-istio + org: openshift-knative + promotion: + namespace: openshift + repo: eventing-istio + slackChannel: '#knative-eventing-ci' diff --git a/config/eventing-kafka-broker.yaml b/config/eventing-kafka-broker.yaml index e34d2074..653f2288 100644 --- a/config/eventing-kafka-broker.yaml +++ b/config/eventing-kafka-broker.yaml @@ -1,36 +1,35 @@ -# Full struct in cmd/prowgen/prowgen.go#Config - config: branches: - "release-v1.10": + release-next: openShiftVersions: - - version: "4.14" - - version: "4.11" - "release-v1.11": + - version: "4.14" + - version: "4.11" + release-v1.10: openShiftVersions: - - version: "4.14" - - version: "4.11" - "release-v1.12": + - version: "4.14" + - version: "4.11" + release-v1.11: openShiftVersions: - - version: "4.14" - - version: "4.11" - "release-v1.13": + - version: "4.14" + - version: "4.11" + release-v1.12: openShiftVersions: - - version: "4.14" - - version: "4.11" - "release-next": + - version: "4.14" + - version: "4.11" + release-v1.13: openShiftVersions: - - version: "4.14" - - version: "4.11" - + - version: "4.14" + - version: "4.11" repositories: - - org: openshift-knative - repo: eventing-kafka-broker - promotion: - namespace: openshift - imagePrefix: knative-eventing-kafka-broker - slackChannel: "#knative-eventing-ci" - e2e: - - match: ".*e2e$" - - match: ".*reconciler.*" - - match: ".*conformance.*" +- dockerfiles: {} + e2e: + - match: .*e2e$ + - match: .*reconciler.* + - match: .*conformance.* + ignoreConfigs: {} + imagePrefix: knative-eventing-kafka-broker + org: openshift-knative + promotion: + namespace: openshift + repo: eventing-kafka-broker + slackChannel: '#knative-eventing-ci' diff --git a/config/eventing.yaml b/config/eventing.yaml index c7b266db..39e6adaa 100644 --- a/config/eventing.yaml +++ b/config/eventing.yaml @@ -1,36 +1,35 @@ -# Full struct in cmd/prowgen/prowgen.go#Config - config: branches: - "release-v1.10": + release-next: openShiftVersions: - - version: "4.14" - - version: "4.11" - "release-v1.11": + - version: "4.14" + - version: "4.11" + release-v1.10: openShiftVersions: - - version: "4.14" - - version: "4.11" - "release-v1.12": + - version: "4.14" + - version: "4.11" + release-v1.11: openShiftVersions: - - version: "4.14" - - version: "4.11" - "release-v1.13": + - version: "4.14" + - version: "4.11" + release-v1.12: openShiftVersions: - - version: "4.14" - - version: "4.11" - "release-next": + - version: "4.14" + - version: "4.11" + release-v1.13: openShiftVersions: - - version: "4.14" - - version: "4.11" - + - version: "4.14" + - version: "4.11" repositories: - - org: openshift-knative - repo: eventing - promotion: - namespace: openshift - imagePrefix: knative-eventing - slackChannel: "#knative-eventing-ci" - e2e: - - match: ".*e2e$" - - match: ".*reconciler.*" - - match: ".*conformance.*" +- dockerfiles: {} + e2e: + - match: .*e2e$ + - match: .*reconciler.* + - match: .*conformance.* + ignoreConfigs: {} + imagePrefix: knative-eventing + org: openshift-knative + promotion: + namespace: openshift + repo: eventing + slackChannel: '#knative-eventing-ci' diff --git a/config/serverless-operator.yaml b/config/serverless-operator.yaml index 3b895a4d..5437c16f 100644 --- a/config/serverless-operator.yaml +++ b/config/serverless-operator.yaml @@ -1,337 +1,345 @@ -# Full struct in cmd/prowgen/prowgen.go#Config - config: branches: main: openShiftVersions: - - version: "4.14" - cron: "0 4 * * *" - # Note: customConfigs section below hardcodes OCP versions in a few places. They must be updated. - generateCustomConfigs: true - - version: "4.11" - cron: "0 6 * * *" - onDemand: true - -repositories: - - org: openshift-knative - repo: serverless-operator - promotion: - namespace: knative - imagePrefix: serverless - imageNameOverrides: - serverless-operator: bundle - slackChannel: "#serverless-ci" - ignoreConfigs: - matches: - - ".*main.yaml$" - - ".*lp-interop.*" - - ".*lp-interop.*" - dockerfiles: - matches: - - "knative-operator/.*" - - "openshift-knative-operator/.*" - - "serving/metadata-webhook/.*" - - "serving/ingress/.*" - - "olm-catalog/serverless-operator/Dockerfile" - - "olm-catalog/serverless-operator/index/Dockerfile" - e2e: - - match: "^operator-e2e$" - - match: "^test-upgrade$" - - match: "^upstream-e2e$" - ignoreError: true - skipCron: true - - match: "^upstream-e2e-kafka$" - ignoreError: true - - match: "^mesh-e2e$" - onDemand: true - ignoreError: true - timeout: 4h0m0s - - match: "^mesh-upgrade$" - onDemand: true - ignoreError: true - skipCron: true - - match: "^ui-e2e$" + - cron: 0 4 * * * + generateCustomConfigs: true + version: "4.14" + - cron: 0 6 * * * onDemand: true - runIfChanged: "test/ui" - ignoreError: true - - match: "^kitchensink-e2e$" - onDemand: true - ignoreError: true - - match: "^kitchensink-upgrade$" - onDemand: true - ignoreError: true - resources: - '*': - limits: - memory: 6Gi - requests: - cpu: 100m - memory: 200Mi - customConfigs: - - name: "aws-ovn" # Results in openshift-knative-serverless-operator-__-aws-ovn.yaml - releaseBuildConfiguration: - tests: - - as: e2e-aws-ovn-continuous - cron: 0 0 * * 0 - steps: - cluster_profile: aws - test: - - as: operator-e2e - commands: make test-e2e-with-kafka - # dependencies: Generated by openshift-knative/hack - from: serverless-source-image - resources: - limits: - memory: 6Gi - requests: - cpu: 100m - memory: 200Mi - - as: knative-serving-eventing-e2e - commands: make test-upstream-e2e-no-upgrade - # dependencies: Generated by openshift-knative/hack - from: serverless-source-image - resources: - limits: - memory: 6Gi - requests: - cpu: 100m - memory: 200Mi - - as: knative-eventing-kafka-broker-e2e - commands: make test-upstream-e2e-kafka-no-upgrade - # dependencies: Generated by openshift-knative/hack - from: serverless-source-image - resources: - limits: - memory: 6Gi - requests: - cpu: 100m - memory: 200Mi - workflow: openshift-e2e-aws-ovn - timeout: 8h0m0s - - name: "azure" # Results in openshift-knative-serverless-operator-__-azure.yaml - releaseBuildConfiguration: - tests: - - as: e2e-azure-continuous - cron: 0 0 * * 0 - steps: - cluster_profile: azure4 - test: - - as: operator-e2e - commands: make test-e2e-with-kafka - # dependencies: Generated by openshift-knative/hack - from: serverless-source-image - resources: - limits: - memory: 6Gi - requests: - cpu: 100m - memory: 200Mi - - as: knative-serving-eventing-e2e - commands: make test-upstream-e2e-no-upgrade - # dependencies: Generated by openshift-knative/hack - from: serverless-source-image - resources: - limits: - memory: 6Gi - requests: - cpu: 100m - memory: 200Mi - - as: knative-eventing-kafka-broker-e2e - commands: make test-upstream-e2e-kafka-no-upgrade - # dependencies: Generated by openshift-knative/hack - from: serverless-source-image - resources: - limits: - memory: 6Gi - requests: - cpu: 100m - memory: 200Mi - workflow: openshift-e2e-azure - timeout: 8h0m0s - - name: "gcp" # Results in openshift-knative-serverless-operator-__-gcp.yaml - releaseBuildConfiguration: - tests: - - as: e2e-gcp-continuous - cron: 0 0 * * 0 - steps: - cluster_profile: gcp - test: - - as: operator-e2e - commands: make test-e2e-with-kafka - # dependencies: Generated by openshift-knative/hack - from: serverless-source-image - resources: - limits: - memory: 6Gi - requests: - cpu: 100m - memory: 200Mi - - as: knative-serving-eventing-e2e - commands: make test-upstream-e2e-no-upgrade - # dependencies: Generated by openshift-knative/hack - from: serverless-source-image - resources: - limits: - memory: 6Gi - requests: - cpu: 100m - memory: 200Mi - - as: knative-eventing-kafka-broker-e2e - commands: make test-upstream-e2e-kafka-no-upgrade - # dependencies: Generated by openshift-knative/hack - from: serverless-source-image - resources: - limits: - memory: 6Gi - requests: - cpu: 100m - memory: 200Mi - workflow: openshift-e2e-gcp - timeout: 8h0m0s - - name: "hypershift" # Results in openshift-knative-serverless-operator-__-hypershift.yaml - releaseBuildConfiguration: - base_images: - hypershift-operator: - name: "4.14" - namespace: ocp - tag: hypershift-operator - upi-installer: - name: "4.14" - namespace: ocp - tag: upi-installer - releases: - latest: - integration: - include_built_images: true - name: "4.14" - namespace: ocp - tests: - - as: e2e-hypershift-continuous - cron: 0 0 * * 0 - steps: - cluster_profile: aws-cspi-qe - env: - BASE_DOMAIN: cspilp.interop.ccitredhat.com - HYPERSHIFT_BASE_DOMAIN: cspilp.interop.ccitredhat.com - COMPUTE_NODE_REPLICAS: "5" - test: - - as: operator-e2e - commands: make USER_MANAGEMENT_ALLOWED=false test-e2e-with-kafka - # dependencies: Generated by openshift-knative/hack - from: serverless-source-image - resources: - limits: - memory: 6Gi - requests: - cpu: 100m - memory: 200Mi - - as: knative-serving-eventing-e2e - commands: make test-upstream-e2e-no-upgrade - # dependencies: Generated by openshift-knative/hack - from: serverless-source-image - resources: - limits: - memory: 6Gi - requests: - cpu: 100m - memory: 200Mi - - as: knative-eventing-kafka-broker-e2e - commands: make test-upstream-e2e-kafka-no-upgrade - # dependencies: Generated by openshift-knative/hack - from: serverless-source-image - resources: - limits: - memory: 6Gi - requests: - cpu: 100m - memory: 200Mi - workflow: cucushift-installer-rehearse-aws-ipi-ovn-hypershift-guest - timeout: 8h0m0s - - name: "osd" # Results in openshift-knative-serverless-operator-__-osd.yaml - releaseBuildConfiguration: - base_images: - cli-ocm: - name: cli-ocm - namespace: ci - tag: latest - tests: - - as: e2e-osd-continuous - cron: 0 0 * * 0 - steps: - cluster_profile: osd-ephemeral - env: - CLUSTER_DURATION: "10800" - CLUSTER_VERSION: "4.14" - COMPUTE_NODES: "4" - post: - - chain: gather - - ref: osd-delete-delete - pre: - - ref: ipi-install-rbac - - ref: osd-create-create - test: - - as: operator-e2e - commands: make test-e2e-with-kafka - # dependencies: Generated by openshift-knative/hack - from: serverless-source-image - resources: - limits: - memory: 6Gi - requests: - cpu: 100m - memory: 200Mi - - as: knative-serving-eventing-e2e - commands: make test-upstream-e2e-no-upgrade - # dependencies: Generated by openshift-knative/hack - from: serverless-source-image - resources: - limits: - memory: 6Gi - requests: - cpu: 100m - memory: 200Mi - timeout: 8h0m0s - - name: "single-node" # Results in openshift-knative-serverless-operator-__-single-node.yaml - releaseBuildConfiguration: - tests: - - as: e2e-sno-continuous - cron: 0 0 * * 0 - steps: - cluster_profile: gcp - test: - - as: operator-e2e - commands: make HA=false test-e2e-with-kafka - # dependencies: Generated by openshift-knative/hack - from: serverless-source-image - resources: - limits: - memory: 6Gi - requests: - cpu: 100m - memory: 200Mi - workflow: openshift-e2e-gcp-single-node - timeout: 8h0m0s - - name: "vsphere" # Results in openshift-knative-serverless-operator-__-vsphere.yaml - releaseBuildConfiguration: - base_images: - upi-installer: - name: "4.14" - namespace: ocp - tag: upi-installer - tests: - - as: e2e-vsphere-continuous - cron: 0 0 * * 0 - steps: - cluster_profile: vsphere-2 - test: - - as: operator-e2e - commands: make HA=false test-e2e-with-kafka - # dependencies: Generated by openshift-knative/hack - from: serverless-source-image - resources: - limits: - memory: 6Gi - requests: - cpu: 100m - memory: 200Mi - workflow: openshift-e2e-vsphere-upi + version: "4.11" +repositories: +- customConfigs: + - name: aws-ovn + releaseBuildConfiguration: + tests: + - as: e2e-aws-ovn-continuous + cron: 0 0 * * 0 + steps: + cluster_profile: aws + test: + - as: operator-e2e + commands: make test-e2e-with-kafka + from: serverless-source-image + resources: + limits: + memory: 6Gi + requests: + cpu: 100m + memory: 200Mi + - as: knative-serving-eventing-e2e + commands: make test-upstream-e2e-no-upgrade + from: serverless-source-image + resources: + limits: + memory: 6Gi + requests: + cpu: 100m + memory: 200Mi + - as: knative-eventing-kafka-broker-e2e + commands: make test-upstream-e2e-kafka-no-upgrade + from: serverless-source-image + resources: + limits: + memory: 6Gi + requests: + cpu: 100m + memory: 200Mi + workflow: openshift-e2e-aws-ovn + timeout: 8h0m0s + zz_generated_metadata: + branch: "" + org: "" + repo: "" + - name: azure + releaseBuildConfiguration: + tests: + - as: e2e-azure-continuous + cron: 0 0 * * 0 + steps: + cluster_profile: azure4 + test: + - as: operator-e2e + commands: make test-e2e-with-kafka + from: serverless-source-image + resources: + limits: + memory: 6Gi + requests: + cpu: 100m + memory: 200Mi + - as: knative-serving-eventing-e2e + commands: make test-upstream-e2e-no-upgrade + from: serverless-source-image + resources: + limits: + memory: 6Gi + requests: + cpu: 100m + memory: 200Mi + - as: knative-eventing-kafka-broker-e2e + commands: make test-upstream-e2e-kafka-no-upgrade + from: serverless-source-image + resources: + limits: + memory: 6Gi + requests: + cpu: 100m + memory: 200Mi + workflow: openshift-e2e-azure + timeout: 8h0m0s + zz_generated_metadata: + branch: "" + org: "" + repo: "" + - name: gcp + releaseBuildConfiguration: + tests: + - as: e2e-gcp-continuous + cron: 0 0 * * 0 + steps: + cluster_profile: gcp + test: + - as: operator-e2e + commands: make test-e2e-with-kafka + from: serverless-source-image + resources: + limits: + memory: 6Gi + requests: + cpu: 100m + memory: 200Mi + - as: knative-serving-eventing-e2e + commands: make test-upstream-e2e-no-upgrade + from: serverless-source-image + resources: + limits: + memory: 6Gi + requests: + cpu: 100m + memory: 200Mi + - as: knative-eventing-kafka-broker-e2e + commands: make test-upstream-e2e-kafka-no-upgrade + from: serverless-source-image + resources: + limits: + memory: 6Gi + requests: + cpu: 100m + memory: 200Mi + workflow: openshift-e2e-gcp + timeout: 8h0m0s + zz_generated_metadata: + branch: "" + org: "" + repo: "" + - name: hypershift + releaseBuildConfiguration: + base_images: + hypershift-operator: + name: "4.14" + namespace: ocp + tag: hypershift-operator + upi-installer: + name: "4.14" + namespace: ocp + tag: upi-installer + releases: + latest: + integration: + include_built_images: true + name: "4.14" + namespace: ocp + tests: + - as: e2e-hypershift-continuous + cron: 0 0 * * 0 + steps: + cluster_profile: aws-cspi-qe + env: + BASE_DOMAIN: cspilp.interop.ccitredhat.com + COMPUTE_NODE_REPLICAS: "5" + HYPERSHIFT_BASE_DOMAIN: cspilp.interop.ccitredhat.com + test: + - as: operator-e2e + commands: make USER_MANAGEMENT_ALLOWED=false test-e2e-with-kafka + from: serverless-source-image + resources: + limits: + memory: 6Gi + requests: + cpu: 100m + memory: 200Mi + - as: knative-serving-eventing-e2e + commands: make test-upstream-e2e-no-upgrade + from: serverless-source-image + resources: + limits: + memory: 6Gi + requests: + cpu: 100m + memory: 200Mi + - as: knative-eventing-kafka-broker-e2e + commands: make test-upstream-e2e-kafka-no-upgrade + from: serverless-source-image + resources: + limits: + memory: 6Gi + requests: + cpu: 100m + memory: 200Mi + workflow: cucushift-installer-rehearse-aws-ipi-ovn-hypershift-guest + timeout: 8h0m0s + zz_generated_metadata: + branch: "" + org: "" + repo: "" + - name: osd + releaseBuildConfiguration: + base_images: + cli-ocm: + name: cli-ocm + namespace: ci + tag: latest + tests: + - as: e2e-osd-continuous + cron: 0 0 * * 0 + steps: + cluster_profile: osd-ephemeral + env: + CLUSTER_DURATION: "10800" + CLUSTER_VERSION: "4.14" + COMPUTE_NODES: "4" + post: + - chain: gather + - ref: osd-delete-delete + pre: + - ref: ipi-install-rbac + - ref: osd-create-create + test: + - as: operator-e2e + commands: make test-e2e-with-kafka + from: serverless-source-image + resources: + limits: + memory: 6Gi + requests: + cpu: 100m + memory: 200Mi + - as: knative-serving-eventing-e2e + commands: make test-upstream-e2e-no-upgrade + from: serverless-source-image + resources: + limits: + memory: 6Gi + requests: + cpu: 100m + memory: 200Mi + timeout: 8h0m0s + zz_generated_metadata: + branch: "" + org: "" + repo: "" + - name: single-node + releaseBuildConfiguration: + tests: + - as: e2e-sno-continuous + cron: 0 0 * * 0 + steps: + cluster_profile: gcp + test: + - as: operator-e2e + commands: make HA=false test-e2e-with-kafka + from: serverless-source-image + resources: + limits: + memory: 6Gi + requests: + cpu: 100m + memory: 200Mi + workflow: openshift-e2e-gcp-single-node + timeout: 8h0m0s + zz_generated_metadata: + branch: "" + org: "" + repo: "" + - name: vsphere + releaseBuildConfiguration: + base_images: + upi-installer: + name: "4.14" + namespace: ocp + tag: upi-installer + tests: + - as: e2e-vsphere-continuous + cron: 0 0 * * 0 + steps: + cluster_profile: vsphere-2 + test: + - as: operator-e2e + commands: make HA=false test-e2e-with-kafka + from: serverless-source-image + resources: + limits: + memory: 6Gi + requests: + cpu: 100m + memory: 200Mi + workflow: openshift-e2e-vsphere-upi + zz_generated_metadata: + branch: "" + org: "" + repo: "" + dockerfiles: + matches: + - knative-operator/.* + - openshift-knative-operator/.* + - serving/metadata-webhook/.* + - serving/ingress/.* + - olm-catalog/serverless-operator/Dockerfile + - olm-catalog/serverless-operator/index/Dockerfile + e2e: + - match: ^operator-e2e$ + - match: ^test-upgrade$ + - ignoreError: true + match: ^upstream-e2e$ + skipCron: true + - ignoreError: true + match: ^upstream-e2e-kafka$ + - ignoreError: true + match: ^mesh-e2e$ + onDemand: true + timeout: 4h0m0s + - ignoreError: true + match: ^mesh-upgrade$ + onDemand: true + skipCron: true + - ignoreError: true + match: ^ui-e2e$ + onDemand: true + runIfChanged: test/ui + - ignoreError: true + match: ^kitchensink-e2e$ + onDemand: true + - ignoreError: true + match: ^kitchensink-upgrade$ + onDemand: true + ignoreConfigs: + matches: + - .*main.yaml$ + - .*lp-interop.* + - .*lp-interop.* + imageNameOverrides: + serverless-operator: bundle + imagePrefix: serverless + org: openshift-knative + promotion: + namespace: knative + repo: serverless-operator + resources: + '*': + limits: + memory: 6Gi + requests: + cpu: 100m + memory: 200Mi + slackChannel: '#serverless-ci' diff --git a/config/serving-net-istio.yaml b/config/serving-net-istio.yaml index e622a928..31007cb9 100644 --- a/config/serving-net-istio.yaml +++ b/config/serving-net-istio.yaml @@ -1,28 +1,27 @@ -# Full struct in cmd/prowgen/prowgen.go#Config - config: branches: - "release-v1.10": + release-v1.10: openShiftVersions: - - version: "4.14" - - version: "4.11" - "release-v1.11": + - version: "4.14" + - version: "4.11" + release-v1.11: openShiftVersions: - - version: "4.14" - - version: "4.11" - "release-v1.12": + - version: "4.14" + - version: "4.11" + release-v1.12: openShiftVersions: - - version: "4.14" - - version: "4.11" - "release-v1.13": + - version: "4.14" + - version: "4.11" + release-v1.13: openShiftVersions: - - version: "4.14" - - version: "4.11" - + - version: "4.14" + - version: "4.11" repositories: - - org: openshift-knative - repo: net-istio - promotion: - namespace: openshift - imagePrefix: net-istio - slackChannel: "#knative-serving-ci" +- dockerfiles: {} + ignoreConfigs: {} + imagePrefix: net-istio + org: openshift-knative + promotion: + namespace: openshift + repo: net-istio + slackChannel: '#knative-serving-ci' diff --git a/config/serving-net-kourier.yaml b/config/serving-net-kourier.yaml index 430fa0b8..aa3d1c9a 100644 --- a/config/serving-net-kourier.yaml +++ b/config/serving-net-kourier.yaml @@ -1,28 +1,27 @@ -# Full struct in cmd/prowgen/prowgen.go#Config - config: branches: - "release-v1.10": + release-v1.10: openShiftVersions: - - version: "4.14" - - version: "4.11" - "release-v1.11": + - version: "4.14" + - version: "4.11" + release-v1.11: openShiftVersions: - - version: "4.14" - - version: "4.11" - "release-v1.12": + - version: "4.14" + - version: "4.11" + release-v1.12: openShiftVersions: - - version: "4.14" - - version: "4.11" - "release-v1.13": + - version: "4.14" + - version: "4.11" + release-v1.13: openShiftVersions: - - version: "4.14" - - version: "4.11" - + - version: "4.14" + - version: "4.11" repositories: - - org: openshift-knative - repo: net-kourier - promotion: - namespace: openshift - imagePrefix: net-kourier - slackChannel: "#knative-serving-ci" +- dockerfiles: {} + ignoreConfigs: {} + imagePrefix: net-kourier + org: openshift-knative + promotion: + namespace: openshift + repo: net-kourier + slackChannel: '#knative-serving-ci' diff --git a/config/serving.yaml b/config/serving.yaml index f351bc2d..6072cfbd 100644 --- a/config/serving.yaml +++ b/config/serving.yaml @@ -1,68 +1,66 @@ -# Full struct in cmd/prowgen/prowgen.go#Config - config: branches: - "release-v1.10": + release-next: openShiftVersions: - - version: "4.14" - - version: "4.11" - "release-v1.11": + - version: "4.14" + - version: "4.11" + skipDockerFilesMatches: + - openshift/ci-operator/knative-perf-images.* + skipE2EMatches: + - perf-tests$ + - .*e2e-tls$ + release-v1.10: openShiftVersions: - - version: "4.14" - - version: "4.11" - "release-v1.12": + - version: "4.14" + - version: "4.11" + release-v1.11: openShiftVersions: - - version: "4.14" - - version: "4.11" - "release-v1.13": + - version: "4.14" + - version: "4.11" + release-v1.12: openShiftVersions: - - version: "4.14" - - version: "4.11" - "release-next": - skipE2EMatches: - - "perf-tests$" - - ".*e2e-tls$" - skipDockerFilesMatches: - - "openshift/ci-operator/knative-perf-images.*" + - version: "4.14" + - version: "4.11" + release-v1.13: openShiftVersions: - - version: "4.14" - - version: "4.11" - + - version: "4.14" + - version: "4.11" repositories: - - org: openshift-knative - repo: serving - promotion: - namespace: openshift - imagePrefix: knative-serving - imageNameOverrides: - migrate: storage-version-migration - slackChannel: "#knative-serving-ci" - e2e: - - match: ".*e2e$" - skipImages: - - "knative-serving-load-test" - - "knative-serving-dataplane-probe" - - "knative-serving-real-traffic-test" - - "knative-serving-reconciliation-delay" - - "knative-serving-rollout-probe" - - "knative-serving-scale-from-zero" - - match: ".*e2e-tls$" - skipImages: - - "knative-serving-load-test" - - "knative-serving-dataplane-probe" - - "knative-serving-real-traffic-test" - - "knative-serving-reconciliation-delay" - - "knative-serving-rollout-probe" - - "knative-serving-scale-from-zero" - - match: "perf-tests$" - onDemand: true - dockerfiles: - matches: - - "openshift/ci-operator/knative-images.*" - - "openshift/ci-operator/knative-test-images.*" - - "openshift/ci-operator/knative-perf-images.*" - resources: - '*': - requests: - cpu: 500m - memory: 4Gi +- dockerfiles: + matches: + - openshift/ci-operator/knative-images.* + - openshift/ci-operator/knative-test-images.* + - openshift/ci-operator/knative-perf-images.* + e2e: + - match: .*e2e$ + skipImages: + - knative-serving-load-test + - knative-serving-dataplane-probe + - knative-serving-real-traffic-test + - knative-serving-reconciliation-delay + - knative-serving-rollout-probe + - knative-serving-scale-from-zero + - match: .*e2e-tls$ + skipImages: + - knative-serving-load-test + - knative-serving-dataplane-probe + - knative-serving-real-traffic-test + - knative-serving-reconciliation-delay + - knative-serving-rollout-probe + - knative-serving-scale-from-zero + - match: perf-tests$ + onDemand: true + ignoreConfigs: {} + imageNameOverrides: + migrate: storage-version-migration + imagePrefix: knative-serving + org: openshift-knative + promotion: + namespace: openshift + repo: serving + resources: + '*': + requests: + cpu: 500m + memory: 4Gi + slackChannel: '#knative-serving-ci' diff --git a/pkg/discover/discover.go b/pkg/discover/discover.go new file mode 100644 index 00000000..acf21fce --- /dev/null +++ b/pkg/discover/discover.go @@ -0,0 +1,108 @@ +package discover + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "io/fs" + "log" + "os" + "os/signal" + "path/filepath" + "slices" + + gyaml "github.com/ghodss/yaml" + + "github.com/openshift-knative/hack/pkg/prowgen" +) + +func Main() { + + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, os.Kill) + defer cancel() + + inputConfig := flag.String("config", filepath.Join("config"), "Specify repositories config") + flag.Parse() + + err := filepath.Walk(*inputConfig, func(path string, info fs.FileInfo, err error) error { + if info.IsDir() { + return nil + } + + if err := discover(ctx, path); err != nil { + return fmt.Errorf("failed to discover config for %s: %w", path, err) + } + + return nil + }) + if err != nil { + log.Fatalln("Failed to walk path", *inputConfig, err) + } +} + +func discover(ctx context.Context, path string) error { + // Going directly from YAML raw input produces unexpected configs (due to missing YAML tags), + // so we convert YAML to JSON and unmarshal the struct from the JSON object. + y, err := os.ReadFile(path) + if err != nil { + log.Fatalln(err) + } + j, err := gyaml.YAMLToJSON(y) + if err != nil { + log.Fatalln(err) + } + + inConfig := &prowgen.Config{} + if err := json.Unmarshal(j, inConfig); err != nil { + log.Fatalln("Unmarshal input config", err) + } + + for _, r := range inConfig.Repositories { + if len(inConfig.Config.Branches) == 0 { + continue // nothing to do here + } + + var latest string + if _, ok := inConfig.Config.Branches["release-next"]; ok { + latest = "release-next" + } else { + configuredBranches := make([]string, 0, len(inConfig.Config.Branches)) + for branchName, _ := range inConfig.Config.Branches { + configuredBranches = append(configuredBranches, branchName) + } + slices.SortFunc(configuredBranches, prowgen.CmpBranches) + latest = configuredBranches[len(configuredBranches)-1] + } + + availableBranches, err := prowgen.Branches(ctx, r) + if err != nil { + return err + } + + log.Println(r.RepositoryDirectory(), "Latest branch", latest) + + for i := 0; i < len(availableBranches); i++ { + if latest == availableBranches[i] { + for ; i < len(availableBranches); i++ { + if _, ok := inConfig.Config.Branches[availableBranches[i]]; !ok { + inConfig.Config.Branches[availableBranches[i]] = inConfig.Config.Branches[latest] + } + } + } + } + } + + // Going directly from struct to YAML produces unexpected configs (due to missing YAML tags), + // so we produce JSON and then convert it to YAML. + out, err := json.Marshal(inConfig) + if err != nil { + return err + } + out, err = gyaml.JSONToYAML(out) + if err != nil { + return err + } + + return os.WriteFile(path, out, 0777) +} diff --git a/pkg/project/testoutput/openshift/ci-operator/knative-images/discover/Dockerfile b/pkg/project/testoutput/openshift/ci-operator/knative-images/discover/Dockerfile new file mode 100755 index 00000000..58ad3eab --- /dev/null +++ b/pkg/project/testoutput/openshift/ci-operator/knative-images/discover/Dockerfile @@ -0,0 +1,20 @@ +# DO NOT EDIT! Generated Dockerfile for cmd/discover. +FROM registry.ci.openshift.org/openshift/release:rhel-8-release-golang-1.21-openshift-4.16 as builder + +COPY . . + +RUN mkdir -p /var/run/ko && \ + mkdir -p cmd/discover/kodata && \ + go build -o /usr/bin/main ./cmd/discover && \ + cp -r cmd/discover/kodata /var/run/ko + +FROM registry.access.redhat.com/ubi8/ubi-minimal + +# install the missing zoneinfo to ubi-minimal +RUN microdnf install tzdata + +USER 65532 + +COPY --from=builder /usr/bin/main /usr/bin/main +COPY --from=builder /var/run/ko /var/run/ko +ENTRYPOINT ["/usr/bin/main"] diff --git a/pkg/project/testoutput/openshift/images.yaml b/pkg/project/testoutput/openshift/images.yaml index 16915784..b1a2e6bb 100755 --- a/pkg/project/testoutput/openshift/images.yaml +++ b/pkg/project/testoutput/openshift/images.yaml @@ -1,3 +1,4 @@ +github.com/openshift-knative/hack/cmd/discover: registry.ci.openshift.org/openshift/knative-discover:main github.com/openshift-knative/hack/cmd/generate: hello github.com/openshift-knative/hack/cmd/prowcopy: registry.ci.openshift.org/openshift/knative-prowcopy:main github.com/openshift-knative/hack/cmd/prowgen: registry.ci.openshift.org/openshift/knative-prowgen:main diff --git a/pkg/prowgen/prowgen.go b/pkg/prowgen/prowgen.go index c0ab8ca7..75501690 100644 --- a/pkg/prowgen/prowgen.go +++ b/pkg/prowgen/prowgen.go @@ -30,9 +30,9 @@ import ( // Config is the prowgen configuration file struct. type Config struct { - Repositories []Repository `json:"repositories" yaml:"repositories"` + Repositories []Repository `json:"repositories,omitempty" yaml:"repositories,omitempty"` - Config CommonConfig `json:"config" yaml:"config"` + Config CommonConfig `json:"config,omitempty" yaml:"config,omitempty"` } func Main() { diff --git a/pkg/prowgen/prowgen_config.go b/pkg/prowgen/prowgen_config.go index adc38b6f..a3a65a0e 100644 --- a/pkg/prowgen/prowgen_config.go +++ b/pkg/prowgen/prowgen_config.go @@ -14,50 +14,50 @@ import ( ) type Repository struct { - Org string `json:"org" yaml:"org"` - Repo string `json:"repo" yaml:"repo"` - Promotion Promotion `json:"promotion" yaml:"promotion"` - ImagePrefix string `json:"imagePrefix" yaml:"imagePrefix"` - ImageNameOverrides map[string]string `json:"imageNameOverrides" yaml:"imageNameOverrides"` - SlackChannel string `json:"slackChannel" yaml:"slackChannel"` - CanonicalGoRepository *string `json:"canonicalGoRepository" yaml:"canonicalGoRepository"` - E2ETests []E2ETest `json:"e2e" yaml:"e2e"` - Dockerfiles Dockerfiles `json:"dockerfiles" yaml:"dockerfiles"` - IgnoreConfigs IgnoreConfigs `json:"ignoreConfigs" yaml:"ignoreConfigs"` - CustomConfigs []CustomConfigs `json:"customConfigs" yaml:"customConfigs"` - Images []cioperatorapi.ProjectDirectoryImageBuildStepConfiguration `json:"images" yaml:"images"` - Tests []cioperatorapi.TestStepConfiguration `json:"tests" yaml:"tests"` - Resources cioperatorapi.ResourceConfiguration `json:"resources" yaml:"resources"` + Org string `json:"org,omitempty" yaml:"org,omitempty"` + Repo string `json:"repo,omitempty" yaml:"repo,omitempty"` + Promotion Promotion `json:"promotion,omitempty" yaml:"promotion,omitempty"` + ImagePrefix string `json:"imagePrefix,omitempty" yaml:"imagePrefix,omitempty"` + ImageNameOverrides map[string]string `json:"imageNameOverrides,omitempty" yaml:"imageNameOverrides,omitempty"` + SlackChannel string `json:"slackChannel,omitempty" yaml:"slackChannel,omitempty"` + CanonicalGoRepository *string `json:"canonicalGoRepository,omitempty" yaml:"canonicalGoRepository,omitempty"` + E2ETests []E2ETest `json:"e2e,omitempty" yaml:"e2e,omitempty"` + Dockerfiles Dockerfiles `json:"dockerfiles,omitempty" yaml:"dockerfiles,omitempty"` + IgnoreConfigs IgnoreConfigs `json:"ignoreConfigs,omitempty" yaml:"ignoreConfigs,omitempty"` + CustomConfigs []CustomConfigs `json:"customConfigs,omitempty" yaml:"customConfigs,omitempty"` + Images []cioperatorapi.ProjectDirectoryImageBuildStepConfiguration `json:"images,omitempty" yaml:"images,omitempty"` + Tests []cioperatorapi.TestStepConfiguration `json:"tests,omitempty" yaml:"tests,omitempty"` + Resources cioperatorapi.ResourceConfiguration `json:"resources,omitempty" yaml:"resources,omitempty"` } type E2ETest struct { - Match string `json:"match" yaml:"match"` - OnDemand bool `json:"onDemand" yaml:"onDemand"` - IgnoreError bool `json:"ignoreError" yaml:"ignoreError"` - RunIfChanged string `json:"runIfChanged" yaml:"runIfChanged"` - SkipCron bool `json:"skipCron" yaml:"skipCron"` - SkipImages []string `json:"skipImages" yaml:"skipImages"` - Timeout *prowapi.Duration `json:"timeout" yaml:"timeout"` + Match string `json:"match,omitempty" yaml:"match,omitempty"` + OnDemand bool `json:"onDemand,omitempty" yaml:"onDemand,omitempty"` + IgnoreError bool `json:"ignoreError,omitempty" yaml:"ignoreError,omitempty"` + RunIfChanged string `json:"runIfChanged,omitempty" yaml:"runIfChanged,omitempty"` + SkipCron bool `json:"skipCron,omitempty" yaml:"skipCron,omitempty"` + SkipImages []string `json:"skipImages,omitempty" yaml:"skipImages,omitempty"` + Timeout *prowapi.Duration `json:"timeout,omitempty" yaml:"timeout,omitempty"` } type Dockerfiles struct { - Matches []string `json:"matches" yaml:"matches"` + Matches []string `json:"matches,omitempty" yaml:"matches,omitempty"` } type IgnoreConfigs struct { - Matches []string `json:"matches" yaml:"matches"` + Matches []string `json:"matches,omitempty" yaml:"matches,omitempty"` } type Promotion struct { - Namespace string + Namespace string `json:"namespace,omitempty" yaml:"namespace,omitempty"` } type CustomConfigs struct { // Name will be used together with OpenShift version to generate a specific variant. - Name string `json:"name" yaml:"name"` + Name string `json:"name,omitempty" yaml:"name,omitempty"` // ReleaseBuildConfiguration allows defining configuration manually. The final configuration // is extended with images and test steps with dependencies. - ReleaseBuildConfiguration cioperatorapi.ReleaseBuildConfiguration `json:"releaseBuildConfiguration" yaml:"releaseBuildConfiguration"` + ReleaseBuildConfiguration cioperatorapi.ReleaseBuildConfiguration `json:"releaseBuildConfiguration,omitempty" yaml:"releaseBuildConfiguration,omitempty"` } func (r Repository) RepositoryDirectory() string { @@ -65,21 +65,21 @@ func (r Repository) RepositoryDirectory() string { } type Branch struct { - OpenShiftVersions []OpenShift `json:"openShiftVersions" yaml:"openShiftVersions"` - SkipE2EMatches []string `json:"skipE2EMatches" yaml:"skipE2EMatches"` - SkipDockerFilesMatches []string `json:"skipDockerFilesMatches" yaml:"skipDockerFilesMatches"` + OpenShiftVersions []OpenShift `json:"openShiftVersions,omitempty" yaml:"openShiftVersions,omitempty"` + SkipE2EMatches []string `json:"skipE2EMatches,omitempty" yaml:"skipE2EMatches,omitempty"` + SkipDockerFilesMatches []string `json:"skipDockerFilesMatches,omitempty" yaml:"skipDockerFilesMatches,omitempty"` } type OpenShift struct { - Version string `json:"version" yaml:"version"` - Cron string `json:"cron" yaml:"cron"` - OnDemand bool `json:"onDemand" yaml:"onDemand"` - GenerateCustomConfigs bool `json:"generateCustomConfigs" yaml:"generateCustomConfigs"` - CandidateRelease bool `json:"candidateRelease" yaml:"candidateRelease"` + Version string `json:"version,omitempty" yaml:"version,omitempty"` + Cron string `json:"cron,omitempty" yaml:"cron,omitempty"` + OnDemand bool `json:"onDemand,omitempty" yaml:"onDemand,omitempty"` + GenerateCustomConfigs bool `json:"generateCustomConfigs,omitempty" yaml:"generateCustomConfigs,omitempty"` + CandidateRelease bool `json:"candidateRelease,omitempty" yaml:"candidateRelease,omitempty"` } type CommonConfig struct { - Branches map[string]Branch `json:"branches" yaml:"branches"` + Branches map[string]Branch `json:"branches,omitempty" yaml:"branches,omitempty"` } type ReleaseBuildConfigurationOption func(cfg *cioperatorapi.ReleaseBuildConfiguration) error diff --git a/pkg/prowgen/prowgen_git.go b/pkg/prowgen/prowgen_git.go index b6c8bec2..ca4e439b 100644 --- a/pkg/prowgen/prowgen_git.go +++ b/pkg/prowgen/prowgen_git.go @@ -7,7 +7,10 @@ import ( "log" "os" "path/filepath" + "slices" "strings" + + "github.com/coreos/go-semver/semver" ) func GitCheckout(ctx context.Context, r Repository, branch string) error { @@ -23,6 +26,52 @@ func GitClone(ctx context.Context, r Repository) error { return gitClone(ctx, r, false) } +func Branches(ctx context.Context, r Repository) ([]string, error) { + if err := GitMirror(ctx, r); err != nil { + return nil, err + } + + // git --no-pager branch --list "release-v*" + branchesBytes, err := run(ctx, r, "git", "--no-pager", "branch", "--list", "release-v*") + if err != nil { + return nil, err + } + + branchesList := string(branchesBytes) + + sortedBranches := strings.Split(branchesList, "\n") + for i, b := range sortedBranches { + b = strings.TrimSpace(b) + sortedBranches[i] = b + } + slices.SortFunc(sortedBranches, CmpBranches) + + log.Println("Branches for", r.RepositoryDirectory(), sortedBranches) + + return sortedBranches, nil +} + +func CmpBranches(a string, b string) int { + a = strings.ReplaceAll(a, "release-v", "") + b = strings.ReplaceAll(b, "release-v", "") + if strings.Count(a, ".") == 1 { + a += ".0" + } + if strings.Count(b, ".") == 1 { + b += ".0" + } + av, err := semver.NewVersion(a) + if err != nil { + return -1 // this is equivalent to ignoring branches that aren't parseable + } + bv, err := semver.NewVersion(b) + if err != nil { + return 1 // this is equivalent to ignoring branches that aren't parseable + } + + return av.Compare(*bv) +} + func gitClone(ctx context.Context, r Repository, mirror bool) error { select { case <-ctx.Done():