From 9c923e34cea68c9171daa5bf63cd6805a208cac0 Mon Sep 17 00:00:00 2001 From: Emilio Date: Mon, 11 Nov 2024 12:29:28 -0500 Subject: [PATCH] GEOMESA-3049 Add scaladocs to CI build (#3231) * Fix overwrite of scaladocs by javadocs in mixed scala/java modules * Clean up github workflows --- .github/workflows/build-and-test-2.12.yml | 19 -- .github/workflows/build-and-test-2.13.yml | 19 -- .github/workflows/build-and-test.yml | 168 ++++++------------ .github/workflows/codacy.yml | 3 +- .../workflows/{eclipse-dash.yml => dash.yml} | 38 ++-- .github/workflows/dependency-review.yml | 2 +- .github/workflows/integration-tests.yml | 7 +- .github/workflows/javadocs.yml | 47 +++++ .github/workflows/license-check.yml | 33 ---- .../{build-docs.yml => main-build.yml} | 26 ++- .github/workflows/scorecard.yml | 2 +- .github/workflows/unit-tests.yml | 13 +- geomesa-arrow/geomesa-arrow-jts/pom.xml | 23 +++ pom.xml | 12 -- 14 files changed, 190 insertions(+), 222 deletions(-) delete mode 100644 .github/workflows/build-and-test-2.12.yml delete mode 100644 .github/workflows/build-and-test-2.13.yml rename .github/workflows/{eclipse-dash.yml => dash.yml} (54%) create mode 100644 .github/workflows/javadocs.yml delete mode 100644 .github/workflows/license-check.yml rename .github/workflows/{build-docs.yml => main-build.yml} (61%) diff --git a/.github/workflows/build-and-test-2.12.yml b/.github/workflows/build-and-test-2.12.yml deleted file mode 100644 index 2b48068918a3..000000000000 --- a/.github/workflows/build-and-test-2.12.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: scala-2.12 - -on: - push: - branches: - - main - - '[0-9]+.[0-9]+.x' - tags: - - '**' - pull_request: - -permissions: # added using https://github.com/step-security/secure-repo - contents: read - -jobs: - build-and-test: - uses: ./.github/workflows/build-and-test.yml - with: - scala-version: "2.12" diff --git a/.github/workflows/build-and-test-2.13.yml b/.github/workflows/build-and-test-2.13.yml deleted file mode 100644 index 754203575afe..000000000000 --- a/.github/workflows/build-and-test-2.13.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: scala-2.13 - -on: - push: - branches: - - main - - '[0-9]+.[0-9]+.x' - tags: - - '**' - pull_request: - -permissions: # added using https://github.com/step-security/secure-repo - contents: read - -jobs: - build-and-test: - uses: ./.github/workflows/build-and-test.yml - with: - scala-version: "2.13" diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index c7fadbbdd703..0596c03595bd 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -13,10 +13,9 @@ permissions: # added using https://github.com/step-security/secure-repo env: MAVEN_CLI_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dlicense.skip=true --batch-mode - MAVEN_TEST_OPTS: -Dtest.fork.count=1 jobs: - build-scala: + compile: runs-on: ubuntu-latest steps: - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 @@ -41,7 +40,7 @@ jobs: - name: Upload artifacts uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 with: - name: classes-and-jars + name: classes-and-jars-${{ inputs.scala-version }} retention-days: 1 path: | m2-geomesa @@ -49,116 +48,65 @@ jobs: - name: Remove geomesa artifacts if: success() || failure() run: rm -rf ~/.m2/repository/org/locationtech/geomesa - accumulo-ds-tests: - needs: build-scala - uses: ./.github/workflows/unit-tests.yml - with: - module-selector: -pl :geomesa-accumulo-datastore_${{ inputs.scala-version }} - scala-version: ${{ inputs.scala-version }} - accumulo-other-tests: - needs: build-scala - uses: ./.github/workflows/unit-tests.yml - with: - module-selector: -f geomesa-accumulo -pl -:geomesa-accumulo-datastore_${{ inputs.scala-version }} - scala-version: ${{ inputs.scala-version }} - arrow-tests: - needs: build-scala - uses: ./.github/workflows/unit-tests.yml - with: - module-selector: -f geomesa-arrow - scala-version: ${{ inputs.scala-version }} - cassandra-tests: - needs: build-scala - uses: ./.github/workflows/unit-tests.yml - with: - module-selector: -f geomesa-cassandra - scala-version: ${{ inputs.scala-version }} - convert-tests: - needs: build-scala - uses: ./.github/workflows/unit-tests.yml - with: - module-selector: -f geomesa-convert - scala-version: ${{ inputs.scala-version }} - cqengine-tests: - needs: build-scala - uses: ./.github/workflows/unit-tests.yml - with: - module-selector: -f geomesa-memory - scala-version: ${{ inputs.scala-version }} - features-tests: - needs: build-scala - uses: ./.github/workflows/unit-tests.yml - with: - module-selector: -f geomesa-features - scala-version: ${{ inputs.scala-version }} - fs-tests: - needs: build-scala - uses: ./.github/workflows/unit-tests.yml - with: - module-selector: -f geomesa-fs - scala-version: ${{ inputs.scala-version }} - gt-tests: - needs: build-scala - uses: ./.github/workflows/unit-tests.yml - with: - module-selector: -f geomesa-gt - scala-version: ${{ inputs.scala-version }} - hbase-tests: - needs: build-scala - uses: ./.github/workflows/unit-tests.yml - with: - module-selector: -f geomesa-hbase - scala-version: ${{ inputs.scala-version }} - kafka-tests: - needs: build-scala - uses: ./.github/workflows/unit-tests.yml - with: - module-selector: -f geomesa-kafka - scala-version: ${{ inputs.scala-version }} - lambda-tests: - needs: build-scala - uses: ./.github/workflows/unit-tests.yml - with: - module-selector: -f geomesa-lambda - scala-version: ${{ inputs.scala-version }} - metrics-tests: - needs: build-scala - uses: ./.github/workflows/unit-tests.yml - with: - module-selector: -f geomesa-metrics - scala-version: ${{ inputs.scala-version }} - process-tests: - needs: build-scala - uses: ./.github/workflows/unit-tests.yml - with: - module-selector: -f geomesa-process + unit-tests: + name: unit-tests (${{ matrix.tests.name }}) + strategy: + fail-fast: false + matrix: + tests: + - name: accumulo-ds + selector: -pl :geomesa-accumulo-datastore_${{ inputs.scala-version }} + - name: accumulo-other + selector: -f geomesa-accumulo -pl -:geomesa-accumulo-datastore_${{ inputs.scala-version }} + - name: arrow + selector: -f geomesa-arrow + - name: cassandra + selector: -f geomesa-cassandra + - name: convert + selector: -f geomesa-convert + - name: cqengine + selector: -f geomesa-memory + - name: features + selector: -f geomesa-features + - name: fs + selector: -f geomesa-fs + - name: gt + selector: -f geomesa-gt + - name: hbase + selector: -f geomesa-hbase + - name: kafka + selector: -f geomesa-kafka + - name: lambda + selector: -f geomesa-lambda + - name: metrics + selector: -f geomesa-metrics + - name: process + selector: -f geomesa-process + - name: redis + selector: -f geomesa-redis + - name: spark + selector: -f geomesa-spark + - name: utils + selector: -f geomesa-utils-parent + - name: others + selector: -pl geomesa-filter,geomesa-index-api,geomesa-jobs,geomesa-security,geomesa-tools,geomesa-z3 + needs: compile + uses: ./.github/workflows/unit-tests.yml + with: + module-selector: ${{ matrix.tests.selector }} scala-version: ${{ inputs.scala-version }} - redis-tests: - needs: build-scala - uses: ./.github/workflows/unit-tests.yml - with: - module-selector: -f geomesa-redis - scala-version: ${{ inputs.scala-version }} - spark-tests: - needs: build-scala - uses: ./.github/workflows/unit-tests.yml - with: - module-selector: -f geomesa-spark - scala-version: ${{ inputs.scala-version }} - utils-tests: - needs: build-scala - uses: ./.github/workflows/unit-tests.yml + integration-tests: + needs: compile + uses: ./.github/workflows/integration-tests.yml with: - module-selector: -f geomesa-utils-parent scala-version: ${{ inputs.scala-version }} - other-tests: - needs: build-scala - uses: ./.github/workflows/unit-tests.yml + javadocs: + needs: compile + uses: ./.github/workflows/javadocs.yml with: - module-selector: -pl geomesa-filter,geomesa-index-api,geomesa-jobs,geomesa-security,geomesa-tools,geomesa-z3 scala-version: ${{ inputs.scala-version }} - integration-tests: - needs: build-scala - uses: ./.github/workflows/integration-tests.yml + dash: + needs: compile + uses: ./.github/workflows/dash.yml with: - scala-version: ${{ inputs.scala-version }} + scala-version: ${{ inputs.scala-version }} \ No newline at end of file diff --git a/.github/workflows/codacy.yml b/.github/workflows/codacy.yml index 4721325b7ab8..77a32180584f 100644 --- a/.github/workflows/codacy.yml +++ b/.github/workflows/codacy.yml @@ -11,7 +11,7 @@ # For more information on Codacy Analysis CLI in general, see # https://github.com/codacy/codacy-analysis-cli. -name: Codacy Security Scan +name: codacy on: push: @@ -32,7 +32,6 @@ jobs: permissions: contents: read # for actions/checkout to fetch code security-events: write # for github/codeql-action/upload-sarif to upload SARIF results - name: Codacy Security Scan runs-on: ubuntu-latest strategy: fail-fast: false diff --git a/.github/workflows/eclipse-dash.yml b/.github/workflows/dash.yml similarity index 54% rename from .github/workflows/eclipse-dash.yml rename to .github/workflows/dash.yml index 48dd5d9129f8..a8e006f6ef53 100644 --- a/.github/workflows/eclipse-dash.yml +++ b/.github/workflows/dash.yml @@ -1,22 +1,21 @@ -name: eclipse-dash +name: dash on: - push: - branches: - - main - - '[0-9]+.[0-9]+.x' - tags: - - '**' - pull_request: + workflow_call: + inputs: + scala-version: + required: false + type: string + default: "2.12" permissions: # added using https://github.com/step-security/secure-repo contents: read env: - MAVEN_CLI_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false --batch-mode + MAVEN_CLI_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dlicense.skip=true --batch-mode jobs: - license-check: + dependency-check: runs-on: ubuntu-latest steps: - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 @@ -26,12 +25,19 @@ jobs: java-version: '11' - uses: actions/cache@3624ceb22c1c5a301c8db4169662070a689d9ea8 # v4.1.1 with: - key: ${{ hashFiles('**/pom.xml') }}-dash - path: | - ~/.m2/repository/ - build/zinc-0.3.15/ - - name: Build with Maven - run: ./build/mvn clean install $MAVEN_CLI_OPTS -DskipTests -T4 + key: ${{ hashFiles('**/pom.xml') }}-dash-${{ inputs.scala-version }} + path: ~/.m2/repository/ + - name: Set Scala version + run: ./build/scripts/change-scala-version.sh ${{ inputs.scala-version }} + - name: Download artifacts + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: classes-and-jars-${{ inputs.scala-version }} + - name: Install artifacts + run: | + tar -xf classes.tgz + mkdir -p ~/.m2/repository/org/locationtech/geomesa + mv m2-geomesa/* ~/.m2/repository/org/locationtech/geomesa/ - name: License check run: mvn org.eclipse.dash:license-tool-plugin:license-check $MAVEN_CLI_OPTS -Pdash -Ddash.fail=true -Ddash.projectId=locationtech.geomesa -DexcludeGroupIds=org.locationtech.geomesa - name: Show license diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index 448b1bd14479..bfcbcb9c17d0 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -6,7 +6,7 @@ # PRs introducing known-vulnerable packages will be blocked from merging. # # Source repository: https://github.com/actions/dependency-review-action -name: 'Dependency Review' +name: dependency-review on: [pull_request] permissions: diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 6ee7fa5bb61d..865c99b1b1a6 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -11,6 +11,9 @@ on: permissions: # added using https://github.com/step-security/secure-repo contents: read +env: + MAVEN_CLI_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dlicense.skip=true -Dtest.fork.count=1 --batch-mode + jobs: integration-tests: runs-on: ubuntu-latest @@ -29,14 +32,14 @@ jobs: - name: Download artifacts uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 with: - name: classes-and-jars + name: classes-and-jars-${{ inputs.scala-version }} - name: Install artifacts run: | tar -xf classes.tgz mkdir -p ~/.m2/repository/org/locationtech/geomesa mv m2-geomesa/* ~/.m2/repository/org/locationtech/geomesa/ - name: Integration Tests - run: mvn failsafe:integration-test failsafe:verify $MAVEN_CLI_OPTS $MAVEN_TEST_OPTS + run: mvn failsafe:integration-test failsafe:verify $MAVEN_CLI_OPTS - name: Remove geomesa artifacts if: success() || failure() run: rm -rf ~/.m2/repository/org/locationtech/geomesa diff --git a/.github/workflows/javadocs.yml b/.github/workflows/javadocs.yml new file mode 100644 index 000000000000..b558cb65b663 --- /dev/null +++ b/.github/workflows/javadocs.yml @@ -0,0 +1,47 @@ +name: javadocs + +on: + workflow_call: + inputs: + scala-version: + required: false + type: string + default: "2.12" + +permissions: # added using https://github.com/step-security/secure-repo + contents: read + +env: + MAVEN_CLI_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dlicense.skip=true --batch-mode + +jobs: + build-javadocs: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 + - uses: actions/setup-java@b36c23c0d998641eff861008f374ee103c25ac73 # v4.4.0 + with: + distribution: 'temurin' + java-version: '11' + - uses: actions/cache@3624ceb22c1c5a301c8db4169662070a689d9ea8 # v4.1.1 + with: + key: ${{ hashFiles('**/pom.xml') }}-javadocs-${{ inputs.scala-version }} + path: ~/.m2/repository/ + - name: Set Scala version + run: ./build/scripts/change-scala-version.sh ${{ inputs.scala-version }} + - name: Download artifacts + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: classes-and-jars-${{ inputs.scala-version }} + - name: Install artifacts + run: | + tar -xf classes.tgz + mkdir -p ~/.m2/repository/org/locationtech/geomesa + mv m2-geomesa/* ~/.m2/repository/org/locationtech/geomesa/ + - name: Build scaladocs + run: mvn generate-sources scala:doc-jar $MAVEN_CLI_OPTS + - name: Build javadocs + run: mvn generate-sources javadoc:jar $MAVEN_CLI_OPTS -pl :geomesa-arrow-jts + - name: Remove geomesa artifacts + if: success() || failure() + run: rm -rf ~/.m2/repository/org/locationtech/geomesa diff --git a/.github/workflows/license-check.yml b/.github/workflows/license-check.yml deleted file mode 100644 index f2498ceffcfa..000000000000 --- a/.github/workflows/license-check.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: license-check - -on: - push: - branches: - - main - - '[0-9]+.[0-9]+.x' - tags: - - '**' - pull_request: - -permissions: # added using https://github.com/step-security/secure-repo - contents: read - -env: - MAVEN_CLI_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false --batch-mode - MAVEN_TEST_OPTS: -Dtest.fork.count=1 - -jobs: - check-headers: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 - - uses: actions/setup-java@b36c23c0d998641eff861008f374ee103c25ac73 # v4.4.0 - with: - distribution: 'temurin' - java-version: '11' - - uses: actions/cache@3624ceb22c1c5a301c8db4169662070a689d9ea8 # v4.1.1 - with: - key: ${{ hashFiles('**/pom.xml') }}-license-header-check - path: ~/.m2/repository/ - - name: Run license check - run: mvn license:check $MAVEN_CLI_OPTS diff --git a/.github/workflows/build-docs.yml b/.github/workflows/main-build.yml similarity index 61% rename from .github/workflows/build-docs.yml rename to .github/workflows/main-build.yml index e5c48422ee6d..540992f31aae 100644 --- a/.github/workflows/build-docs.yml +++ b/.github/workflows/main-build.yml @@ -1,4 +1,4 @@ -name: build-docs +name: build on: push: @@ -13,9 +13,17 @@ permissions: # added using https://github.com/step-security/secure-repo contents: read env: - MAVEN_CLI_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dlicense.skip=true --batch-mode + MAVEN_CLI_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false --batch-mode jobs: + build-scala: + strategy: + fail-fast: false + matrix: + version: [ "2.12", "2.13" ] + uses: ./.github/workflows/build-and-test.yml + with: + scala-version: ${{ matrix.version }} build-docs: runs-on: ubuntu-latest steps: @@ -40,3 +48,17 @@ jobs: - name: Remove geomesa artifacts if: success() || failure() run: rm -rf ~/.m2/repository/org/locationtech/geomesa + check-license-headers: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 + - uses: actions/setup-java@b36c23c0d998641eff861008f374ee103c25ac73 # v4.4.0 + with: + distribution: 'temurin' + java-version: '11' + - uses: actions/cache@3624ceb22c1c5a301c8db4169662070a689d9ea8 # v4.1.1 + with: + key: ${{ hashFiles('**/pom.xml') }}-license-header-check + path: ~/.m2/repository/ + - name: Run license check + run: mvn license:check $MAVEN_CLI_OPTS diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 3f8f643a7c7f..e223015c7c63 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -2,7 +2,7 @@ # by a third-party and are governed by separate terms of service, privacy # policy, and support documentation. -name: Scorecard supply-chain security +name: scorecard-security on: # For Branch-Protection check. Only the default branch is supported. See # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index ebfa58c18134..2d42ca8e2f1e 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -14,8 +14,11 @@ on: permissions: # added using https://github.com/step-security/secure-repo contents: read +env: + MAVEN_CLI_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dlicense.skip=true -Dtest.fork.count=1 --batch-mode + jobs: - unit-tests: + test: runs-on: ubuntu-latest steps: - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 @@ -36,7 +39,7 @@ jobs: - name: Download artifacts uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 with: - name: classes-and-jars + name: classes-and-jars-${{ inputs.scala-version }} - name: Install artifacts run: | tar -xf classes.tgz @@ -47,7 +50,7 @@ jobs: continue-on-error: true run: | set -o pipefail - mvn surefire:test $MAVEN_CLI_OPTS $MAVEN_TEST_OPTS ${{ inputs.module-selector }} | tee -a test.log + mvn surefire:test $MAVEN_CLI_OPTS ${{ inputs.module-selector }} | tee -a test.log - name: Unit tests (retry) id: test-retry if: steps.test.outcome=='failure' @@ -55,14 +58,14 @@ jobs: run: | set -o pipefail RESUME_FROM="$({ grep --text 'mvn -rf ' test.log || test $? = 1; } | tail -n1 | sed 's/.*-rf/-rf/')" - mvn surefire:test $MAVEN_CLI_OPTS $MAVEN_TEST_OPTS $RESUME_FROM ${{ inputs.module-selector }} | tee -a test.log + mvn surefire:test $MAVEN_CLI_OPTS $RESUME_FROM ${{ inputs.module-selector }} | tee -a test.log - name: Unit tests (retry) id: test-retry-retry if: steps.test-retry.outcome=='failure' run: | set -o pipefail RESUME_FROM="$({ grep --text 'mvn -rf ' test.log || test $? = 1; } | tail -n1 | sed 's/.*-rf/-rf/')" - mvn surefire:test $MAVEN_CLI_OPTS $MAVEN_TEST_OPTS $RESUME_FROM ${{ inputs.module-selector }} | tee -a test.log + mvn surefire:test $MAVEN_CLI_OPTS $RESUME_FROM ${{ inputs.module-selector }} | tee -a test.log - name: Remove geomesa artifacts if: success() || failure() run: rm -rf ~/.m2/repository/org/locationtech/geomesa diff --git a/geomesa-arrow/geomesa-arrow-jts/pom.xml b/geomesa-arrow/geomesa-arrow-jts/pom.xml index 3a4b7f6d2622..29d975ebdab6 100644 --- a/geomesa-arrow/geomesa-arrow-jts/pom.xml +++ b/geomesa-arrow/geomesa-arrow-jts/pom.xml @@ -55,4 +55,27 @@ + + + central + + + + + org.apache.maven.plugins + maven-javadoc-plugin + + + attach-javadocs + + jar + + + + + + + + + diff --git a/pom.xml b/pom.xml index 99d84f49ee99..79f4a4de0739 100644 --- a/pom.xml +++ b/pom.xml @@ -282,18 +282,6 @@ - - org.apache.maven.plugins - maven-javadoc-plugin - - - attach-javadocs - - jar - - - - org.apache.maven.plugins maven-deploy-plugin