diff --git a/.github/workflows/documentbot.yml b/.github/workflows/documentbot.yml index 9ea3814..8a94526 100644 --- a/.github/workflows/documentbot.yml +++ b/.github/workflows/documentbot.yml @@ -1,4 +1,3 @@ - # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file @@ -21,13 +20,13 @@ name: Auto Labeling on: - pull_request_target : + pull_request_target: types: - - opened - - edited - - labeled + - opened + - edited + - labeled + - # A GitHub token created for a PR coming from a fork doesn't have # 'admin' or 'write' permission (which is required to add labels) @@ -39,13 +38,20 @@ jobs: labeling: if: ${{ github.repository == 'streamnative/pulsar-io-template' }} permissions: - pull-requests: write + pull-requests: write runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - - uses: streamnative/github-workflow-libraries/doc-label-check@master - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - label-pattern: '- \[(.*?)\] ?`(.+?)`' # matches '- [x] `label`' + - uses: s4u/maven-settings-action@v2.6.0 + with: + servers: '[{"id": "ossrh", "username": "${{ secrets.SONATYPE_USERNAME }}", + "password": "${{ secrets.SONATYPE_PASSWORD }}"}]' + - name: Login to cloudsmith + run: docker login -u="${{ secrets.CLOUDSMITH_USERNAME }}" -p="${{ secrets.CLOUDSMITH_API_KEY + }}" docker.cloudsmith.io + - uses: actions/checkout@v2 + + - uses: streamnative/github-workflow-libraries/doc-label-check@master + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + label-pattern: '- \[(.*?)\] ?`(.+?)`' # matches '- [x] `label`' diff --git a/.github/workflows/pr-unit-tests.yml b/.github/workflows/pr-unit-tests.yml index c3079ba..13e14f0 100644 --- a/.github/workflows/pr-unit-tests.yml +++ b/.github/workflows/pr-unit-tests.yml @@ -3,12 +3,12 @@ name: lakehouse mvn build check and unit tests on: pull_request: branches: - - master - - branch-* + - master + - branch-* push: branches: - - master - - branch-* + - master + - branch-* jobs: build: @@ -16,44 +16,51 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 - - name: Set up JDK 1.8 - uses: actions/setup-java@v1 - with: - java-version: 1.8 - - - name: License check - run: mvn -ntp -B license:check - - - name: Build cloud package with Maven skipTests - run: mvn clean install -ntp -B -DskipTests -P cloud - - - name: Build with Maven skipTests - run: mvn clean install -ntp -B -DskipTests - - - name: Style check - run: mvn -ntp -B checkstyle:check - - - name: Spotbugs check - run: mvn -ntp -B spotbugs:check - - - name: unit test after build - env: - CLOUD_BUCKET_NAME: ${{ secrets.CLOUD_BUCKET_NAME }} - AWS_SECRET_KEY: ${{ secrets.AWS_SECRET_KEY }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - run: mvn test -Pcloud - - - name: package surefire artifacts - if: failure() - run: | - rm -rf artifacts - mkdir artifacts - find . -type d -name "*surefire*" -exec cp --parents -R {} artifacts/ \; - zip -r artifacts.zip artifacts - - uses: actions/upload-artifact@master - name: upload surefire-artifacts - if: failure() - with: - name: surefire-artifacts - path: artifacts.zip + - uses: s4u/maven-settings-action@v2.6.0 + with: + servers: '[{"id": "ossrh", "username": "${{ secrets.SONATYPE_USERNAME }}", + "password": "${{ secrets.SONATYPE_PASSWORD }}"}]' + - name: Login to cloudsmith + run: docker login -u="${{ secrets.CLOUDSMITH_USERNAME }}" -p="${{ secrets.CLOUDSMITH_API_KEY + }}" docker.cloudsmith.io + - uses: actions/checkout@v1 + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + with: + java-version: 1.8 + + - name: License check + run: mvn -ntp -B license:check + + - name: Build cloud package with Maven skipTests + run: mvn clean install -ntp -B -DskipTests -P cloud + + - name: Build with Maven skipTests + run: mvn clean install -ntp -B -DskipTests + + - name: Style check + run: mvn -ntp -B checkstyle:check + + - name: Spotbugs check + run: mvn -ntp -B spotbugs:check + + - name: unit test after build + env: + CLOUD_BUCKET_NAME: ${{ secrets.CLOUD_BUCKET_NAME }} + AWS_SECRET_KEY: ${{ secrets.AWS_SECRET_KEY }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + run: mvn test -Pcloud + + - name: package surefire artifacts + if: failure() + run: | + rm -rf artifacts + mkdir artifacts + find . -type d -name "*surefire*" -exec cp --parents -R {} artifacts/ \; + zip -r artifacts.zip artifacts + - uses: actions/upload-artifact@master + name: upload surefire-artifacts + if: failure() + with: + name: surefire-artifacts + path: artifacts.zip diff --git a/docs/lakehouse-sink.md b/docs/lakehouse-sink.md index 2245e87..27c09dd 100644 --- a/docs/lakehouse-sink.md +++ b/docs/lakehouse-sink.md @@ -42,7 +42,7 @@ To build the Lakehouse sink connector from the source code, follow these steps. ```bash ls target - pulsar-io-lakehouse-{{connector:version}}.nar + pulsar-io-lakehouse-2.9.3.14.nar ``` # How to configure @@ -133,7 +133,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "inputs": [ "test-hudi-pulsar" ], - "archive": "connectors/pulsar-io-hudi-{{connector:version}}.nar", + "archive": "connectors/pulsar-io-hudi-2.9.3.14.nar", "parallelism": 1, "configs": { "type": "hudi", @@ -156,7 +156,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "inputs": [ "test-hudi-pulsar" ], - "archive": "connectors/pulsar-io-hudi-{{connector:version}}.nar", + "archive": "connectors/pulsar-io-hudi-2.9.3.14.nar", "parallelism": 1, "configs": { "type": "hudi", @@ -185,7 +185,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "inputs": [ "test-iceberg-pulsar" ], - "archive": "connectors/pulsar-io-lakehouse-{{connector:version}}.nar", + "archive": "connectors/pulsar-io-lakehouse-2.9.3.14.nar", "processingGuarantees":"EFFECTIVELY_ONCE", "configs":{ "type":"iceberg", @@ -213,7 +213,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "inputs": [ "test-iceberg-pulsar" ], - "archive": "connectors/pulsar-io-lakehouse-{{connector:version}}.nar", + "archive": "connectors/pulsar-io-lakehouse-2.9.3.14.nar", "processingGuarantees":"EFFECTIVELY_ONCE", "configs":{ "type":"iceberg", @@ -246,7 +246,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "inputs": [ "test-delta-pulsar" ], - "archive": "connectors/pulsar-io-lakehouse-{{connector:version}}.nar", + "archive": "connectors/pulsar-io-lakehouse-2.9.3.14.nar", "processingGuarantees":"EFFECTIVELY_ONCE", "configs":{ "type":"delta", @@ -268,7 +268,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "inputs": [ "test-delta-pulsar" ], - "archive": "connectors/pulsar-io-lakehouse-{{connector:version}}.nar", + "archive": "connectors/pulsar-io-lakehouse-2.9.3.14.nar", "processingGuarantees":"EFFECTIVELY_ONCE", "configs":{ "type":"delta", @@ -339,7 +339,7 @@ This example describes how to use the Lakehouse sink connector to fetch data fro 1. Copy the NAR package to the Pulsar connectors directory. ``` - cp pulsar-io-lakehouse-{{connector:version}}.nar PULSAR_HOME/connectors/pulsar-io-lakehouse-{{connector:version}}.nar + cp pulsar-io-lakehouse-2.9.3.14.nar PULSAR_HOME/connectors/pulsar-io-lakehouse-2.9.3.14.nar ``` 2. Start Pulsar in standalone mode. @@ -372,7 +372,7 @@ This example explains how to create a Lakehouse sink connector in an on-premises 1. Copy the NAR package of the Lakehouse sink connector to the Pulsar connectors directory. ```bash - cp pulsar-io-lakehouse-{{connector:version}}.nar $PULSAR_HOME/connectors/pulsar-io-lakehouse-{{connector:version}}.nar + cp pulsar-io-lakehouse-2.9.3.14.nar $PULSAR_HOME/connectors/pulsar-io-lakehouse-2.9.3.14.nar ``` 2. Reload all [built-in connectors](https://pulsar.apache.org/docs/en/next/io-connectors/). diff --git a/docs/lakehouse-source.md b/docs/lakehouse-source.md index 8195c07..bbbf19b 100644 --- a/docs/lakehouse-source.md +++ b/docs/lakehouse-source.md @@ -42,7 +42,7 @@ To build the Lakehouse source connector from the source code, follow these steps ```bash ls target - pulsar-io-lakehouse-{{connector:version}}.nar + pulsar-io-lakehouse-2.9.3.14.nar ``` # How to configure @@ -92,7 +92,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "parallelism":1, "topicName": "delta_source", "processingGuarantees":"ATLEAST_ONCE", - "archive": "connectors/pulsar-io-lakehouse-{{connector:version}}.nar", + "archive": "connectors/pulsar-io-lakehouse-2.9.3.14.nar", "configs":{ "type":"delta", "checkpointInterval": 180, @@ -117,7 +117,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "parallelism":1, "topicName": "delta_source", "processingGuarantees":"ATLEAST_ONCE", - "archive": "connectors/pulsar-io-lakehouse-{{connector:version}}.nar", + "archive": "connectors/pulsar-io-lakehouse-2.9.3.14.nar", "configs":{ "type":"delta", "checkpointInterval": 180, @@ -178,7 +178,7 @@ This example describes how to use the Lakehouse source connector to fetch data f 1. Copy the NAR package to the Pulsar connectors directory. ``` - cp pulsar-io-lakehouse-{{connector:version}}.nar PULSAR_HOME/connectors/pulsar-io-lakehouse-{{connector:version}}.nar + cp pulsar-io-lakehouse-2.9.3.14.nar PULSAR_HOME/connectors/pulsar-io-lakehouse-2.9.3.14.nar ``` 2. Start Pulsar in standalone mode. @@ -209,7 +209,7 @@ This example explains how to create a Lakehouse source connector in an on-premis 1. Copy the NAR package of the Lakehouse source connector to the Pulsar connectors directory. ``` - cp pulsar-io-lakehouse-{{connector:version}}.nar $PULSAR_HOME/connectors/pulsar-io-lakehouse-{{connector:version}}.nar + cp pulsar-io-lakehouse-2.9.3.14.nar $PULSAR_HOME/connectors/pulsar-io-lakehouse-2.9.3.14.nar ``` 2. Reload all [built-in connectors](https://pulsar.apache.org/docs/en/next/io-connectors/). diff --git a/pom.xml b/pom.xml index 50e1cf7..49a4083 100644 --- a/pom.xml +++ b/pom.xml @@ -1,4 +1,4 @@ - + - + org.apache apache 18 - 4.0.0 - org.apache.pulsar.ecosystem pulsar-io-lakehouse - 2.9.2 + 2.9.3.14 Pulsar Ecosystem :: IO Connector :: Lake House It is a lakehouse connector streaming convert data between lakehouse and Apache Pulsar. - 1.8 1.8 @@ -45,11 +40,10 @@ true 3.1.8 0 - 2.13.2.1 1.18.22 - 2.9.2.17 + 2.9.3.14 2.17.2 1.7.25 3.2.2 @@ -62,12 +56,10 @@ 1.12.220 hadoop3-2.2.1 2.12.0 - 7.3.0 4.0.3 3.12.4 - 3.0 3.1.2 @@ -76,9 +68,8 @@ 1.2.0 8.37 3.1.8 - + - Apache License, Version 2.0 @@ -86,7 +77,6 @@ repo - @@ -126,7 +116,6 @@ netty-common ${netty.version} - org.apache.iceberg @@ -158,20 +147,17 @@ iceberg-hive-metastore ${iceberg.version} - org.apache.parquet parquet-avro ${parquet.avro.version} - io.delta delta-standalone_2.12 ${delta.version} - org.apache.parquet parquet-column @@ -182,7 +168,6 @@ parquet-hadoop ${parquet.version} - org.slf4j slf4j-api @@ -203,7 +188,6 @@ log4j-core ${log4j2.version} - org.apache.hudi @@ -211,7 +195,6 @@ ${hudi.version} - org.apache.hadoop @@ -229,23 +212,18 @@ aws-java-sdk-bundle ${aws.sdk.version} - com.google.cloud.bigdataoss gcs-connector ${gcs.version} shaded - org.apache.hadoop hadoop-azure ${hadoop.version} - - - org.testng @@ -271,13 +249,10 @@ ${mockito.version} test - - - org.projectlombok @@ -289,13 +264,11 @@ spotbugs-annotations provided - io.streamnative pulsar-io-core - com.fasterxml.jackson.core jackson-databind @@ -331,37 +304,30 @@ - org.apache.iceberg iceberg-api - org.apache.iceberg iceberg-core - org.apache.iceberg iceberg-parquet - org.apache.iceberg iceberg-common - org.apache.iceberg iceberg-data - org.apache.iceberg iceberg-hive-metastore - io.delta delta-standalone_2.12 @@ -370,7 +336,6 @@ org.apache.parquet parquet-avro - org.apache.parquet parquet-column @@ -379,7 +344,6 @@ org.apache.parquet parquet-hadoop - org.slf4j slf4j-api @@ -388,7 +352,6 @@ org.slf4j slf4j-simple - org.apache.logging.log4j log4j-slf4j-impl @@ -397,12 +360,10 @@ org.apache.logging.log4j log4j-core - io.netty netty-common - org.apache.hudi hudi-java-client @@ -421,13 +382,11 @@ - org.apache.curator curator-framework ${curator.version} - org.apache.hudi hudi-client-common @@ -435,7 +394,6 @@ test tests - org.testng @@ -452,9 +410,7 @@ mockito-core test - - cloud @@ -478,13 +434,11 @@ com.amazonaws aws-java-sdk-bundle - com.google.cloud.bigdataoss gcs-connector shaded - org.apache.hadoop hadoop-azure @@ -492,7 +446,6 @@ - ${project.artifactId}-${project.version}${cloudPkgSuffix} @@ -509,7 +462,7 @@ -Xlint:unchecked -Xpkginfo:always - + @@ -621,38 +574,32 @@ - - + org.apache.maven.plugins maven-compiler-plugin - org.apache.maven.plugins maven-surefire-plugin - org.apache.nifi nifi-nar-maven-plugin - com.mycila license-maven-plugin - org.apache.maven.plugins maven-checkstyle-plugin - com.github.spotbugs @@ -660,5 +607,20 @@ - + + + ossrh + https://s01.oss.sonatype.org/service/local/repositories/iostreamnative-1674/content + + + + + ossrh + https://s01.oss.sonatype.org/content/repositories/snapshots + + + ossrh + https://s01.oss.sonatype.org/service/local/staging/deploy/maven2/ + +