From 014d8eedbac9e8fc65c18f303a1b19068e4425fc Mon Sep 17 00:00:00 2001 From: streamnativebot Date: Mon, 11 Nov 2024 04:50:07 +0000 Subject: [PATCH] "Created by snbot" --- docs/lakehouse-sink.md | 18 ++++---- docs/lakehouse-source.md | 10 ++--- image/Dockerfile | 2 +- pom.xml | 92 ++++++++++++---------------------------- 4 files changed, 43 insertions(+), 79 deletions(-) diff --git a/docs/lakehouse-sink.md b/docs/lakehouse-sink.md index fbcc04f..0d31a2b 100644 --- a/docs/lakehouse-sink.md +++ b/docs/lakehouse-sink.md @@ -42,7 +42,7 @@ To build the Lakehouse sink connector from the source code, follow these steps. ```bash ls target - pulsar-io-lakehouse-{{connector:version}}.nar + pulsar-io-lakehouse-4.0.0.4.nar ``` # How to configure @@ -127,7 +127,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "inputs": [ "test-hudi-pulsar" ], - "archive": "connectors/pulsar-io-hudi-{{connector:version}}.nar", + "archive": "connectors/pulsar-io-hudi-4.0.0.4.nar", "processingGuarantees": "EFFECTIVELY_ONCE", "parallelism": 1, "configs": { @@ -151,7 +151,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "inputs": [ "test-hudi-pulsar" ], - "archive": "connectors/pulsar-io-hudi-{{connector:version}}-cloud.nar", + "archive": "connectors/pulsar-io-hudi-4.0.0.4-cloud.nar", "parallelism": 1, "processingGuarantees": "EFFECTIVELY_ONCE", "configs": { @@ -179,7 +179,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "inputs": [ "test-iceberg-pulsar" ], - "archive": "connectors/pulsar-io-lakehouse-{{connector:version}}.nar", + "archive": "connectors/pulsar-io-lakehouse-4.0.0.4.nar", "processingGuarantees":"EFFECTIVELY_ONCE", "configs":{ "type":"iceberg", @@ -207,7 +207,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "inputs": [ "test-iceberg-pulsar" ], - "archive": "connectors/pulsar-io-lakehouse-{{connector:version}}-cloud.nar", + "archive": "connectors/pulsar-io-lakehouse-4.0.0.4-cloud.nar", "processingGuarantees":"EFFECTIVELY_ONCE", "configs":{ "type":"iceberg", @@ -238,7 +238,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "inputs": [ "test-delta-pulsar" ], - "archive": "connectors/pulsar-io-lakehouse-{{connector:version}}.nar", + "archive": "connectors/pulsar-io-lakehouse-4.0.0.4.nar", "processingGuarantees":"EFFECTIVELY_ONCE", "configs":{ "type":"delta", @@ -260,7 +260,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "inputs": [ "test-delta-pulsar" ], - "archive": "connectors/pulsar-io-lakehouse-{{connector:version}}-cloud.nar", + "archive": "connectors/pulsar-io-lakehouse-4.0.0.4-cloud.nar", "processingGuarantees":"EFFECTIVELY_ONCE", "configs":{ "type":"delta", @@ -330,7 +330,7 @@ This example describes how to use the Lakehouse sink connector to fetch data fro 1. Copy the NAR package to the Pulsar connectors directory. ``` - cp pulsar-io-lakehouse-{{connector:version}}.nar PULSAR_HOME/connectors/pulsar-io-lakehouse-{{connector:version}}.nar + cp pulsar-io-lakehouse-4.0.0.4.nar PULSAR_HOME/connectors/pulsar-io-lakehouse-4.0.0.4.nar ``` 2. Start Pulsar in standalone mode. @@ -363,7 +363,7 @@ This example explains how to create a Lakehouse sink connector in an on-premises 1. Copy the NAR package of the Lakehouse sink connector to the Pulsar connectors directory. ```bash - cp pulsar-io-lakehouse-{{connector:version}}.nar $PULSAR_HOME/connectors/pulsar-io-lakehouse-{{connector:version}}.nar + cp pulsar-io-lakehouse-4.0.0.4.nar $PULSAR_HOME/connectors/pulsar-io-lakehouse-4.0.0.4.nar ``` 2. Reload all [built-in connectors](https://pulsar.apache.org/docs/en/next/io-connectors/). diff --git a/docs/lakehouse-source.md b/docs/lakehouse-source.md index ccf4194..eeaa9bd 100644 --- a/docs/lakehouse-source.md +++ b/docs/lakehouse-source.md @@ -42,7 +42,7 @@ To build the Lakehouse source connector from the source code, follow these steps ```bash ls target - pulsar-io-lakehouse-{{connector:version}}.nar + pulsar-io-lakehouse-4.0.0.4.nar ``` # How to configure @@ -90,7 +90,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "parallelism":1, "topicName": "delta_source", "processingGuarantees":"ATLEAST_ONCE", - "archive": "connectors/pulsar-io-lakehouse-{{connector:version}}.nar", + "archive": "connectors/pulsar-io-lakehouse-4.0.0.4.nar", "configs":{ "type":"delta", "checkpointInterval": 180, @@ -115,7 +115,7 @@ You can create a configuration file (JSON or YAML) to set the properties if you "parallelism":1, "topicName": "delta_source", "processingGuarantees":"ATLEAST_ONCE", - "archive": "connectors/pulsar-io-lakehouse-{{connector:version}}-cloud.nar", + "archive": "connectors/pulsar-io-lakehouse-4.0.0.4-cloud.nar", "configs":{ "type":"delta", "checkpointInterval": 180, @@ -175,7 +175,7 @@ This example describes how to use the Lakehouse source connector to fetch data f 1. Copy the NAR package to the Pulsar connectors directory. ``` - cp pulsar-io-lakehouse-{{connector:version}}.nar PULSAR_HOME/connectors/pulsar-io-lakehouse-{{connector:version}}.nar + cp pulsar-io-lakehouse-4.0.0.4.nar PULSAR_HOME/connectors/pulsar-io-lakehouse-4.0.0.4.nar ``` 2. Start Pulsar in standalone mode. @@ -206,7 +206,7 @@ This example explains how to create a Lakehouse source connector in an on-premis 1. Copy the NAR package of the Lakehouse source connector to the Pulsar connectors directory. ``` - cp pulsar-io-lakehouse-{{connector:version}}.nar $PULSAR_HOME/connectors/pulsar-io-lakehouse-{{connector:version}}.nar + cp pulsar-io-lakehouse-4.0.0.4.nar $PULSAR_HOME/connectors/pulsar-io-lakehouse-4.0.0.4.nar ``` 2. Reload all [built-in connectors](https://pulsar.apache.org/docs/en/next/io-connectors/). diff --git a/image/Dockerfile b/image/Dockerfile index de62491..d551b27 100644 --- a/image/Dockerfile +++ b/image/Dockerfile @@ -28,7 +28,7 @@ ENV JAVA_HOME=/usr # we need to update the download link once the version is updated RUN curl -Ls https://github.com/xerial/snappy-java/archive/refs/tags/$SNAPPY_VERSION.tar.gz | tar zxf - && cd snappy-java-$SNAPPY_VERSION && make clean-native native -FROM streamnative/pulsar-functions-java-runner:${PULSAR_VERSION} +FROM snstage/pulsar-functions-java-runner:${PULSAR_VERSION} USER 0 COPY --from=snappy-java /tmp/libsnappyjava.so /usr/lib/libsnappyjava.so # need to configure -Dorg.xerial.snappy.use.systemlib=true in the jvm options to use the system lib diff --git a/pom.xml b/pom.xml index 46f6fda..da26e50 100644 --- a/pom.xml +++ b/pom.xml @@ -1,4 +1,4 @@ - + - + org.apache apache 18 - 4.0.0 - org.apache.pulsar.ecosystem pulsar-io-lakehouse - 4.0.0-SNAPSHOT + 4.0.0.4 Pulsar Ecosystem :: IO Connector :: Lake House It is a lakehouse connector streaming convert data between lakehouse and Apache Pulsar. - 8 8 @@ -45,11 +40,10 @@ true 4.2.2 0 - 2.13.2.1 1.18.22 - 2.10.0.4 + 4.0.0.4 2.17.2 1.7.25 3.2.4 @@ -63,12 +57,10 @@ hadoop3-2.2.1 2.12.0 1.1.8.4 - 7.3.0 4.0.3 3.12.4 - 3.0 3.1.2 @@ -77,10 +69,9 @@ 1.2.0 8.37 4.2.2 - - + + - Apache License, Version 2.0 @@ -88,7 +79,6 @@ repo - @@ -182,8 +172,6 @@ log4j-slf4j-impl ${log4j2.version} - - org.apache.iceberg @@ -215,20 +203,17 @@ iceberg-hive-metastore ${iceberg.version} - org.apache.parquet parquet-avro ${parquet.avro.version} - io.delta delta-standalone_2.12 ${delta.version} - org.apache.parquet parquet-column @@ -239,7 +224,6 @@ parquet-hadoop ${parquet.version} - org.slf4j slf4j-api @@ -250,13 +234,11 @@ slf4j-simple ${slf4j.version} - org.xerial.snappy snappy-java ${snappy.java.version} - org.apache.hudi @@ -270,7 +252,6 @@ - org.apache.hadoop @@ -288,14 +269,12 @@ aws-java-sdk-bundle ${aws.sdk.version} - com.google.cloud.bigdataoss gcs-connector ${gcs.version} shaded - org.apache.hadoop hadoop-azure @@ -311,10 +290,7 @@ - - - org.testng @@ -340,13 +316,10 @@ ${mockito.version} test - - - org.projectlombok @@ -358,13 +331,11 @@ spotbugs-annotations provided - io.streamnative pulsar-io-core - com.fasterxml.jackson.core jackson-databind @@ -378,37 +349,30 @@ org.apache.hadoop hadoop-client - org.apache.iceberg iceberg-api - org.apache.iceberg iceberg-core - org.apache.iceberg iceberg-parquet - org.apache.iceberg iceberg-common - org.apache.iceberg iceberg-data - org.apache.iceberg iceberg-hive-metastore - io.delta delta-standalone_2.12 @@ -417,7 +381,6 @@ org.apache.parquet parquet-avro - org.apache.parquet parquet-column @@ -426,7 +389,6 @@ org.apache.parquet parquet-hadoop - org.slf4j slf4j-api @@ -435,12 +397,10 @@ org.slf4j slf4j-simple - org.xerial.snappy snappy-java - org.apache.logging.log4j log4j-slf4j-impl @@ -461,12 +421,10 @@ org.apache.logging.log4j log4j-slf4j-impl - io.netty netty-common - org.apache.hudi hudi-java-client @@ -485,7 +443,6 @@ - org.apache.curator curator-framework @@ -497,7 +454,6 @@ - org.apache.hudi hudi-client-common @@ -511,7 +467,6 @@ - org.testng @@ -529,7 +484,6 @@ test - cloud @@ -553,13 +507,11 @@ com.amazonaws aws-java-sdk-bundle - com.google.cloud.bigdataoss gcs-connector shaded - org.apache.hadoop hadoop-azure @@ -597,7 +549,6 @@ - ${project.artifactId}-${project.version}${cloudPkgSuffix} @@ -613,7 +564,7 @@ -Xlint:unchecked -Xpkginfo:always - + @@ -725,38 +676,32 @@ - - + org.apache.maven.plugins maven-compiler-plugin - org.apache.maven.plugins maven-surefire-plugin - org.apache.nifi nifi-nar-maven-plugin - com.mycila license-maven-plugin - org.apache.maven.plugins maven-checkstyle-plugin - com.github.spotbugs @@ -764,5 +709,24 @@ - + + + ossrh + https://s01.oss.sonatype.org/service/local/repositories/iostreamnative-3429/content + + + nexus-snapshot + https://s01.oss.sonatype.org/content/repositories/snapshots + + + + + ossrh + https://s01.oss.sonatype.org/content/repositories/snapshots + + + ossrh + https://s01.oss.sonatype.org/service/local/staging/deploy/maven2/ + +