offset = httpConnectRecord.getData().getPosition().getOffset().getOffset();
+
+ // send the request
+ return this.webClient.post(url.getPath())
+ .host(url.getHost())
+ .port(url.getPort() == -1 ? (Objects.equals(url.getScheme(), "https") ? 443 : 80) : url.getPort())
+ .putHeaders(headers)
+ .ssl(Objects.equals(url.getScheme(), "https"))
+ .sendJson(httpConnectRecord)
+ .onSuccess(res -> {
+ log.info("Request sent successfully. Record: timestamp={}, offset={}", timestamp, offset);
+ // log the response
+ if (HttpUtils.is2xxSuccessful(res.statusCode())) {
+ if (log.isDebugEnabled()) {
+ log.debug("Received successful response: statusCode={}. Record: timestamp={}, offset={}, responseBody={}",
+ res.statusCode(), timestamp, offset, res.bodyAsString());
+ } else {
+ log.info("Received successful response: statusCode={}. Record: timestamp={}, offset={}", res.statusCode(), timestamp, offset);
+ }
+ } else {
+ if (log.isDebugEnabled()) {
+ log.warn("Received non-2xx response: statusCode={}. Record: timestamp={}, offset={}, responseBody={}",
+ res.statusCode(), timestamp, offset, res.bodyAsString());
+ } else {
+ log.warn("Received non-2xx response: statusCode={}. Record: timestamp={}, offset={}", res.statusCode(), timestamp, offset);
+ }
+ }
+
+ })
+ .onFailure(err -> log.error("Request failed to send. Record: timestamp={}, offset={}", timestamp, offset, err));
+ }
+
+
+ /**
+ * Cleans up and releases resources used by the HTTP/HTTPS handler.
+ */
+ @Override
+ public void stop() {
+ if (this.webClient != null) {
+ this.webClient.close();
+ } else {
+ log.warn("WebClient is null, ignore.");
+ }
+ }
+
+
+}
\ No newline at end of file
diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/handle/HttpSinkHandler.java b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/handle/HttpSinkHandler.java
new file mode 100644
index 0000000000..09fd66a762
--- /dev/null
+++ b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/handle/HttpSinkHandler.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eventmesh.connector.http.sink.handle;
+
+import org.apache.eventmesh.connector.http.sink.data.HttpConnectRecord;
+import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord;
+
+import java.net.URI;
+
+import io.vertx.core.Future;
+import io.vertx.core.buffer.Buffer;
+import io.vertx.ext.web.client.HttpResponse;
+
+/**
+ * Interface for handling ConnectRecords via HTTP or HTTPS. Classes implementing this interface are responsible for processing ConnectRecords by
+ * sending them over HTTP or HTTPS, with additional support for handling multiple requests and asynchronous processing.
+ *
+ * Any class that needs to process ConnectRecords via HTTP or HTTPS should implement this interface.
+ * Implementing classes must provide implementations for the {@link #start()}, {@link #handle(ConnectRecord)},
+ * {@link #deliver(URI, HttpConnectRecord)}, and {@link #stop()} methods.
+ *
+ * Implementing classes should ensure thread safety and handle HTTP/HTTPS communication efficiently.
+ * The {@link #start()} method initializes any necessary resources for HTTP/HTTPS communication. The {@link #handle(ConnectRecord)} method processes a
+ * ConnectRecord by sending it over HTTP or HTTPS. The {@link #deliver(URI, HttpConnectRecord)} method processes HttpConnectRecord on specified URL
+ * while returning its own processing logic {@link #stop()} method releases any resources used for HTTP/HTTPS communication.
+ *
+ * It's recommended to handle exceptions gracefully within the {@link #deliver(URI, HttpConnectRecord)} method
+ * to prevent message loss or processing interruptions.
+ */
+public interface HttpSinkHandler {
+
+ /**
+ * Initializes the HTTP/HTTPS handler. This method should be called before using the handler.
+ */
+ void start();
+
+ /**
+ * Processes a ConnectRecord by sending it over HTTP or HTTPS. This method should be called for each ConnectRecord that needs to be processed.
+ *
+ * @param record the ConnectRecord to process
+ */
+ void handle(ConnectRecord record);
+
+
+ /**
+ * Processes HttpConnectRecord on specified URL while returning its own processing logic
+ *
+ * @param url URI to which the HttpConnectRecord should be sent
+ * @param httpConnectRecord HttpConnectRecord to process
+ * @return processing chain
+ */
+ Future> deliver(URI url, HttpConnectRecord httpConnectRecord);
+
+ /**
+ * Cleans up and releases resources used by the HTTP/HTTPS handler. This method should be called when the handler is no longer needed.
+ */
+ void stop();
+}
+
diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/handle/RetryHttpSinkHandler.java b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/handle/RetryHttpSinkHandler.java
new file mode 100644
index 0000000000..06700261d5
--- /dev/null
+++ b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/handle/RetryHttpSinkHandler.java
@@ -0,0 +1,206 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eventmesh.connector.http.sink.handle;
+
+import org.apache.eventmesh.connector.http.sink.config.HttpRetryConfig;
+import org.apache.eventmesh.connector.http.sink.config.SinkConnectorConfig;
+import org.apache.eventmesh.connector.http.sink.data.HttpConnectRecord;
+import org.apache.eventmesh.connector.http.sink.data.HttpExportMetadata;
+import org.apache.eventmesh.connector.http.sink.data.HttpExportRecord;
+import org.apache.eventmesh.connector.http.util.HttpUtils;
+import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord;
+
+import java.net.ConnectException;
+import java.net.URI;
+import java.time.Duration;
+import java.time.LocalDateTime;
+import java.util.Arrays;
+import java.util.List;
+import java.util.UUID;
+import java.util.stream.Collectors;
+
+import io.vertx.core.Future;
+import io.vertx.core.buffer.Buffer;
+import io.vertx.ext.web.client.HttpResponse;
+
+import lombok.extern.slf4j.Slf4j;
+
+import dev.failsafe.Failsafe;
+import dev.failsafe.RetryPolicy;
+import dev.failsafe.RetryPolicyBuilder;
+import dev.failsafe.event.ExecutionEvent;
+
+
+@Slf4j
+public class RetryHttpSinkHandler implements HttpSinkHandler {
+
+ private final SinkConnectorConfig connectorConfig;
+
+ // Retry policy builder
+ private RetryPolicyBuilder> retryPolicyBuilder;
+
+ private final List urls;
+
+ private final HttpSinkHandler sinkHandler;
+
+
+ public RetryHttpSinkHandler(SinkConnectorConfig connectorConfig, HttpSinkHandler sinkHandler) {
+ this.connectorConfig = connectorConfig;
+ this.sinkHandler = sinkHandler;
+
+ // Initialize retry
+ initRetry();
+
+ // Initialize URLs
+ String[] urlStrings = connectorConfig.getUrls();
+ this.urls = Arrays.stream(urlStrings)
+ .map(URI::create)
+ .collect(Collectors.toList());
+ }
+
+ private void initRetry() {
+ HttpRetryConfig httpRetryConfig = this.connectorConfig.getRetryConfig();
+
+ this.retryPolicyBuilder = RetryPolicy.>builder()
+ .handleIf(e -> e instanceof ConnectException)
+ .handleResultIf(response -> httpRetryConfig.isRetryOnNonSuccess() && !HttpUtils.is2xxSuccessful(response.statusCode()))
+ .withMaxRetries(httpRetryConfig.getMaxRetries())
+ .withDelay(Duration.ofMillis(httpRetryConfig.getInterval()));
+ }
+
+
+ /**
+ * Initializes the WebClient for making HTTP requests based on the provided SinkConnectorConfig.
+ */
+ @Override
+ public void start() {
+ sinkHandler.start();
+ }
+
+
+ /**
+ * Processes a ConnectRecord by sending it over HTTP or HTTPS. This method should be called for each ConnectRecord that needs to be processed.
+ *
+ * @param record the ConnectRecord to process
+ */
+ @Override
+ public void handle(ConnectRecord record) {
+ for (URI url : this.urls) {
+ // convert ConnectRecord to HttpConnectRecord
+ String type = String.format("%s.%s.%s",
+ this.connectorConfig.getConnectorName(), url.getScheme(),
+ this.connectorConfig.getWebhookConfig().isActivate() ? "webhook" : "common");
+ HttpConnectRecord httpConnectRecord = HttpConnectRecord.convertConnectRecord(record, type);
+ // handle the HttpConnectRecord
+ deliver(url, httpConnectRecord);
+ }
+ }
+
+
+ /**
+ * Processes HttpConnectRecord on specified URL while returning its own processing logic This method provides the retry power to process the
+ * HttpConnectRecord
+ *
+ * @param url URI to which the HttpConnectRecord should be sent
+ * @param httpConnectRecord HttpConnectRecord to process
+ * @return processing chain
+ */
+ @Override
+ public Future> deliver(URI url, HttpConnectRecord httpConnectRecord) {
+ // Only webhook mode needs to use the UUID to identify the request
+ String id = httpConnectRecord.getUuid();
+
+ // Build the retry policy
+ RetryPolicy> retryPolicy = retryPolicyBuilder
+ .onSuccess(event -> {
+ if (connectorConfig.getWebhookConfig().isActivate()) {
+ // convert the result to an HttpExportRecord
+ HttpExportRecord exportRecord = covertToExportRecord(httpConnectRecord, event, event.getResult(), event.getException(), url, id);
+ // add the data to the queue
+ ((WebhookHttpSinkHandler) sinkHandler).addDataToQueue(exportRecord);
+ }
+ })
+ .onRetry(event -> {
+ if (log.isDebugEnabled()) {
+ log.warn("Retrying the request to {} for the {} time. HttpConnectRecord= {}", url, event.getAttemptCount(), httpConnectRecord);
+ } else {
+ log.warn("Retrying the request to {} for the {} time.", url, event.getAttemptCount());
+ }
+ if (connectorConfig.getWebhookConfig().isActivate()) {
+ HttpExportRecord exportRecord =
+ covertToExportRecord(httpConnectRecord, event, event.getLastResult(), event.getLastException(), url, id);
+ ((WebhookHttpSinkHandler) sinkHandler).addDataToQueue(exportRecord);
+ }
+ // update the HttpConnectRecord
+ httpConnectRecord.setTime(LocalDateTime.now().toString());
+ httpConnectRecord.setUuid(UUID.randomUUID().toString());
+ })
+ .onFailure(event -> {
+ if (log.isDebugEnabled()) {
+ log.error("Failed to send the request to {} after {} attempts. HttpConnectRecord= {}", url, event.getAttemptCount(),
+ httpConnectRecord, event.getException());
+ } else {
+ log.error("Failed to send the request to {} after {} attempts.", url, event.getAttemptCount(), event.getException());
+ }
+ if (connectorConfig.getWebhookConfig().isActivate()) {
+ HttpExportRecord exportRecord = covertToExportRecord(httpConnectRecord, event, event.getResult(), event.getException(), url, id);
+ ((WebhookHttpSinkHandler) sinkHandler).addDataToQueue(exportRecord);
+ }
+ }).build();
+
+ // Handle the HttpConnectRecord with retry
+ Failsafe.with(retryPolicy)
+ .getStageAsync(() -> sinkHandler.deliver(url, httpConnectRecord).toCompletionStage());
+
+ return null;
+ }
+
+ /**
+ * Converts the ExecutionCompletedEvent to an HttpExportRecord.
+ *
+ * @param httpConnectRecord HttpConnectRecord
+ * @param event ExecutionEvent
+ * @param response the response of the request, may be null
+ * @param e the exception thrown during the request, may be null
+ * @param url the URL the request was sent to
+ * @param id UUID
+ * @return the converted HttpExportRecord
+ */
+ private HttpExportRecord covertToExportRecord(HttpConnectRecord httpConnectRecord, ExecutionEvent event, HttpResponse response,
+ Throwable e, URI url, String id) {
+
+ HttpExportMetadata httpExportMetadata = HttpExportMetadata.builder()
+ .url(url.toString())
+ .code(response != null ? response.statusCode() : -1)
+ .message(response != null ? response.statusMessage() : e.getMessage())
+ .receivedTime(LocalDateTime.now())
+ .uuid(httpConnectRecord.getUuid())
+ .retriedBy(event.getAttemptCount() > 1 ? id : null)
+ .retryNum(event.getAttemptCount() - 1).build();
+
+ return new HttpExportRecord(httpExportMetadata, response == null ? null : response.bodyAsString());
+ }
+
+ /**
+ * Cleans up and releases resources used by the HTTP/HTTPS handler.
+ */
+ @Override
+ public void stop() {
+ sinkHandler.stop();
+ }
+}
diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/handle/WebhookHttpSinkHandler.java b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/handle/WebhookHttpSinkHandler.java
new file mode 100644
index 0000000000..e07683fcfa
--- /dev/null
+++ b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/handle/WebhookHttpSinkHandler.java
@@ -0,0 +1,348 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eventmesh.connector.http.sink.handle;
+
+import org.apache.eventmesh.common.exception.EventMeshException;
+import org.apache.eventmesh.connector.http.sink.config.HttpWebhookConfig;
+import org.apache.eventmesh.connector.http.sink.config.SinkConnectorConfig;
+import org.apache.eventmesh.connector.http.sink.data.HttpConnectRecord;
+import org.apache.eventmesh.connector.http.sink.data.HttpExportMetadata;
+import org.apache.eventmesh.connector.http.sink.data.HttpExportRecord;
+import org.apache.eventmesh.connector.http.sink.data.HttpExportRecordPage;
+import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord;
+
+import org.apache.commons.lang3.StringUtils;
+
+import java.net.URI;
+import java.time.LocalDateTime;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Objects;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import io.netty.handler.codec.http.HttpResponseStatus;
+import io.vertx.core.Future;
+import io.vertx.core.MultiMap;
+import io.vertx.core.Vertx;
+import io.vertx.core.buffer.Buffer;
+import io.vertx.core.http.HttpHeaders;
+import io.vertx.core.http.HttpMethod;
+import io.vertx.core.http.HttpServer;
+import io.vertx.core.http.HttpServerOptions;
+import io.vertx.ext.web.Router;
+import io.vertx.ext.web.client.HttpResponse;
+import io.vertx.ext.web.handler.LoggerHandler;
+
+import com.alibaba.fastjson2.JSON;
+import com.alibaba.fastjson2.JSONWriter;
+
+import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * Extends CommonHttpSinkHandler to provide additional functionality for handling webhook features, including sending requests to callback servers,
+ * allowing longer response wait times, storing responses returned from callback servers, and exposing received data through an HTTP service.
+ */
+@Slf4j
+public class WebhookHttpSinkHandler extends CommonHttpSinkHandler {
+
+ private final SinkConnectorConfig sinkConnectorConfig;
+
+ // the configuration for webhook
+ private final HttpWebhookConfig webhookConfig;
+
+ // the server for exporting the received data
+ private HttpServer exportServer;
+
+ // store the received data, when webhook is enabled
+ private final ConcurrentLinkedQueue receivedDataQueue;
+
+ // the maximum queue size
+ private final int maxQueueSize;
+
+ // the current queue size
+ private final AtomicInteger currentQueueSize;
+
+ public WebhookHttpSinkHandler(SinkConnectorConfig sinkConnectorConfig) {
+ super(sinkConnectorConfig);
+ this.sinkConnectorConfig = sinkConnectorConfig;
+ this.webhookConfig = sinkConnectorConfig.getWebhookConfig();
+ this.maxQueueSize = this.webhookConfig.getMaxStorageSize();
+ this.currentQueueSize = new AtomicInteger(0);
+ this.receivedDataQueue = new ConcurrentLinkedQueue<>();
+ // init the export server
+ doInitExportServer();
+ }
+
+ /**
+ * Initialize the server for exporting the received data
+ */
+ private void doInitExportServer() {
+ final Vertx vertx = Vertx.vertx();
+ final Router router = Router.router(vertx);
+ // add logger handler
+ router.route().handler(LoggerHandler.create());
+ // add export handler
+ router.route()
+ .path(this.webhookConfig.getExportPath())
+ .method(HttpMethod.GET)
+ .produces("application/json")
+ .handler(ctx -> {
+ // Validate the request parameters
+ MultiMap params = ctx.request().params();
+ String pageNumStr = params.get(ParamEnum.PAGE_NUM.getValue());
+ String pageSizeStr = params.get(ParamEnum.PAGE_SIZE.getValue());
+ String type = params.get(ParamEnum.TYPE.getValue());
+
+ // 1. type must be "poll" or "peek" or null
+ // 2. if type is "peek", pageNum must be greater than 0
+ // 3. pageSize must be greater than 0
+ if ((type != null && !Objects.equals(type, TypeEnum.PEEK.getValue()) && !Objects.equals(type, TypeEnum.POLL.getValue()))
+ || (Objects.equals(type, TypeEnum.PEEK.getValue()) && (StringUtils.isBlank(pageNumStr) || Integer.parseInt(pageNumStr) < 1))
+ || (StringUtils.isBlank(pageSizeStr) || Integer.parseInt(pageSizeStr) < 1)) {
+
+ // Return 400 Bad Request if the request parameters are invalid
+ ctx.response()
+ .putHeader(HttpHeaders.CONTENT_TYPE, "application/json; charset=utf-8")
+ .setStatusCode(HttpResponseStatus.BAD_REQUEST.code())
+ .end();
+ log.info("Invalid request parameters. pageNum: {}, pageSize: {}, type: {}", pageNumStr, pageSizeStr, type);
+ return;
+ }
+
+ // Parse the request parameters
+ if (type == null) {
+ type = TypeEnum.PEEK.getValue();
+ }
+ int pageNum = StringUtils.isBlank(pageNumStr) ? 1 : Integer.parseInt(pageNumStr);
+ int pageSize = Integer.parseInt(pageSizeStr);
+
+ if (currentQueueSize.get() == 0) {
+ ctx.response()
+ .putHeader(HttpHeaders.CONTENT_TYPE, "application/json; charset=utf-8")
+ .setStatusCode(HttpResponseStatus.NO_CONTENT.code())
+ .end();
+ log.info("No callback data to export.");
+ return;
+ }
+
+ // Get the received data
+ List exportRecords;
+ if (Objects.equals(type, TypeEnum.POLL.getValue())) {
+ // If the type is poll, only the first page of data is exported and removed
+ exportRecords = getDataFromQueue(0, pageSize, true);
+ } else {
+ // If the type is peek, the specified page of data is exported without removing
+ int startIndex = (pageNum - 1) * pageSize;
+ int endIndex = startIndex + pageSize;
+ exportRecords = getDataFromQueue(startIndex, endIndex, false);
+ }
+
+ // Create HttpExportRecordPage
+ HttpExportRecordPage page = new HttpExportRecordPage(pageNum, exportRecords.size(), exportRecords);
+
+ // export the received data
+ ctx.response()
+ .putHeader(HttpHeaders.CONTENT_TYPE, "application/json; charset=utf-8")
+ .setStatusCode(HttpResponseStatus.OK.code())
+ .send(JSON.toJSONString(page, JSONWriter.Feature.WriteMapNullValue));
+ if (log.isDebugEnabled()) {
+ log.debug("Succeed to export callback data. Data: {}", page);
+ } else {
+ log.info("Succeed to export callback data.");
+ }
+ });
+ // create the export server
+ this.exportServer = vertx.createHttpServer(new HttpServerOptions()
+ .setPort(this.webhookConfig.getPort())
+ .setIdleTimeout(this.webhookConfig.getServerIdleTimeout())
+ .setIdleTimeoutUnit(TimeUnit.MILLISECONDS)).requestHandler(router);
+ }
+
+ /**
+ * Starts the HTTP/HTTPS handler by creating a WebClient with configured options and starting the export server.
+ */
+ @Override
+ public void start() {
+ // start the webclient
+ super.start();
+ // start the export server
+ Throwable t = this.exportServer.listen().cause();
+ if (t != null) {
+ throw new EventMeshException("Failed to start Vertx server. ", t);
+ }
+ }
+
+ /**
+ * Processes a ConnectRecord by sending it over HTTP or HTTPS. This method should be called for each ConnectRecord that needs to be processed.
+ *
+ * @param record the ConnectRecord to process
+ */
+ @Override
+ public void handle(ConnectRecord record) {
+ for (URI url : super.getUrls()) {
+ // convert ConnectRecord to HttpConnectRecord
+ String type = String.format("%s.%s.%s", this.getConnectorConfig().getConnectorName(), url.getScheme(), "webhook");
+ HttpConnectRecord httpConnectRecord = HttpConnectRecord.convertConnectRecord(record, type);
+ // handle the HttpConnectRecord
+ deliver(url, httpConnectRecord);
+ }
+ }
+
+
+ /**
+ * Processes HttpConnectRecord on specified URL while returning its own processing logic This method sends the HttpConnectRecord to the specified
+ * URL by super class method and stores the received data.
+ *
+ * @param url URI to which the HttpConnectRecord should be sent
+ * @param httpConnectRecord HttpConnectRecord to process
+ * @return processing chain
+ */
+ @Override
+ public Future> deliver(URI url, HttpConnectRecord httpConnectRecord) {
+ // send the request
+ Future> responseFuture = super.deliver(url, httpConnectRecord);
+ // store the received data
+ return responseFuture.onComplete(arr -> {
+ // If open retry, return directly and handled by RetryHttpSinkHandler
+ if (sinkConnectorConfig.getRetryConfig().getMaxRetries() > 0) {
+ return;
+ }
+ // create ExportMetadataBuilder
+ HttpResponse response = arr.succeeded() ? arr.result() : null;
+
+ HttpExportMetadata httpExportMetadata = HttpExportMetadata.builder()
+ .url(url.toString())
+ .code(response != null ? response.statusCode() : -1)
+ .message(response != null ? response.statusMessage() : arr.cause().getMessage())
+ .receivedTime(LocalDateTime.now())
+ .retriedBy(null)
+ .uuid(httpConnectRecord.getUuid())
+ .retryNum(0)
+ .build();
+
+ // create ExportRecord
+ HttpExportRecord exportRecord = new HttpExportRecord(httpExportMetadata, arr.succeeded() ? arr.result().bodyAsString() : null);
+ // add the data to the queue
+ addDataToQueue(exportRecord);
+ });
+ }
+
+
+ /**
+ * Adds the received data to the queue.
+ *
+ * @param exportRecord the received data to add to the queue
+ */
+ public void addDataToQueue(HttpExportRecord exportRecord) {
+ // If the current queue size is greater than or equal to the maximum queue size, remove the oldest element
+ if (currentQueueSize.get() >= maxQueueSize) {
+ Object removedData = receivedDataQueue.poll();
+ if (log.isDebugEnabled()) {
+ log.debug("The queue is full, remove the oldest element: {}", removedData);
+ } else {
+ log.info("The queue is full, remove the oldest element");
+ }
+ currentQueueSize.decrementAndGet();
+ }
+ // Try to put the received data into the queue
+ if (receivedDataQueue.offer(exportRecord)) {
+ currentQueueSize.incrementAndGet();
+ log.debug("Successfully put the received data into the queue: {}", exportRecord);
+ } else {
+ log.error("Failed to put the received data into the queue: {}", exportRecord);
+ }
+ }
+
+ /**
+ * Gets the received data from the queue.
+ *
+ * @param startIndex the start index of the data to get
+ * @param endIndex the end index of the data to get
+ * @param removed whether to remove the data from the queue
+ * @return the received data
+ */
+ private List getDataFromQueue(int startIndex, int endIndex, boolean removed) {
+ Iterator iterator = receivedDataQueue.iterator();
+
+ List pageItems = new ArrayList<>(endIndex - startIndex);
+ int count = 0;
+ while (iterator.hasNext() && count < endIndex) {
+ HttpExportRecord item = iterator.next();
+ if (count >= startIndex) {
+ pageItems.add(item);
+ if (removed) {
+ iterator.remove();
+ currentQueueSize.decrementAndGet();
+ }
+ }
+ count++;
+ }
+ return pageItems;
+ }
+
+ /**
+ * Cleans up and releases resources used by the HTTP/HTTPS handler.
+ */
+ @Override
+ public void stop() {
+ // stop the webclient
+ super.stop();
+ // stop the export server
+ if (this.exportServer != null) {
+ Throwable t = this.exportServer.close().cause();
+ if (t != null) {
+ throw new EventMeshException("Failed to stop Vertx server. ", t);
+ }
+ } else {
+ log.warn("Callback server is null, ignore.");
+ }
+ }
+
+
+ @Getter
+ public enum ParamEnum {
+ PAGE_NUM("pageNum"),
+ PAGE_SIZE("pageSize"),
+ TYPE("type");
+
+ private final String value;
+
+ ParamEnum(String value) {
+ this.value = value;
+ }
+
+ }
+
+
+ @Getter
+ public enum TypeEnum {
+ POLL("poll"),
+ PEEK("peek");
+
+ private final String value;
+
+ TypeEnum(String value) {
+ this.value = value;
+ }
+
+ }
+}
\ No newline at end of file
diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/util/HttpUtils.java b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/util/HttpUtils.java
new file mode 100644
index 0000000000..79f9fd120d
--- /dev/null
+++ b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/util/HttpUtils.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eventmesh.connector.http.util;
+
+public class HttpUtils {
+
+ /**
+ * Checks if the status code represents a successful response (2xx).
+ *
+ * @param statusCode the HTTP status code to check
+ * @return true if the status code is 2xx, false otherwise
+ */
+ public static boolean is2xxSuccessful(int statusCode) {
+ int seriesCode = statusCode / 100;
+ return seriesCode == 2;
+ }
+}
diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/resources/sink-config.yml b/eventmesh-connectors/eventmesh-connector-http/src/main/resources/sink-config.yml
new file mode 100644
index 0000000000..5103525456
--- /dev/null
+++ b/eventmesh-connectors/eventmesh-connector-http/src/main/resources/sink-config.yml
@@ -0,0 +1,45 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+pubSubConfig:
+ meshAddress: 127.0.0.1:10000
+ subject: TopicTest
+ idc: FT
+ env: PRD
+ group: httpSink
+ appId: 5032
+ userName: httpSinkUser
+ passWord: httpPassWord
+connectorConfig:
+ connectorName: httpSink
+ urls:
+ - http://127.0.0.1:8987/test
+ keepAlive: true
+ keepAliveTimeout: 60000
+ idleTimeout: 5000 # timeunit: ms, recommended scope: common(5s - 10s), webhook(15s - 60s)
+ connectionTimeout: 5000 # timeunit: ms, recommended scope: 5 - 10s
+ maxConnectionPoolSize: 5
+ retryConfig:
+ maxRetries: 2
+ interval: 1000
+ retryOnNonSuccess: false
+ webhookConfig:
+ activate: false
+ exportPath: /export
+ port: 8988
+ serverIdleTimeout: 5000
+ maxStorageSize: 5000
diff --git a/eventmesh-connectors/eventmesh-connector-http/src/test/java/org/apache/eventmesh/connector/http/source/connector/HttpSinkConnectorTest.java b/eventmesh-connectors/eventmesh-connector-http/src/test/java/org/apache/eventmesh/connector/http/source/connector/HttpSinkConnectorTest.java
new file mode 100644
index 0000000000..738df6430b
--- /dev/null
+++ b/eventmesh-connectors/eventmesh-connector-http/src/test/java/org/apache/eventmesh/connector/http/source/connector/HttpSinkConnectorTest.java
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eventmesh.connector.http.source.connector;
+
+import static org.mockserver.model.HttpRequest.request;
+
+import org.apache.eventmesh.connector.http.sink.HttpSinkConnector;
+import org.apache.eventmesh.connector.http.sink.config.HttpSinkConfig;
+import org.apache.eventmesh.connector.http.sink.config.HttpWebhookConfig;
+import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord;
+import org.apache.eventmesh.openconnect.offsetmgmt.api.data.RecordOffset;
+import org.apache.eventmesh.openconnect.offsetmgmt.api.data.RecordPartition;
+import org.apache.eventmesh.openconnect.util.ConfigUtil;
+
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.UUID;
+
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockserver.integration.ClientAndServer;
+import org.mockserver.model.HttpRequest;
+import org.mockserver.model.HttpResponse;
+import org.mockserver.model.MediaType;
+
+import com.alibaba.fastjson2.JSON;
+import com.alibaba.fastjson2.JSONArray;
+import com.alibaba.fastjson2.JSONObject;
+
+import okhttp3.HttpUrl;
+import okhttp3.OkHttpClient;
+import okhttp3.Request;
+import okhttp3.Response;
+import okhttp3.ResponseBody;
+
+public class HttpSinkConnectorTest {
+
+ private HttpSinkConnector sinkConnector;
+
+ private HttpSinkConfig sinkConfig;
+
+ private URI severUri;
+
+ private ClientAndServer mockServer;
+
+
+ @BeforeEach
+ void before() throws Exception {
+ // init sinkConnector
+ this.sinkConnector = new HttpSinkConnector();
+ this.sinkConfig = (HttpSinkConfig) ConfigUtil.parse(sinkConnector.configClass());
+ this.sinkConnector.init(this.sinkConfig);
+ this.sinkConnector.start();
+
+ this.severUri = URI.create(sinkConfig.connectorConfig.getUrls()[0]);
+ // start mockServer
+ mockServer = ClientAndServer.startClientAndServer(severUri.getPort());
+ mockServer.reset()
+ .when(
+ request()
+ .withMethod("POST")
+ .withPath(severUri.getPath())
+ )
+ .respond(
+ httpRequest -> {
+ JSONObject requestBody = JSON.parseObject(httpRequest.getBodyAsString());
+ return HttpResponse.response()
+ .withContentType(MediaType.APPLICATION_JSON)
+ .withStatusCode(200)
+ .withBody(new JSONObject()
+ .fluentPut("code", 0)
+ .fluentPut("message", "success")
+ .fluentPut("data", requestBody.getJSONObject("data").get("data"))
+ .toJSONString()
+ ); // .withDelay(TimeUnit.SECONDS, 10);
+ }
+ );
+ }
+
+ @AfterEach
+ void after() throws Exception {
+ this.sinkConnector.stop();
+ this.mockServer.close();
+ }
+
+ @Test
+ void testPut() throws Exception {
+ // Create a list of ConnectRecord
+ final int times = 10;
+ List connectRecords = new ArrayList<>();
+ for (int i = 0; i < times; i++) {
+ ConnectRecord record = createConnectRecord();
+ connectRecords.add(record);
+ }
+ // Put ConnectRecord
+ sinkConnector.put(connectRecords);
+
+ // sleep 5s
+ Thread.sleep(5000);
+
+ // verify request
+ HttpRequest[] recordedRequests = mockServer.retrieveRecordedRequests(null);
+ assert recordedRequests.length == times;
+
+ // verify response
+ HttpWebhookConfig webhookConfig = sinkConfig.connectorConfig.getWebhookConfig();
+ String url = new HttpUrl.Builder()
+ .scheme("http")
+ .host(severUri.getHost())
+ .port(webhookConfig.getPort())
+ .addPathSegments(webhookConfig.getExportPath())
+ .addQueryParameter("pageNum", "1")
+ .addQueryParameter("pageSize", "10")
+ .addQueryParameter("type", "poll")
+ .build().toString();
+
+ // build request
+ Request request = new Request.Builder()
+ .url(url)
+ .addHeader("Content-Type", "application/json")
+ .build();
+
+ OkHttpClient client = new OkHttpClient();
+ try (Response response = client.newCall(request).execute()) {
+ // check response code
+ if (!response.isSuccessful()) {
+ throw new RuntimeException("Unexpected response code: " + response);
+ }
+ // check response body
+ ResponseBody responseBody = response.body();
+ if (responseBody != null) {
+ JSONObject jsonObject = JSON.parseObject(responseBody.string());
+ JSONArray pageItems = jsonObject.getJSONArray("pageItems");
+
+ assert pageItems != null && pageItems.size() == times;
+
+ for (int i = 0; i < times; i++) {
+ JSONObject pageItem = pageItems.getJSONObject(i);
+ assert pageItem != null;
+ assert pageItem.getJSONObject("data") != null;
+ assert pageItem.getJSONObject("metadata") != null;
+ }
+ }
+ }
+ }
+
+ private ConnectRecord createConnectRecord() {
+ RecordPartition partition = new RecordPartition();
+ RecordOffset offset = new RecordOffset();
+ long timestamp = System.currentTimeMillis();
+ return new ConnectRecord(partition, offset, timestamp, UUID.randomUUID().toString());
+ }
+}
diff --git a/eventmesh-connectors/eventmesh-connector-http/src/test/resources/sink-config.yml b/eventmesh-connectors/eventmesh-connector-http/src/test/resources/sink-config.yml
new file mode 100644
index 0000000000..3ea4238790
--- /dev/null
+++ b/eventmesh-connectors/eventmesh-connector-http/src/test/resources/sink-config.yml
@@ -0,0 +1,45 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+pubSubConfig:
+ meshAddress: 127.0.0.1:10000
+ subject: TopicTest
+ idc: FT
+ env: PRD
+ group: httpSink
+ appId: 5032
+ userName: httpSinkUser
+ passWord: httpPassWord
+connectorConfig:
+ connectorName: httpSink
+ urls:
+ - http://127.0.0.1:8987/test
+ keepAlive: true
+ keepAliveTimeout: 60000
+ idleTimeout: 15000 # timeunit: ms, recommended scope: common(5s - 10s), webhook(15s - 60s)
+ connectionTimeout: 5000 # timeunit: ms, recommended scope: 5 - 10s
+ maxConnectionPoolSize: 10
+ retryConfig:
+ maxRetries: 2
+ interval: 1000
+ retryOnNonSuccess: true
+ webhookConfig:
+ activate: true
+ exportPath: /export
+ port: 8988
+ serverIdleTimeout: 5000
+ maxStorageSize: 5000
\ No newline at end of file
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/build.gradle b/eventmesh-connectors/eventmesh-connector-jdbc/build.gradle
index b6d33aebe3..5118072e1d 100644
--- a/eventmesh-connectors/eventmesh-connector-jdbc/build.gradle
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/build.gradle
@@ -36,11 +36,14 @@ packageSources {
dependencies {
antlr("org.antlr:antlr4:4.13.0")
implementation 'org.antlr:antlr4-runtime:4.13.0'
+ implementation 'com.alibaba:druid:1.2.20'
+ compileOnly 'org.hibernate:hibernate-core:5.6.15.Final'
implementation project(":eventmesh-common")
implementation project(":eventmesh-openconnect:eventmesh-openconnect-java")
implementation project(":eventmesh-spi")
implementation 'com.zendesk:mysql-binlog-connector-java:0.28.0'
- implementation 'mysql:mysql-connector-java:8.0.32'
+ compileOnly 'mysql:mysql-connector-java:8.0.32'
+
compileOnly 'org.projectlombok:lombok'
annotationProcessor 'org.projectlombok:lombok'
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/CatalogChanges.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/CatalogChanges.java
index 34d2554f65..c835862eda 100644
--- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/CatalogChanges.java
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/CatalogChanges.java
@@ -25,11 +25,13 @@
import java.util.List;
import lombok.Data;
+import lombok.NoArgsConstructor;
-@Data
/**
* Represents changes in a catalog, such as schema or table modifications.
*/
+@Data
+@NoArgsConstructor
public class CatalogChanges {
/**
@@ -52,10 +54,10 @@ public class CatalogChanges {
// The table associated with the changes
private Table table;
// The list of columns affected by the changes
- private List columns;
+ private List> columns;
private CatalogChanges(String type, String operationType, CatalogSchema catalog, Table table,
- List columns) {
+ List> columns) {
this.type = type;
this.operationType = operationType;
this.catalog = catalog;
@@ -81,7 +83,7 @@ public static class Builder {
private String operationType;
private CatalogSchema catalog;
private Table table;
- private List columns;
+ private List> columns;
/**
* Sets the operation type for the change.
@@ -123,7 +125,7 @@ public Builder table(Table table) {
* @param columns The list of Column instances.
* @return The Builder instance.
*/
- public Builder columns(List columns) {
+ public Builder columns(List> columns) {
this.columns = columns;
return this;
}
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/DataChanges.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/DataChanges.java
index cb0298e91f..a23e521cf2 100644
--- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/DataChanges.java
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/DataChanges.java
@@ -18,52 +18,107 @@
package org.apache.eventmesh.connector.jdbc;
import lombok.Data;
+import lombok.NoArgsConstructor;
@Data
+@NoArgsConstructor
+/**
+ * DataChanges class representing changes in data associated with a JDBC connection.
+ */
public class DataChanges {
private Object after;
private Object before;
+ /**
+ * The type of change.
+ *
+ * {@link org.apache.eventmesh.connector.jdbc.event.DataChangeEventType}
+ *
+ */
private String type;
+ /**
+ * Constructs a DataChanges instance with 'after' and 'before' data.
+ *
+ * @param after The data after the change.
+ * @param before The data before the change.
+ */
public DataChanges(Object after, Object before) {
this.after = after;
this.before = before;
}
+ /**
+ * Constructs a DataChanges instance with 'after', 'before' data, and a change type.
+ *
+ * @param after The data after the change.
+ * @param before The data before the change.
+ * @param type The type of change.
+ */
public DataChanges(Object after, Object before, String type) {
this.after = after;
this.before = before;
this.type = type;
}
+ /**
+ * Creates a new DataChanges builder.
+ *
+ * @return The DataChanges builder.
+ */
public static Builder newBuilder() {
return new Builder();
}
+ /**
+ * Builder class for constructing DataChanges instances.
+ */
public static class Builder {
private String type;
private Object after;
private Object before;
+ /**
+ * Sets the change type in the builder.
+ *
+ * @param type The type of change.
+ * @return The DataChanges builder.
+ */
public Builder withType(String type) {
this.type = type;
return this;
}
+ /**
+ * Sets the 'after' data in the builder.
+ *
+ * @param after The data after the change.
+ * @return The DataChanges builder.
+ */
public Builder withAfter(Object after) {
this.after = after;
return this;
}
+ /**
+ * Sets the 'before' data in the builder.
+ *
+ * @param before The data before the change.
+ * @return The DataChanges builder.
+ */
public Builder withBefore(Object before) {
this.before = before;
return this;
}
+ /**
+ * Builds the DataChanges instance.
+ *
+ * @return The constructed DataChanges.
+ */
public DataChanges build() {
return new DataChanges(after, before, type);
}
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/Field.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/Field.java
index 66d583de73..bdbf7aa686 100644
--- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/Field.java
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/Field.java
@@ -17,6 +17,8 @@
package org.apache.eventmesh.connector.jdbc;
+import org.apache.eventmesh.connector.jdbc.table.catalog.Column;
+
import java.util.List;
import lombok.AllArgsConstructor;
@@ -28,25 +30,25 @@
@AllArgsConstructor
public class Field {
- private String type;
-
private boolean required;
private String field;
private String name;
+ private Column column;
+
private List fields;
- public Field(String type, boolean required, String field, String name) {
- this.type = type;
+ public Field(Column column, boolean required, String field, String name) {
+ this.column = column;
this.required = required;
this.field = field;
this.name = name;
}
- public Field withType(String type) {
- this.type = type;
+ public Field withColumn(Column column) {
+ this.column = column;
return this;
}
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/JdbcConnectData.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/JdbcConnectData.java
index 3dca4f42f4..df749947e7 100644
--- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/JdbcConnectData.java
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/JdbcConnectData.java
@@ -17,17 +17,26 @@
package org.apache.eventmesh.connector.jdbc;
+/**
+ * Represents data associated with a JDBC connector.
+ */
public final class JdbcConnectData {
+ /**
+ * Constant representing data changes in the JDBC connector.
+ */
public static final byte DATA_CHANGES = 1;
+ /**
+ * Constant representing schema changes in the JDBC connector.
+ */
public static final byte SCHEMA_CHANGES = 1 << 1;
private Payload payload = new Payload();
private Schema schema;
- private byte type;
+ private byte type = 0;
public JdbcConnectData() {
}
@@ -67,4 +76,12 @@ public void markDataChanges() {
public void markSchemaChanges() {
this.type |= SCHEMA_CHANGES;
}
+
+ public boolean isDataChanges() {
+ return (this.type & DATA_CHANGES) != 0;
+ }
+
+ public boolean isSchemaChanges() {
+ return (this.type & SCHEMA_CHANGES) != 0;
+ }
}
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/Payload.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/Payload.java
index f9b66d3d53..f861445aa3 100644
--- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/Payload.java
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/Payload.java
@@ -19,53 +19,157 @@
import org.apache.eventmesh.connector.jdbc.source.SourceMateData;
-import java.util.HashMap;
+import lombok.Getter;
+import lombok.Setter;
-public final class Payload extends HashMap {
+/**
+ * Payload class representing the data associated with a JDBC connection.
+ */
+public final class Payload {
+ /**
+ * Field name for the 'after' payload.
+ */
public static final String AFTER_FIELD = "after";
+ /**
+ * Field name for the 'before' payload.
+ */
public static final String BEFORE_FIELD = "before";
+ /**
+ * Field name for the 'source' payload.
+ */
public static final String SOURCE = "source";
- public static final String DDL = "ddl";
+ /**
+ * Field name for the 'payload.before' payload.
+ */
+ public static final String PAYLOAD_BEFORE = "payload.before";
+
+ /**
+ * Field name for the 'payload.after' payload.
+ */
+ public static final String PAYLOAD_AFTER = "payload.after";
+
+ @Getter
+ @Setter
+ private SourceMateData source;
+
+ // Source connector's original DDL script
+ @Getter
+ @Setter
+ private String ddl;
+
+ @Getter
+ @Setter
+ private CatalogChanges catalogChanges;
+
+ @Getter
+ @Setter
+ private DataChanges dataChanges;
+
+ @Getter
+ @Setter
+ private long timestamp;
/**
- * Constructs an empty HashMap with the default initial capacity (16) and the default load factor (0.75).
+ * Constructs an empty Payload with the default initial capacity (16) and the default load factor (0.75).
*/
public Payload() {
- this.put("timestamp", System.currentTimeMillis());
+ this.timestamp = System.currentTimeMillis();
}
- public Payload withSource(SourceMateData source) {
- this.put(SOURCE, source);
+ /**
+ * Sets the 'source' field in the payload.
+ *
+ * @param source The SourceMateData to set.
+ * @return The Payload instance.
+ */
+ public Payload withSource(S source) {
+ this.source = source;
return this;
}
+ /**
+ * Sets the 'ddl' field in the payload.
+ *
+ * @param ddl The DDL string to set.
+ * @return The Payload instance.
+ */
public Payload withDdl(String ddl) {
- this.put(DDL, ddl);
+ this.ddl = ddl;
return this;
}
+ /**
+ * Sets the 'catalogChanges' field in the payload.
+ *
+ * @param catalogChanges The CatalogChanges to set.
+ * @return The Payload instance.
+ */
public Payload withCatalogChanges(CatalogChanges catalogChanges) {
- this.put("catalogChanges", catalogChanges);
+ this.catalogChanges = catalogChanges;
return this;
}
+ /**
+ * Sets the 'dataChanges' field in the payload.
+ *
+ * @param dataChanges The DataChanges to set.
+ * @return The Payload instance.
+ */
public Payload withDataChanges(DataChanges dataChanges) {
- this.put("dataChanges", dataChanges);
+ this.dataChanges = dataChanges;
return this;
}
+ /**
+ * Retrieves the 'source' field from the payload.
+ *
+ * @return The SourceMateData.
+ */
public SourceMateData ofSourceMateData() {
- return (SourceMateData) super.get(SOURCE);
+ return getSource();
+ }
+
+ /**
+ * Retrieves the 'catalogChanges' field from the payload.
+ *
+ * @return The CatalogChanges.
+ */
+ public CatalogChanges ofCatalogChanges() {
+ return getCatalogChanges();
+ }
+
+ /**
+ * Retrieves the 'dataChanges' field from the payload.
+ *
+ * @return The DataChanges.
+ */
+ public DataChanges ofDataChanges() {
+ return getDataChanges();
}
+ /**
+ * Retrieves the 'ddl' field from the payload.
+ *
+ * @return The DDL string.
+ */
+ public String ofDdl() {
+ return getDdl();
+ }
+
+ /**
+ * Builder class for constructing Payload instances.
+ */
public static Builder builder() {
return new Builder();
}
+ /**
+ * Builder class for constructing Payload instances.
+ */
public static class Builder {
private final Payload payload;
@@ -74,19 +178,24 @@ private Builder() {
payload = new Payload();
}
- public Builder put(String key, Object value) {
- payload.put(key, value);
- return this;
- }
-
+ /**
+ * Sets the 'source' field in the payload.
+ *
+ * @param source The SourceMateData to set.
+ * @return The Builder instance.
+ */
public Builder withSource(SourceMateData source) {
- payload.put(SOURCE, source);
+ payload.withSource(source);
return this;
}
+ /**
+ * Builds the Payload instance.
+ *
+ * @return The constructed Payload.
+ */
public Payload build() {
return payload;
}
}
-
}
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/Schema.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/Schema.java
index 084f9759bc..75674bd798 100644
--- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/Schema.java
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/Schema.java
@@ -18,13 +18,19 @@
package org.apache.eventmesh.connector.jdbc;
import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
import lombok.Data;
@Data
public class Schema {
+ // contains primary key and unique key
+ private Set keySet;
+
private List fields;
public Schema(List fields) {
@@ -38,4 +44,22 @@ public Schema() {
public void add(Field field) {
this.fields.add(field);
}
+
+ public void addKey(String key) {
+ if (keySet == null) {
+ keySet = new HashSet<>();
+ }
+ this.keySet.add(key);
+ }
+
+ public void addKeys(Collection keys) {
+ if (keySet == null) {
+ keySet = new HashSet<>();
+ }
+ this.keySet.addAll(keys);
+ }
+
+ public boolean containsKey() {
+ return keySet != null && !keySet.isEmpty();
+ }
}
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/UniversalJdbcContext.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/UniversalJdbcContext.java
index af89638149..e0f656da23 100644
--- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/UniversalJdbcContext.java
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/UniversalJdbcContext.java
@@ -52,9 +52,6 @@ public OffSetCtx getOffsetContext() {
return poCtx.getOffsetContext();
}
- /**
- * @return
- */
@Override
public TableId ofCurrentTableId() {
return tableId;
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/common/EnumeratedValue.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/common/EnumeratedValue.java
new file mode 100644
index 0000000000..484b6a2910
--- /dev/null
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/common/EnumeratedValue.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eventmesh.connector.jdbc.common;
+
+/**
+ * An interface representing an enumerated value.
+ *
+ * @param The type of the enumerated value.
+ */
+public interface EnumeratedValue {
+
+ /**
+ * Gets the value of the enumerated item.
+ *
+ * @return The value of the enumerated item.
+ */
+ T getValue();
+}
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/config/JdbcConfig.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/config/JdbcConfig.java
index 9c7558426c..d40801854c 100644
--- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/config/JdbcConfig.java
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/config/JdbcConfig.java
@@ -49,6 +49,9 @@ public class JdbcConfig {
private int connectTimeout;
+ // e.g jdbc:mysql://127.0.0.1:3306
+ private String url;
+
/**
* Converts the JdbcConfig object to a Properties object containing the user and password.
*
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/connection/JdbcConnection.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/connection/JdbcConnection.java
index 78ddcad8e4..d1802d8b96 100644
--- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/connection/JdbcConnection.java
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/connection/JdbcConnection.java
@@ -32,6 +32,7 @@
import javax.annotation.concurrent.ThreadSafe;
+import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
/**
@@ -45,6 +46,7 @@ public class JdbcConnection implements AutoCloseable {
private static final String STATEMENT_DELIMITER = ";";
+ @Getter
private final JdbcConfig jdbcConfig;
private volatile Connection connection;
@@ -88,15 +90,6 @@ public void close() throws Exception {
}
}
- /**
- * Retrieves the JDBC configuration.
- *
- * @return The JDBC configuration.
- */
- public JdbcConfig getJdbcConfig() {
- return jdbcConfig;
- }
-
/**
* Sets the auto-commit mode for the connection.
*
@@ -223,7 +216,6 @@ public JdbcConnection executeWithoutCommitting(String... sqlStatements) throws S
statement.execute(sqlStatement);
}
}
-
return this;
}
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/dialect/AbstractGeneralDatabaseDialect.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/dialect/AbstractGeneralDatabaseDialect.java
new file mode 100644
index 0000000000..0a7463a187
--- /dev/null
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/dialect/AbstractGeneralDatabaseDialect.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eventmesh.connector.jdbc.dialect;
+
+import org.apache.eventmesh.connector.jdbc.config.JdbcConfig;
+import org.apache.eventmesh.connector.jdbc.connection.JdbcConnection;
+import org.apache.eventmesh.connector.jdbc.exception.JdbcConnectionException;
+import org.apache.eventmesh.connector.jdbc.table.catalog.Column;
+import org.apache.eventmesh.connector.jdbc.table.catalog.TableId;
+import org.apache.eventmesh.connector.jdbc.type.Type;
+import org.apache.eventmesh.connector.jdbc.type.eventmesh.BooleanEventMeshDataType;
+import org.apache.eventmesh.connector.jdbc.type.eventmesh.BytesEventMeshDataType;
+import org.apache.eventmesh.connector.jdbc.type.eventmesh.DateEventMeshDataType;
+import org.apache.eventmesh.connector.jdbc.type.eventmesh.DateTimeEventMeshDataType;
+import org.apache.eventmesh.connector.jdbc.type.eventmesh.DecimalEventMeshDataType;
+import org.apache.eventmesh.connector.jdbc.type.eventmesh.Float32EventMeshDataType;
+import org.apache.eventmesh.connector.jdbc.type.eventmesh.Float64EventMeshDataType;
+import org.apache.eventmesh.connector.jdbc.type.eventmesh.Int16EventMeshDataType;
+import org.apache.eventmesh.connector.jdbc.type.eventmesh.Int32EventMeshDataType;
+import org.apache.eventmesh.connector.jdbc.type.eventmesh.Int64EventMeshDataType;
+import org.apache.eventmesh.connector.jdbc.type.eventmesh.Int8EventMeshDataType;
+import org.apache.eventmesh.connector.jdbc.type.eventmesh.StringEventMeshDataType;
+import org.apache.eventmesh.connector.jdbc.type.eventmesh.TimeEventMeshDataType;
+import org.apache.eventmesh.connector.jdbc.type.eventmesh.YearEventMeshDataType;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+
+import org.hibernate.dialect.Dialect;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public abstract class AbstractGeneralDatabaseDialect implements DatabaseDialect {
+
+ private static final int DEFAULT_BATCH_MAX_ROWS = 20;
+
+ private JdbcConfig config;
+
+ private int batchMaxRows = DEFAULT_BATCH_MAX_ROWS;
+
+ private final Map typeRegisters = new HashMap<>(32);
+
+ private Dialect hibernateDialect;
+
+ public AbstractGeneralDatabaseDialect(JdbcConfig config) {
+ this.config = config;
+ }
+
+ @Override
+ public void configure(Dialect hibernateDialect) {
+ this.hibernateDialect = hibernateDialect;
+ }
+
+ @Override
+ public boolean isValid(Connection connection, int timeout) throws JdbcConnectionException, SQLException {
+ return connection == null ? false : connection.isValid(timeout);
+ }
+
+ @Override
+ public PreparedStatement createPreparedStatement(Connection connection, String sql) throws SQLException {
+ PreparedStatement preparedStatement = connection.prepareStatement(sql);
+ if (batchMaxRows > 0) {
+ preparedStatement.setFetchSize(batchMaxRows);
+ }
+ return preparedStatement;
+ }
+
+ @Override
+ public Type getType(Column column) {
+ final String nativeType = column.getNativeType();
+ if (nativeType != null) {
+ final Type type = typeRegisters.get(nativeType);
+ if (type != null) {
+ log.debug("found type {} for column {}", type.getClass().getName(), column.getName());
+ return type;
+ }
+ }
+ final String dataTypeName = column.getDataType().getName();
+ if (dataTypeName != null) {
+ final Type type = typeRegisters.get(dataTypeName);
+ if (type != null) {
+ log.debug("found type {} for column {}", type.getClass().getName(), column.getName());
+ return type;
+ }
+ }
+
+ final String jdbcTypeName = column.getJdbcType().name();
+ if (jdbcTypeName != null) {
+ final Type type = typeRegisters.get(jdbcTypeName);
+ if (type != null) {
+ log.debug("found type {} for column {}", type.getClass().getName(), column.getName());
+ return type;
+ }
+ }
+
+ return null;
+ }
+
+ protected void registerTypes() {
+ registerType(BooleanEventMeshDataType.INSTANCE);
+ registerType(Float32EventMeshDataType.INSTANCE);
+ registerType(Float64EventMeshDataType.INSTANCE);
+ registerType(Int8EventMeshDataType.INSTANCE);
+ registerType(Int16EventMeshDataType.INSTANCE);
+ registerType(Int32EventMeshDataType.INSTANCE);
+ registerType(Int64EventMeshDataType.INSTANCE);
+ registerType(StringEventMeshDataType.INSTANCE);
+ registerType(DateEventMeshDataType.INSTANCE);
+ registerType(TimeEventMeshDataType.INSTANCE);
+ registerType(DateTimeEventMeshDataType.INSTANCE);
+ registerType(DecimalEventMeshDataType.INSTANCE);
+ registerType(BytesEventMeshDataType.INSTANCE);
+ registerType(YearEventMeshDataType.INSTANCE);
+ }
+
+ protected void registerType(Type type) {
+ type.configure(this, hibernateDialect);
+ Optional.ofNullable(type.ofRegistrationKeys()).orElse(new ArrayList<>(0)).forEach(key -> typeRegisters.put(key, type));
+ }
+
+ public abstract String getQualifiedTableName(TableId tableId);
+
+ public abstract String getQualifiedText(String text);
+
+ @Override
+ public String getTypeName(Dialect hibernateDialect, Column column) {
+ Type type = this.getType(column);
+ if (type != null) {
+ return type.getTypeName(column);
+ }
+ Long length = Optional.ofNullable(column.getColumnLength()).orElse(0L);
+ return hibernateDialect.getTypeName(column.getJdbcType().getVendorTypeNumber(), length, length.intValue(),
+ Optional.ofNullable(column.getDecimal()).orElse(0));
+ }
+}
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/DatabaseDialect.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/dialect/DatabaseDialect.java
similarity index 85%
rename from eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/DatabaseDialect.java
rename to eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/dialect/DatabaseDialect.java
index 11378270a4..86befeae85 100644
--- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/DatabaseDialect.java
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/dialect/DatabaseDialect.java
@@ -15,11 +15,13 @@
* limitations under the License.
*/
-package org.apache.eventmesh.connector.jdbc;
+package org.apache.eventmesh.connector.jdbc.dialect;
+import org.apache.eventmesh.connector.jdbc.JdbcDriverMetaData;
import org.apache.eventmesh.connector.jdbc.connection.JdbcConnection;
import org.apache.eventmesh.connector.jdbc.connection.JdbcConnectionProvider;
import org.apache.eventmesh.connector.jdbc.table.catalog.Catalog;
+import org.apache.eventmesh.connector.jdbc.type.DatabaseTypeDialect;
import java.sql.Connection;
import java.sql.PreparedStatement;
@@ -28,7 +30,7 @@
/**
* Interface for a database dialect, which extends the ConnectionProvider and Catalog interfaces.
*/
-public interface DatabaseDialect extends JdbcConnectionProvider, Catalog {
+public interface DatabaseDialect extends JdbcConnectionProvider, Catalog, DatabaseTypeDialect {
/**
* Initializes the database dialect.
@@ -41,11 +43,11 @@ public interface DatabaseDialect extends JdbcConnecti
void start();
/**
- * Retrieves the name of the database dialect.
+ * Retrieves the type of the database dialect.
*
- * @return The name of the database dialect.
+ * @return The type of the database dialect.
*/
- String getName();
+ DatabaseType getDatabaseType();
/**
* Creates a prepared statement for the given SQL query using the provided database connection.
@@ -70,4 +72,5 @@ public interface DatabaseDialect extends JdbcConnecti
* @return The JDBC protocol.
*/
String jdbcProtocol();
+
}
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/DatabaseDialectFactory.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/dialect/DatabaseDialectFactory.java
similarity index 79%
rename from eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/DatabaseDialectFactory.java
rename to eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/dialect/DatabaseDialectFactory.java
index 8f73f9f53e..aad7984520 100644
--- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/DatabaseDialectFactory.java
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/dialect/DatabaseDialectFactory.java
@@ -15,10 +15,9 @@
* limitations under the License.
*/
-package org.apache.eventmesh.connector.jdbc.source.dialect;
+package org.apache.eventmesh.connector.jdbc.dialect;
-import org.apache.eventmesh.connector.jdbc.DatabaseDialect;
-import org.apache.eventmesh.openconnect.api.config.SourceConfig;
+import org.apache.eventmesh.connector.jdbc.config.JdbcConfig;
import org.apache.eventmesh.spi.EventMeshExtensionType;
import org.apache.eventmesh.spi.EventMeshSPI;
@@ -31,9 +30,9 @@ public interface DatabaseDialectFactory {
/**
* Creates a database dialect based on the provided source configuration.
*
- * @param config the source configuration to create a database dialect for
+ * @param config the jdbc configuration to create a database dialect for
* @return the created database dialect
*/
- DatabaseDialect createDatabaseDialect(SourceConfig config);
+ DatabaseDialect createDatabaseDialect(JdbcConfig config);
}
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/DatabaseType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/dialect/DatabaseType.java
similarity index 83%
rename from eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/DatabaseType.java
rename to eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/dialect/DatabaseType.java
index 2e56932004..b1db56c526 100644
--- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/DatabaseType.java
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/dialect/DatabaseType.java
@@ -15,24 +15,27 @@
* limitations under the License.
*/
-package org.apache.eventmesh.connector.jdbc.source.dialect;
+package org.apache.eventmesh.connector.jdbc.dialect;
import org.apache.commons.lang3.StringUtils;
+import lombok.Getter;
+
public enum DatabaseType {
MYSQL("mysql");
- private String name;
+ @Getter
+ private String code;
- DatabaseType(String name) {
- this.name = name;
+ DatabaseType(String code) {
+ this.code = code;
}
public static DatabaseType ofValue(String name) {
DatabaseType[] databaseTypes = values();
for (DatabaseType databaseType : databaseTypes) {
- if (StringUtils.equalsIgnoreCase(databaseType.name, name)) {
+ if (StringUtils.equalsIgnoreCase(databaseType.code, name)) {
return databaseType;
}
}
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/dialect/SqlStatementAssembler.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/dialect/SqlStatementAssembler.java
new file mode 100644
index 0000000000..6e3858369a
--- /dev/null
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/dialect/SqlStatementAssembler.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eventmesh.connector.jdbc.dialect;
+
+import org.apache.eventmesh.connector.jdbc.table.catalog.Column;
+
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.function.Function;
+
+/**
+ * The {@code SqlStatementAssembler} class is used to assemble SQL statements by appending SQL slices.
+ */
+public final class SqlStatementAssembler {
+
+ private final StringBuilder statement;
+
+ public SqlStatementAssembler() {
+ statement = new StringBuilder();
+ }
+
+ public SqlStatementAssembler appendSqlSlice(String slice) {
+ statement.append(slice);
+ return this;
+ }
+
+ public SqlStatementAssembler appendSqlSliceLists(String delimiter, Collection columnNames, Function function) {
+ for (Iterator iterator = columnNames.iterator(); iterator.hasNext();) {
+ statement.append(function.apply(iterator.next()));
+ if (iterator.hasNext()) {
+ statement.append(delimiter);
+ }
+ }
+ return this;
+ }
+
+ public SqlStatementAssembler appendSqlSliceOfColumns(String delimiter, Collection> columns, Function, String> function) {
+ for (Iterator> iterator = columns.iterator(); iterator.hasNext();) {
+ statement.append(function.apply(iterator.next()));
+ if (iterator.hasNext()) {
+ statement.append(delimiter);
+ }
+ }
+ return this;
+ }
+
+ public String build() {
+ return statement.toString();
+ }
+
+}
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/mysql/MysqlDatabaseDialect.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/dialect/mysql/MysqlDatabaseDialect.java
similarity index 62%
rename from eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/mysql/MysqlDatabaseDialect.java
rename to eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/dialect/mysql/MysqlDatabaseDialect.java
index bfd6a73006..acd8730c2e 100644
--- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/mysql/MysqlDatabaseDialect.java
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/dialect/mysql/MysqlDatabaseDialect.java
@@ -15,22 +15,52 @@
* limitations under the License.
*/
-package org.apache.eventmesh.connector.jdbc.source.dialect.mysql;
+package org.apache.eventmesh.connector.jdbc.dialect.mysql;
import org.apache.eventmesh.connector.jdbc.DataTypeConvertor;
import org.apache.eventmesh.connector.jdbc.JdbcDriverMetaData;
+import org.apache.eventmesh.connector.jdbc.config.JdbcConfig;
import org.apache.eventmesh.connector.jdbc.connection.mysql.MysqlJdbcConnection;
+import org.apache.eventmesh.connector.jdbc.dialect.AbstractGeneralDatabaseDialect;
+import org.apache.eventmesh.connector.jdbc.dialect.DatabaseType;
import org.apache.eventmesh.connector.jdbc.exception.CatalogException;
import org.apache.eventmesh.connector.jdbc.exception.DatabaseNotExistException;
import org.apache.eventmesh.connector.jdbc.exception.TableNotExistException;
-import org.apache.eventmesh.connector.jdbc.source.config.JdbcSourceConfig;
-import org.apache.eventmesh.connector.jdbc.source.config.SourceConnectorConfig;
-import org.apache.eventmesh.connector.jdbc.source.dialect.AbstractGeneralDatabaseDialect;
+import org.apache.eventmesh.connector.jdbc.source.dialect.mysql.MysqlDataTypeConvertor;
+import org.apache.eventmesh.connector.jdbc.source.dialect.mysql.MysqlDialectSql;
import org.apache.eventmesh.connector.jdbc.table.catalog.CatalogTable;
+import org.apache.eventmesh.connector.jdbc.table.catalog.Column;
import org.apache.eventmesh.connector.jdbc.table.catalog.DefaultColumn;
+import org.apache.eventmesh.connector.jdbc.table.catalog.Options;
import org.apache.eventmesh.connector.jdbc.table.catalog.PrimaryKey;
+import org.apache.eventmesh.connector.jdbc.table.catalog.Table;
import org.apache.eventmesh.connector.jdbc.table.catalog.TableId;
import org.apache.eventmesh.connector.jdbc.table.catalog.TableSchema;
+import org.apache.eventmesh.connector.jdbc.table.catalog.mysql.MysqlColumn;
+import org.apache.eventmesh.connector.jdbc.table.catalog.mysql.MysqlOptions.MysqlTableOptions;
+import org.apache.eventmesh.connector.jdbc.type.Type;
+import org.apache.eventmesh.connector.jdbc.type.mysql.BitType;
+import org.apache.eventmesh.connector.jdbc.type.mysql.BytesType;
+import org.apache.eventmesh.connector.jdbc.type.mysql.DecimalType;
+import org.apache.eventmesh.connector.jdbc.type.mysql.EnumType;
+import org.apache.eventmesh.connector.jdbc.type.mysql.GeometryCollectionType;
+import org.apache.eventmesh.connector.jdbc.type.mysql.GeometryType;
+import org.apache.eventmesh.connector.jdbc.type.mysql.IntType;
+import org.apache.eventmesh.connector.jdbc.type.mysql.JsonType;
+import org.apache.eventmesh.connector.jdbc.type.mysql.LineStringType;
+import org.apache.eventmesh.connector.jdbc.type.mysql.MediumintType;
+import org.apache.eventmesh.connector.jdbc.type.mysql.MultiLineStringType;
+import org.apache.eventmesh.connector.jdbc.type.mysql.MultiPointType;
+import org.apache.eventmesh.connector.jdbc.type.mysql.MultiPolygonType;
+import org.apache.eventmesh.connector.jdbc.type.mysql.PointType;
+import org.apache.eventmesh.connector.jdbc.type.mysql.PolygonType;
+import org.apache.eventmesh.connector.jdbc.type.mysql.SetType;
+import org.apache.eventmesh.connector.jdbc.type.mysql.TextType;
+import org.apache.eventmesh.connector.jdbc.type.mysql.TinyIntType;
+import org.apache.eventmesh.connector.jdbc.type.mysql.YearType;
+import org.apache.eventmesh.connector.jdbc.utils.MysqlUtils;
+
+import org.apache.commons.lang3.StringUtils;
import java.sql.Connection;
import java.sql.PreparedStatement;
@@ -48,17 +78,17 @@
import lombok.extern.slf4j.Slf4j;
@Slf4j
-public class MysqlDatabaseDialect extends AbstractGeneralDatabaseDialect {
+public class MysqlDatabaseDialect extends AbstractGeneralDatabaseDialect {
private MysqlJdbcConnection connection;
private DataTypeConvertor dataTypeConvertor = new MysqlDataTypeConvertor();
- private SourceConnectorConfig config;
+ private JdbcConfig config;
- public MysqlDatabaseDialect(JdbcSourceConfig config) {
- super(config.getSourceConnectorConfig());
- this.config = config.getSourceConnectorConfig();
+ public MysqlDatabaseDialect(JdbcConfig config) {
+ super(config);
+ this.config = config;
}
@Override
@@ -80,6 +110,31 @@ public void init() {
}
} while (!initSuccess);
+ // handle type register
+ super.registerTypes();
+ registerType(BitType.INSTANCE);
+ registerType(SetType.INSTANCE);
+ registerType(EnumType.INSTANCE);
+ registerType(TinyIntType.INSTANCE);
+ registerType(JsonType.INSTANCE);
+ registerType(IntType.INSTANCE);
+ registerType(MediumintType.INSTANCE);
+ registerType(DecimalType.INSTANCE);
+ registerType(TextType.INSTANCE);
+
+ // override YearEventMeshDateType
+ registerType(YearType.INSTANCE);
+ registerType(BytesType.INSTANCE);
+
+ // Spatial Data Types
+ registerType(PointType.INSTANCE);
+ registerType(MultiPointType.INSTANCE);
+ registerType(GeometryType.INSTANCE);
+ registerType(GeometryCollectionType.INSTANCE);
+ registerType(LineStringType.INSTANCE);
+ registerType(MultiLineStringType.INSTANCE);
+ registerType(PolygonType.INSTANCE);
+ registerType(MultiPolygonType.INSTANCE);
}
@Override
@@ -89,7 +144,7 @@ public void start() {
private MysqlJdbcConnection initJdbcConnection() {
try {
- return new MysqlJdbcConnection(config.getJdbcConfig(), null, false);
+ return new MysqlJdbcConnection(config, null, false);
} catch (Exception e) {
throw new CatalogException(e);
}
@@ -107,7 +162,7 @@ public String getDefaultDatabase() {
@Override
public boolean databaseExists(String databaseName) throws CatalogException {
- if (null == databaseName || databaseName.trim().isEmpty()) {
+ if (databaseName == null || databaseName.trim().isEmpty()) {
return false;
}
List databases = listDatabases();
@@ -196,7 +251,7 @@ public CatalogTable getTable(TableId tableId) throws CatalogException, TableNotE
DefaultColumn column = columns.computeIfAbsent(columnName, key -> new DefaultColumn());
column.setName(columnName);
int precision = tableMetaData.getPrecision(columnIndex);
- column.setColumnLength(precision);
+ // column.setColumnLength(precision);
Map dataTypeProperties = new HashMap<>();
dataTypeProperties.put(MysqlDataTypeConvertor.PRECISION, precision);
int scale = tableMetaData.getScale(columnIndex);
@@ -250,8 +305,8 @@ public CatalogTable getTable(TableId tableId) throws CatalogException, TableNotE
}
@Override
- public String getName() {
- return null;
+ public DatabaseType getDatabaseType() {
+ return DatabaseType.MYSQL;
}
@Override
@@ -261,6 +316,16 @@ public PreparedStatement createPreparedStatement(Connection connection, String s
return connection.prepareStatement(sql);
}
+ @Override
+ public String getQualifiedTableName(TableId tableId) {
+ return MysqlUtils.wrapper(tableId);
+ }
+
+ @Override
+ public String getQualifiedText(String text) {
+ return MysqlUtils.wrapper(text);
+ }
+
/**
* Retrieves the JDBC driver meta-data associated with the database dialect.
*
@@ -302,4 +367,72 @@ public void close() throws Exception {
this.connection.close();
}
}
+
+ @Override
+ public String getAutoIncrementFormatted(Column column) {
+ return " AUTO_INCREMENT ";
+ }
+
+ @Override
+ public String getDefaultValueFormatted(Column column) {
+ Type type = this.getType(column);
+ String defaultValue = type.getDefaultValue(this, column);
+ return defaultValue;
+ }
+
+ @Override
+ public String getCharsetOrCollateFormatted(Column column) {
+ StringBuilder builder = new StringBuilder();
+ String charsetName = column.getCharsetName();
+ if (StringUtils.isNotBlank(charsetName)) {
+ builder.append(" CHARACTER SET ").append(charsetName).append(" ");
+ }
+ String collationName = column.getCollationName();
+ if (StringUtils.isNotBlank(collationName)) {
+ builder.append(" COLLATE ").append(collationName).append(" ");
+ }
+
+ return builder.toString();
+ }
+
+ @Override
+ public String getTableOptionsFormatted(Table table) {
+
+ Options options = table.getOptions();
+ if (Objects.isNull(options) || options.isEmpty()) {
+ return EMPTY_STRING;
+ }
+ StringBuilder builder = new StringBuilder();
+ String engine = (String) options.get(MysqlTableOptions.ENGINE);
+ if (StringUtils.isNotBlank(engine)) {
+ builder.append(String.format("ENGINE=%s ", engine));
+ }
+ String autoIncrementNumber = (String) options.get(MysqlTableOptions.AUTO_INCREMENT);
+ if (StringUtils.isNotBlank(autoIncrementNumber)) {
+ builder.append(String.format("AUTO_INCREMENT=%s ", autoIncrementNumber));
+ }
+ String charset = (String) options.get(MysqlTableOptions.CHARSET);
+ if (StringUtils.isNotBlank(charset)) {
+ builder.append(String.format("DEFAULT CHARSET=%s ", charset));
+ }
+
+ String collate = (String) options.get(MysqlTableOptions.COLLATE);
+ if (StringUtils.isNotBlank(collate)) {
+ builder.append(String.format(" COLLATE=%s ", collate));
+ }
+
+ String comment = table.getComment();
+ if (StringUtils.isNotBlank(comment)) {
+ builder.append(String.format(" COMMENT='%s' ", comment));
+ }
+ return builder.toString();
+ }
+
+ @Override
+ public String getCommentFormatted(Column column) {
+ if (StringUtils.isEmpty(column.getComment())) {
+ return EMPTY_STRING;
+ }
+ return "COMMENT '" + column.getComment() + "'";
+ }
}
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/mysql/MysqlDatabaseDialectFactory.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/dialect/mysql/MysqlDatabaseDialectFactory.java
similarity index 68%
rename from eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/mysql/MysqlDatabaseDialectFactory.java
rename to eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/dialect/mysql/MysqlDatabaseDialectFactory.java
index 5f34a8a40f..eb4fbe3275 100644
--- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/mysql/MysqlDatabaseDialectFactory.java
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/dialect/mysql/MysqlDatabaseDialectFactory.java
@@ -15,18 +15,17 @@
* limitations under the License.
*/
-package org.apache.eventmesh.connector.jdbc.source.dialect.mysql;
+package org.apache.eventmesh.connector.jdbc.dialect.mysql;
-import org.apache.eventmesh.connector.jdbc.DatabaseDialect;
-import org.apache.eventmesh.connector.jdbc.source.config.JdbcSourceConfig;
-import org.apache.eventmesh.connector.jdbc.source.dialect.DatabaseDialectFactory;
-import org.apache.eventmesh.openconnect.api.config.SourceConfig;
+import org.apache.eventmesh.connector.jdbc.config.JdbcConfig;
+import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialect;
+import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialectFactory;
public class MysqlDatabaseDialectFactory implements DatabaseDialectFactory {
@Override
- public DatabaseDialect createDatabaseDialect(SourceConfig config) {
- DatabaseDialect databaseDialect = new MysqlDatabaseDialect((JdbcSourceConfig) config);
+ public DatabaseDialect createDatabaseDialect(JdbcConfig config) {
+ DatabaseDialect databaseDialect = new MysqlDatabaseDialect(config);
return databaseDialect;
}
}
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/AbstractEvent.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/AbstractEvent.java
index a5c7adf985..a2aaf6a5e4 100644
--- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/AbstractEvent.java
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/AbstractEvent.java
@@ -46,9 +46,6 @@ public TableId getTableId() {
return tableId;
}
- /**
- * @return
- */
@Override
public JdbcConnectData getJdbcConnectData() {
return data;
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/DataChangeEventType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/DataChangeEventType.java
index be79cb3ebf..67051603ca 100644
--- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/DataChangeEventType.java
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/DataChangeEventType.java
@@ -17,17 +17,59 @@
package org.apache.eventmesh.connector.jdbc.event;
+/**
+ * Enumeration representing different types of data change events.
+ */
public enum DataChangeEventType {
- INSERT("I"), UPDATE("U"), DELETE("D");
+ /**
+ * Represents an INSERT data change event.
+ */
+ INSERT("I"),
+
+ /**
+ * Represents an UPDATE data change event.
+ */
+ UPDATE("U"),
+
+ /**
+ * Represents a DELETE data change event.
+ */
+ DELETE("D");
+
private final String code;
+ /**
+ * Constructs a DataChangeEventType with the specified code.
+ *
+ * @param code The code representing the data change event type.
+ */
DataChangeEventType(String code) {
this.code = code;
}
+ /**
+ * Parses a DataChangeEventType from the given code.
+ *
+ * @param code The code to parse.
+ * @return The corresponding DataChangeEventType.
+ * @throws IllegalArgumentException If the provided code is unknown.
+ */
+ public static DataChangeEventType parseFromCode(String code) {
+ for (DataChangeEventType type : DataChangeEventType.values()) {
+ if (type.code.equals(code)) {
+ return type;
+ }
+ }
+ throw new IllegalArgumentException("Unknown DataChangeEventType code: " + code);
+ }
+
+ /**
+ * Gets the code representing the DataChangeEventType.
+ *
+ * @return The code of the DataChangeEventType.
+ */
public String ofCode() {
return this.code;
}
-
}
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/SchemaChangeEventType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/SchemaChangeEventType.java
index fbf51ef2e8..19558f2dfe 100644
--- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/SchemaChangeEventType.java
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/SchemaChangeEventType.java
@@ -17,6 +17,8 @@
package org.apache.eventmesh.connector.jdbc.event;
+import org.apache.commons.lang3.StringUtils;
+
public enum SchemaChangeEventType {
DATABASE_CREATE("D", "C"),
@@ -41,4 +43,14 @@ public String ofType() {
public String ofOperationType() {
return this.operationType;
}
+
+ public static SchemaChangeEventType ofSchemaChangeEventType(String type, String operationType) {
+ SchemaChangeEventType[] types = values();
+ for (SchemaChangeEventType eventType : types) {
+ if (StringUtils.equalsIgnoreCase(eventType.type, type) && StringUtils.equalsIgnoreCase(eventType.operationType, operationType)) {
+ return eventType;
+ }
+ }
+ return null;
+ }
}
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/server/JdbcConnectorServer.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/server/JdbcConnectorServer.java
index 0309aa3442..4c40370671 100644
--- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/server/JdbcConnectorServer.java
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/server/JdbcConnectorServer.java
@@ -18,6 +18,7 @@
package org.apache.eventmesh.connector.jdbc.server;
import org.apache.eventmesh.connector.jdbc.config.JdbcServerConfig;
+import org.apache.eventmesh.connector.jdbc.sink.JdbcSinkConnector;
import org.apache.eventmesh.connector.jdbc.source.JdbcSourceConnector;
import org.apache.eventmesh.openconnect.Application;
import org.apache.eventmesh.openconnect.util.ConfigUtil;
@@ -36,7 +37,8 @@ public static void main(String[] args) throws Exception {
}
if (serverConfig.isSinkEnable()) {
- // TODO support
+ Application jdbcSinkApp = new Application();
+ jdbcSinkApp.run(JdbcSinkConnector.class);
}
}
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/JdbcSinkConnector.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/JdbcSinkConnector.java
new file mode 100644
index 0000000000..7a5c68f581
--- /dev/null
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/JdbcSinkConnector.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eventmesh.connector.jdbc.sink;
+
+import org.apache.eventmesh.common.utils.JsonUtils;
+import org.apache.eventmesh.connector.jdbc.JdbcConnectData;
+import org.apache.eventmesh.connector.jdbc.config.JdbcConfig;
+import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialect;
+import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialectFactory;
+import org.apache.eventmesh.connector.jdbc.sink.config.JdbcSinkConfig;
+import org.apache.eventmesh.connector.jdbc.sink.handle.DefaultSinkRecordHandler;
+import org.apache.eventmesh.connector.jdbc.sink.handle.SinkRecordHandler;
+import org.apache.eventmesh.connector.jdbc.sink.hibernate.HibernateConfiguration;
+import org.apache.eventmesh.connector.jdbc.source.JdbcAllFactoryLoader;
+import org.apache.eventmesh.openconnect.api.config.Config;
+import org.apache.eventmesh.openconnect.api.connector.ConnectorContext;
+import org.apache.eventmesh.openconnect.api.connector.SinkConnectorContext;
+import org.apache.eventmesh.openconnect.api.sink.Sink;
+import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord;
+
+import java.util.List;
+
+import org.hibernate.SessionFactory;
+import org.hibernate.dialect.Dialect;
+import org.hibernate.engine.spi.SessionFactoryImplementor;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class JdbcSinkConnector implements Sink {
+
+ private JdbcSinkConfig sinkConfig;
+
+ private SessionFactory sessionFactory;
+
+ private DatabaseDialect databaseDialect;
+
+ private SinkRecordHandler sinkRecordHandler;
+
+ /**
+ * Returns the class type of the configuration for this Connector.
+ *
+ * @return Class type of the configuration
+ */
+ @Override
+ public Class configClass() {
+ return JdbcSinkConfig.class;
+ }
+
+ /**
+ * Initializes the Connector with the provided configuration.
+ *
+ * @param config Configuration object
+ * @throws Exception if initialization fails
+ */
+ @Override
+ public void init(Config config) throws Exception {
+ if (!(config instanceof JdbcSinkConfig)) {
+ throw new IllegalArgumentException("Config not be JdbcSinkConfig");
+ }
+ this.sinkConfig = (JdbcSinkConfig) config;
+ doInit();
+ }
+
+ /**
+ * Initializes the Connector with the provided context.
+ *
+ * @param connectorContext connectorContext
+ * @throws Exception if initialization fails
+ */
+ @Override
+ public void init(ConnectorContext connectorContext) throws Exception {
+ SinkConnectorContext sinkConnectorContext = (SinkConnectorContext) connectorContext;
+ this.sinkConfig = (JdbcSinkConfig) sinkConnectorContext.getSinkConfig();
+ doInit();
+ }
+
+ private void doInit() {
+ JdbcConfig jdbcConfig = this.sinkConfig.getSinkConnectorConfig().getJdbcConfig();
+ this.sessionFactory = HibernateConfiguration.newBuilder().withDruidMaxActive("20").withPassword(jdbcConfig.getPassword())
+ .withUrl(jdbcConfig.getUrl())
+ .withShowSql(true)
+ .withUser(jdbcConfig.getUser()).build();
+
+ String databaseType = this.sinkConfig.getSinkConnectorConfig().getDatabaseType();
+
+ // Get the database dialect factory and create the database dialect.
+ final DatabaseDialectFactory databaseDialectFactory = JdbcAllFactoryLoader.getDatabaseDialectFactory(databaseType);
+ this.databaseDialect = databaseDialectFactory.createDatabaseDialect(this.sinkConfig.getSinkConnectorConfig().getJdbcConfig());
+ Dialect dialect = this.sessionFactory.unwrap(SessionFactoryImplementor.class).getJdbcServices().getDialect();
+ this.databaseDialect.configure(dialect);
+ this.databaseDialect.init();
+ this.sinkRecordHandler = new DefaultSinkRecordHandler(databaseDialect, sessionFactory, sinkConfig);
+
+ }
+
+ /**
+ * Starts the Connector.
+ *
+ * @throws Exception if the start operation fails
+ */
+ @Override
+ public void start() throws Exception {
+
+ }
+
+ /**
+ * Commits the specified ConnectRecord object.
+ *
+ * @param record ConnectRecord object to commit
+ */
+ @Override
+ public void commit(ConnectRecord record) {
+
+ }
+
+ /**
+ * Returns the name of the Connector.
+ *
+ * @return String name of the Connector
+ */
+ @Override
+ public String name() {
+ return this.sinkConfig.getSinkConnectorConfig().getConnectorName();
+ }
+
+ /**
+ * Stops the Connector.
+ *
+ * @throws Exception if stopping fails
+ */
+ @Override
+ public void stop() throws Exception {
+
+ }
+
+ @Override
+ public void put(List sinkRecords) {
+
+ for (ConnectRecord record : sinkRecords) {
+ Object data = record.getData();
+ try {
+ JdbcConnectData jdbcConnectData = JsonUtils.parseObject((byte[]) data, JdbcConnectData.class);
+ this.sinkRecordHandler.handle(jdbcConnectData);
+ } catch (Exception e) {
+ log.error("Handle ConnectRecord error", e);
+ }
+ }
+ }
+}
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/config/JdbcSinkConfig.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/config/JdbcSinkConfig.java
new file mode 100644
index 0000000000..2a5af32cad
--- /dev/null
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/config/JdbcSinkConfig.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eventmesh.connector.jdbc.sink.config;
+
+import org.apache.eventmesh.openconnect.api.config.SinkConfig;
+
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+
+@Data
+@EqualsAndHashCode(callSuper = true)
+public class JdbcSinkConfig extends SinkConfig {
+
+ private boolean supportUpsert = true;
+
+ private boolean supportDelete = true;
+
+ public SinkConnectorConfig sinkConnectorConfig;
+
+}
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/config/SinkConnectorConfig.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/config/SinkConnectorConfig.java
new file mode 100644
index 0000000000..e971b7b000
--- /dev/null
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/config/SinkConnectorConfig.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eventmesh.connector.jdbc.sink.config;
+
+import org.apache.eventmesh.connector.jdbc.config.JdbcConfig;
+
+import lombok.Data;
+
+/**
+ * Configuration parameters for a sink connector.
+ */
+@Data
+public class SinkConnectorConfig {
+
+ /**
+ * The name of the sink connector.
+ */
+ private String connectorName;
+
+ /**
+ * JDBC configuration for connecting to a database.
+ */
+ private JdbcConfig jdbcConfig;
+
+ public String getDatabaseType() {
+ return "mysql";
+ }
+}
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/handle/DefaultSinkRecordHandler.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/handle/DefaultSinkRecordHandler.java
new file mode 100644
index 0000000000..a4ba77ae5d
--- /dev/null
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/handle/DefaultSinkRecordHandler.java
@@ -0,0 +1,333 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eventmesh.connector.jdbc.sink.handle;
+
+import org.apache.eventmesh.common.utils.LogUtil;
+import org.apache.eventmesh.connector.jdbc.CatalogChanges;
+import org.apache.eventmesh.connector.jdbc.DataChanges;
+import org.apache.eventmesh.connector.jdbc.Field;
+import org.apache.eventmesh.connector.jdbc.JdbcConnectData;
+import org.apache.eventmesh.connector.jdbc.Payload;
+import org.apache.eventmesh.connector.jdbc.Schema;
+import org.apache.eventmesh.connector.jdbc.common.EnumeratedValue;
+import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialect;
+import org.apache.eventmesh.connector.jdbc.event.DataChangeEventType;
+import org.apache.eventmesh.connector.jdbc.event.SchemaChangeEventType;
+import org.apache.eventmesh.connector.jdbc.sink.config.JdbcSinkConfig;
+import org.apache.eventmesh.connector.jdbc.source.SourceMateData;
+import org.apache.eventmesh.connector.jdbc.table.catalog.Column;
+import org.apache.eventmesh.connector.jdbc.type.Type;
+
+import org.apache.commons.collections4.CollectionUtils;
+
+import java.sql.SQLException;
+import java.util.Comparator;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+
+import org.hibernate.SessionFactory;
+import org.hibernate.StatelessSession;
+import org.hibernate.Transaction;
+import org.hibernate.dialect.Dialect;
+import org.hibernate.engine.spi.SessionFactoryImplementor;
+import org.hibernate.query.NativeQuery;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class DefaultSinkRecordHandler implements SinkRecordHandler {
+
+ protected DatabaseDialect eventMeshDialect;
+
+ protected Dialect hibernateDialect;
+
+ protected DialectAssemblyLine dialectAssemblyLine;
+
+ private SessionFactory sessionFactory;
+
+ private final StatelessSession session;
+
+ private final JdbcSinkConfig jdbcSinkConfig;
+
+ public DefaultSinkRecordHandler(DatabaseDialect eventMeshDialect, SessionFactory sessionFactory, JdbcSinkConfig jdbcSinkConfig) {
+ this.eventMeshDialect = eventMeshDialect;
+ this.sessionFactory = sessionFactory;
+ this.hibernateDialect = sessionFactory.unwrap(SessionFactoryImplementor.class).getJdbcServices().getDialect();
+ this.session = this.sessionFactory.openStatelessSession();
+ this.dialectAssemblyLine = DialectAssemblyLineFactory.build(eventMeshDialect, hibernateDialect);
+ this.jdbcSinkConfig = jdbcSinkConfig;
+ }
+
+ /**
+ * Handles schema and data changes using the specified JDBC connection data. The method determines the type of changes (schema or data) and
+ * performs the necessary operations accordingly.
+ *
+ * @param connectData the JDBC connection data
+ * @throws Exception if an error occurs during the handling of schema or data changes
+ */
+ @Override
+ public void handle(JdbcConnectData connectData) throws Exception {
+ Payload payload = connectData.getPayload();
+ SourceMateData sourceMateData = payload.ofSourceMateData();
+ if (connectData.isSchemaChanges()) {
+ //DDL
+ schemaChangeHandle(sourceMateData, payload);
+ } else if (connectData.isDataChanges()) {
+ //DML
+ dataChangesHandle(connectData, sourceMateData, payload);
+ } else {
+ log.warn("Unknown connect data type: {}", connectData.getType());
+ }
+ }
+
+ private void dataChangesHandle(JdbcConnectData connectData, SourceMateData sourceMateData, Payload payload) throws SQLException {
+ String sql;
+ // If the connectData requests for data changes
+ // Parse the data change event type from the payload
+ DataHandleMode dataHandleMode = convert2DataHandleMode(
+ DataChangeEventType.parseFromCode(connectData.getPayload().getDataChanges().getType()));
+ // Depending on the type of the data change event, handle INSERT, UPDATE or DELETE operations
+ switch (dataHandleMode) {
+ case INSERT:
+ // For INSERT event, create an insert statement and execute it
+ sql = this.dialectAssemblyLine.getInsertStatement(sourceMateData, connectData.getSchema(), payload.ofDdl());
+ insert(sql, connectData.getSchema(), payload.ofDataChanges());
+ break;
+ case UPDATE:
+ sql = this.dialectAssemblyLine.getUpdateStatement(sourceMateData, connectData.getSchema(), payload.ofDdl());
+ update(sql, connectData.getSchema(), payload.ofDataChanges());
+ break;
+ case UPSERT:
+ sql = this.dialectAssemblyLine.getUpsertStatement(sourceMateData, connectData.getSchema(), payload.ofDdl());
+ upsert(sql, connectData.getSchema(), payload.ofDataChanges());
+ break;
+ case DELETE:
+ // If support for DELETE is set, create a delete statement and execute it
+ if (jdbcSinkConfig.isSupportDelete()) {
+ sql = this.dialectAssemblyLine.getDeleteStatement(sourceMateData, connectData.getSchema(), payload.ofDdl());
+ delete(sql, connectData.getSchema(), payload.ofDataChanges());
+ } else {
+ log.warn("No support for DELETE");
+ }
+ break;
+ case NONE:
+ log.warn("No data changes to handle");
+ break;
+ default:
+ log.warn("Unknown data changes type: {}", connectData.getPayload().getDataChanges().getType());
+ break;
+ }
+ }
+
+ private void schemaChangeHandle(SourceMateData sourceMateData, Payload payload) throws SQLException {
+ final CatalogChanges catalogChanges = payload.ofCatalogChanges();
+ final SchemaChangeEventType schemaChangeEventType = SchemaChangeEventType.ofSchemaChangeEventType(catalogChanges.getType(),
+ catalogChanges.getOperationType());
+ if (schemaChangeEventType == SchemaChangeEventType.DATABASE_CREATE && this.eventMeshDialect.databaseExists(
+ catalogChanges.getCatalog().getName())) {
+ log.warn("Database {} already exists", catalogChanges.getCatalog().getName());
+ return;
+ }
+ if (schemaChangeEventType == SchemaChangeEventType.TABLE_CREATE && this.eventMeshDialect.tableExists(
+ catalogChanges.getTable().getTableId())) {
+ log.warn("Table {} already exists", catalogChanges.getTable().getTableId());
+ return;
+ }
+ // Create a SQL statement for database or table changes
+ String sql = this.dialectAssemblyLine.getDatabaseOrTableStatement(sourceMateData, catalogChanges, payload.ofDdl());
+ // Apply the database and table changes with the created SQL statement
+ applyDatabaseAndTableChanges(sql);
+ }
+
+ private void applyDatabaseAndTableChanges(String sql) {
+ Transaction transaction = session.beginTransaction();
+ try {
+ LogUtil.debug(log, "Execute database/table sql: {}", () -> sql);
+ session.createNativeQuery(sql).executeUpdate();
+ transaction.commit();
+ } catch (Exception e) {
+ transaction.rollback();
+ throw new RuntimeException(e);
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private void insert(String sql, Schema schema, DataChanges dataChanges) throws SQLException {
+ final Transaction transaction = session.beginTransaction();
+ try {
+ if (log.isDebugEnabled()) {
+ log.debug("execute insert sql: {}", sql);
+ }
+ final NativeQuery query = session.createNativeQuery(sql);
+ AtomicInteger index = new AtomicInteger(1);
+ Map dataChangesAfter = (Map) dataChanges.getAfter();
+ Field after = schema.getFields().get(0);
+ after.getFields().stream().map(field -> field.getColumn()).sorted(Comparator.comparingInt(Column::getOrder)).forEach(column -> {
+ Type type = eventMeshDialect.getType(column);
+ final int bindValueNum = type.bindValue(index.get(), type.convert2DatabaseTypeValue(dataChangesAfter.get(column.getName())), query);
+ index.addAndGet(bindValueNum);
+ });
+ final int result = query.executeUpdate();
+ if (result != 1) {
+ throw new SQLException("Failed to insert row from table");
+ }
+ transaction.commit();
+ } catch (SQLException e) {
+ transaction.rollback();
+ throw e;
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private void update(String sql, Schema schema, DataChanges dataChanges) throws SQLException {
+ final Transaction transaction = session.beginTransaction();
+ try {
+ if (log.isDebugEnabled()) {
+ log.debug("execute update sql: {}", sql);
+ }
+ final NativeQuery query = session.createNativeQuery(sql);
+ AtomicInteger index = new AtomicInteger(1);
+ Map dataChangesAfter = (Map) dataChanges.getAfter();
+ Field after = schema.getFields().get(0);
+ final Map> columnMap = after.getFields().stream().map(field -> field.getColumn())
+ .collect(Collectors.toMap(Column::getName, column -> column));
+ final Set keySet = schema.getKeySet();
+ after.getFields().stream().map(field -> field.getColumn()).sorted(Comparator.comparingInt(Column::getOrder))
+ .filter(column -> !keySet.contains(column.getName())).forEach(column -> {
+ Type type = eventMeshDialect.getType(column);
+ int bindValueNum = type.bindValue(index.get(), type.convert2DatabaseTypeValue(dataChangesAfter.get(column.getName())), query);
+ index.addAndGet(bindValueNum);
+ });
+ schema.getKeySet().stream().forEach(key -> {
+ if (columnMap.containsKey(key)) {
+ Type type = eventMeshDialect.getType(columnMap.get(key));
+ final int bindValueNum = type.bindValue(index.get(), type.convert2DatabaseTypeValue(dataChangesAfter.get(key)), query);
+ index.addAndGet(bindValueNum);
+ }
+ });
+ final int result = query.executeUpdate();
+ if (result != 1) {
+ throw new SQLException("Failed to update row from table");
+ }
+ transaction.commit();
+ } catch (SQLException e) {
+ transaction.rollback();
+ throw e;
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private void upsert(String sql, Schema schema, DataChanges dataChanges) throws SQLException {
+ final Transaction transaction = session.beginTransaction();
+ try {
+ if (log.isDebugEnabled()) {
+ log.debug("execute upsert sql: {}", sql);
+ }
+ final NativeQuery query = session.createNativeQuery(sql);
+ AtomicInteger index = new AtomicInteger(1);
+ Map dataChangesAfter = (Map) dataChanges.getAfter();
+ Field after = schema.getFields().get(0);
+ after.getFields().stream().map(field -> field.getColumn()).sorted(Comparator.comparingInt(Column::getOrder)).forEach(column -> {
+ Type type = eventMeshDialect.getType(column);
+ final int bindValueNum = type.bindValue(index.get(), type.convert2DatabaseTypeValue(dataChangesAfter.get(column.getName())), query);
+ index.addAndGet(bindValueNum);
+ });
+ final int result = query.executeUpdate();
+ if (result == 0) {
+ throw new SQLException("Failed to update row from table");
+ }
+ transaction.commit();
+ } catch (SQLException e) {
+ transaction.rollback();
+ throw e;
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private void delete(String sql, Schema schema, DataChanges dataChanges) throws SQLException {
+ final Transaction transaction = session.beginTransaction();
+
+ try {
+ LogUtil.debug(log, "execute delete sql: {}", () -> sql);
+ if (CollectionUtils.isEmpty(schema.getKeySet())) {
+ log.warn("No primary key found, skip delete");
+ return;
+ }
+ final NativeQuery query = session.createNativeQuery(sql);
+ AtomicInteger index = new AtomicInteger(1);
+ Map dataChangesAfter = (Map) dataChanges.getBefore();
+ final Map> columnMap = schema.getFields().get(0).getFields().stream().map(field -> field.getColumn())
+ .collect(Collectors.toMap(Column::getName, column -> column));
+ schema.getKeySet().stream().forEach(columnName -> {
+ final Column column = columnMap.get(columnName);
+ Type type = eventMeshDialect.getType(column);
+ final int bindValueNum = type.bindValue(index.get(), type.convert2DatabaseTypeValue(dataChangesAfter.get(column.getName())), query);
+ index.addAndGet(bindValueNum);
+ });
+ final int result = query.executeUpdate();
+ if (result != 1) {
+ throw new SQLException("Failed to delete row from table");
+ }
+ transaction.commit();
+ } catch (SQLException e) {
+ transaction.rollback();
+ throw e;
+ }
+ }
+
+ private DataHandleMode convert2DataHandleMode(DataChangeEventType type) {
+
+ switch (type) {
+ case INSERT:
+ return DataHandleMode.INSERT;
+ case UPDATE:
+ return this.jdbcSinkConfig.isSupportUpsert() ? DataHandleMode.UPSERT : DataHandleMode.UPDATE;
+ case DELETE:
+ return this.jdbcSinkConfig.isSupportDelete() ? DataHandleMode.DELETE : DataHandleMode.NONE;
+ default:
+ return DataHandleMode.NONE;
+ }
+ }
+
+ public enum DataHandleMode implements EnumeratedValue {
+
+ INSERT("insert"), UPSERT("upsert"), UPDATE("update"), DELETE("delete"), NONE("none");
+
+ private String value;
+
+ DataHandleMode(String value) {
+ this.value = value;
+ }
+
+ public static DataHandleMode forValue(String value) {
+ for (DataHandleMode mode : DataHandleMode.values()) {
+ if (mode.getValue().equalsIgnoreCase(value)) {
+ return mode;
+ }
+ }
+ throw new IllegalArgumentException("No enum constant " + DataHandleMode.class.getName() + "." + value);
+ }
+
+ @Override
+ public String getValue() {
+ return value;
+ }
+ }
+}
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/handle/DialectAssemblyLine.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/handle/DialectAssemblyLine.java
new file mode 100644
index 0000000000..f0ca24ecb2
--- /dev/null
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/handle/DialectAssemblyLine.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eventmesh.connector.jdbc.sink.handle;
+
+import org.apache.eventmesh.connector.jdbc.CatalogChanges;
+import org.apache.eventmesh.connector.jdbc.Schema;
+import org.apache.eventmesh.connector.jdbc.source.SourceMateData;
+
+/**
+ * Represents an assembly line for transforming and generating SQL statements based on specific database dialects.
+ */
+public interface DialectAssemblyLine {
+
+ /**
+ * Retrieves the database or table statement from the given {@code SourceMateData}, {@code CatalogChanges}, and {@code statement}.
+ *
+ * @param sourceMateData the source mate data object containing the information about the source
+ * @param catalogChanges the catalog changes object containing the information about the catalog changes
+ * @param statement the statement for which the database or table statement needs to be retrieved
+ * @return the database or table statement of the given {@code statement}
+ */
+ String getDatabaseOrTableStatement(SourceMateData sourceMateData, CatalogChanges catalogChanges, String statement);
+
+ /**
+ * Generates an insert statement for the given source mate data, schema, and origin statement.
+ *
+ * @param sourceMateData The source mate data of the database.
+ * @param schema The schema of the table.
+ * @param originStatement The original insert statement.
+ * @return The insert statement with the correct syntax for the given database and table.
+ */
+ String getInsertStatement(SourceMateData sourceMateData, Schema schema, String originStatement);
+
+ /**
+ * Generates an upsert statement using the given sourceMateData, schema, and originStatement.
+ *
+ * @param sourceMateData The metadata of the data source.
+ * @param schema The schema to upsert into.
+ * @param originStatement The original upsert statement.
+ * @return The upsert statement as a string.
+ */
+ String getUpsertStatement(SourceMateData sourceMateData, Schema schema, String originStatement);
+
+ /**
+ * Generates a delete statement based on the given sourceMateData, schema, and original statement.
+ *
+ * @param sourceMateData The source metadata used to generate the delete statement.
+ * @param schema The schema used to generate the delete statement.
+ * @param originStatement The original statement used as a basis for the delete statement.
+ * @return The generated delete statement as a string.
+ */
+ String getDeleteStatement(SourceMateData sourceMateData, Schema schema, String originStatement);
+
+ /**
+ * Generates an SQL update statement based on the provided source metadata, schema, and origin statement.
+ *
+ * @param sourceMateData The source metadata to be used for generating the update statement.
+ * @param schema The schema to be used for generating the update statement.
+ * @param originStatement The original SQL statement that needs to be updated.
+ * @return The generated SQL update statement as a string.
+ */
+ String getUpdateStatement(SourceMateData sourceMateData, Schema schema, String originStatement);
+
+}
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/handle/DialectAssemblyLineFactory.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/handle/DialectAssemblyLineFactory.java
new file mode 100644
index 0000000000..185c201f5b
--- /dev/null
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/handle/DialectAssemblyLineFactory.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eventmesh.connector.jdbc.sink.handle;
+
+import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialect;
+import org.apache.eventmesh.connector.jdbc.sink.mysql.MysqlDialectAssemblyLine;
+
+import org.hibernate.dialect.Dialect;
+
+public final class DialectAssemblyLineFactory {
+
+ public static DialectAssemblyLine build(DatabaseDialect databaseDialect, Dialect hibernateDialect) {
+ switch (databaseDialect.getDatabaseType()) {
+ case MYSQL:
+ return new MysqlDialectAssemblyLine(databaseDialect, hibernateDialect);
+ default:
+ return new GeneralDialectAssemblyLine(databaseDialect, hibernateDialect);
+ }
+ }
+
+}
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/handle/GeneralDialectAssemblyLine.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/handle/GeneralDialectAssemblyLine.java
new file mode 100644
index 0000000000..971c8479ca
--- /dev/null
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/handle/GeneralDialectAssemblyLine.java
@@ -0,0 +1,287 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eventmesh.connector.jdbc.sink.handle;
+
+import org.apache.eventmesh.connector.jdbc.CatalogChanges;
+import org.apache.eventmesh.connector.jdbc.Field;
+import org.apache.eventmesh.connector.jdbc.Schema;
+import org.apache.eventmesh.connector.jdbc.dialect.AbstractGeneralDatabaseDialect;
+import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialect;
+import org.apache.eventmesh.connector.jdbc.dialect.SqlStatementAssembler;
+import org.apache.eventmesh.connector.jdbc.event.SchemaChangeEventType;
+import org.apache.eventmesh.connector.jdbc.source.SourceMateData;
+import org.apache.eventmesh.connector.jdbc.table.catalog.Column;
+import org.apache.eventmesh.connector.jdbc.table.catalog.Table;
+import org.apache.eventmesh.connector.jdbc.table.catalog.TableId;
+import org.apache.eventmesh.connector.jdbc.type.Type;
+
+import org.apache.commons.lang3.StringUtils;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.hibernate.dialect.Dialect;
+
+import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class GeneralDialectAssemblyLine implements DialectAssemblyLine {
+
+ @Getter
+ private final DatabaseDialect databaseDialect;
+
+ @Getter
+ private final Dialect hibernateDialect;
+
+ public GeneralDialectAssemblyLine(DatabaseDialect databaseDialect, Dialect hibernateDialect) {
+ this.databaseDialect = databaseDialect;
+ this.hibernateDialect = hibernateDialect;
+ }
+
+ @Override
+ public String getDatabaseOrTableStatement(SourceMateData sourceMateData, CatalogChanges catalogChanges, String statement) {
+ String type = catalogChanges.getType();
+ String operationType = catalogChanges.getOperationType();
+ SchemaChangeEventType schemaChangeEventType = SchemaChangeEventType.ofSchemaChangeEventType(type, operationType);
+ String sql = null;
+ switch (schemaChangeEventType) {
+ case DATABASE_CREATE:
+ sql = assembleCreateDatabaseSql(catalogChanges);
+ break;
+ case DATABASE_DROP:
+ sql = assembleDropDatabaseSql(catalogChanges);
+ break;
+ case DATABASE_ALERT:
+ sql = assembleAlertDatabaseSql(catalogChanges);
+ break;
+ case TABLE_CREATE:
+ sql = assembleCreateTableSql(catalogChanges);
+ break;
+ case TABLE_DROP:
+ sql = assembleDropTableSql(catalogChanges);
+ break;
+ case TABLE_ALERT:
+ sql = assembleAlertTableSql(catalogChanges);
+ break;
+ default:
+ log.warn("Type={}, OperationType={} not support", type, operationType);
+ }
+ return sql;
+ }
+
+ /**
+ * Generates an upsert statement using the given sourceMateData, schema, and originStatement.
+ *
+ * @param sourceMateData The metadata of the data source.
+ * @param schema The schema to upsert into.
+ * @param originStatement The original upsert statement.
+ * @return The upsert statement as a string.
+ */
+ @Override
+ public String getUpsertStatement(SourceMateData sourceMateData, Schema schema, String originStatement) {
+ return null;
+ }
+
+ /**
+ * Generates a delete statement based on the given sourceMateData, schema, and original statement.
+ *
+ * @param sourceMateData The source metadata used to generate the delete statement.
+ * @param schema The schema used to generate the delete statement.
+ * @param originStatement The original statement used as a basis for the delete statement.
+ * @return The generated delete statement as a string.
+ */
+ @Override
+ public String getDeleteStatement(SourceMateData sourceMateData, Schema schema, String originStatement) {
+ SqlStatementAssembler sqlStatementAssembler = new SqlStatementAssembler();
+ sqlStatementAssembler.appendSqlSlice("DELETE FROM ")
+ .appendSqlSlice(((AbstractGeneralDatabaseDialect) databaseDialect).getQualifiedTableName(sourceMateData.ofTableId()));
+ sqlStatementAssembler.appendSqlSlice(" WHERE ");
+ if (schema.containsKey()) {
+ sqlStatementAssembler.appendSqlSliceLists(" AND ", schema.getKeySet(), (columnName) -> columnName + " =?");
+ } else {
+ Field after = schema.getFields().get(0);
+ sqlStatementAssembler.appendSqlSliceOfColumns(" AND ",
+ after.getFields().stream().map(field -> field.getColumn()).sorted(Comparator.comparingInt(Column::getOrder))
+ .collect(Collectors.toList()),
+ (column) -> column.getName() + " =?");
+ }
+ return sqlStatementAssembler.build();
+ }
+
+ /**
+ * Generates an SQL update statement based on the provided source metadata, schema, and origin statement.
+ *
+ * @param sourceMateData The source metadata to be used for generating the update statement.
+ * @param schema The schema to be used for generating the update statement.
+ * @param originStatement The original SQL statement that needs to be updated.
+ * @return The generated SQL update statement as a string.
+ */
+ @Override
+ public String getUpdateStatement(SourceMateData sourceMateData, Schema schema, String originStatement) {
+ final SqlStatementAssembler sqlStatementAssembler = new SqlStatementAssembler();
+ final TableId tableId = sourceMateData.ofTableId();
+ // primary key set
+ final Set keySet = schema.getKeySet();
+ Field tableColumns = schema.getFields().get(0);
+ sqlStatementAssembler.appendSqlSlice("UPDATE ");
+ sqlStatementAssembler.appendSqlSlice(((AbstractGeneralDatabaseDialect) databaseDialect).getQualifiedTableName(tableId));
+ sqlStatementAssembler.appendSqlSlice(" SET ");
+ sqlStatementAssembler.appendSqlSliceLists(", ",
+ tableColumns.getFields().stream().map(Field::getColumn).sorted(Comparator.comparingInt(Column::getOrder))
+ .filter(column -> !keySet.contains(column.getName())).map(column -> column.getName()).collect(Collectors.toList()),
+ (columnName) -> columnName + " =?");
+ if (schema.containsKey()) {
+ sqlStatementAssembler.appendSqlSlice(" WHERE ");
+ sqlStatementAssembler.appendSqlSliceLists(" AND ", keySet, (columnName) -> columnName + " =?");
+ } else {
+ sqlStatementAssembler.appendSqlSlice(" WHERE ");
+ sqlStatementAssembler.appendSqlSliceOfColumns(" AND ",
+ tableColumns.getFields().stream().map(field -> field.getColumn()).sorted(Comparator.comparingInt(Column::getOrder))
+ .collect(Collectors.toList()),
+ column -> column.getName() + " =?");
+ }
+ return sqlStatementAssembler.build();
+ }
+
+ @Override
+ public String getInsertStatement(SourceMateData sourceMateData, Schema schema, String originStatement) {
+ final TableId tableId = sourceMateData.ofTableId();
+
+ List afterFields = schema.getFields().stream().filter(field -> StringUtils.equals(field.getField(), "after"))
+ .collect(Collectors.toList());
+
+ final SqlStatementAssembler sqlAssembler = new SqlStatementAssembler();
+ sqlAssembler.appendSqlSlice("INSERT INTO ");
+ sqlAssembler.appendSqlSlice(((AbstractGeneralDatabaseDialect) databaseDialect).getQualifiedTableName(tableId));
+ sqlAssembler.appendSqlSlice(" (");
+ // assemble columns
+ Field afterField = afterFields.get(0);
+ List> columns = afterField.getFields().stream().map(item -> item.getColumn()).sorted(Comparator.comparingInt(Column::getOrder))
+ .collect(Collectors.toList());
+ sqlAssembler.appendSqlSliceOfColumns(", ", columns, column -> column.getName());
+ sqlAssembler.appendSqlSlice(") VALUES (");
+ // assemble values
+ sqlAssembler.appendSqlSliceOfColumns(", ", columns, column -> getDmlBindingValue(column));
+ sqlAssembler.appendSqlSlice(")");
+
+ return sqlAssembler.build();
+ }
+
+ private String getDmlBindingValue(Column column) {
+ Type type = this.databaseDialect.getType(column);
+ if (type == null) {
+ return this.databaseDialect.getQueryBindingWithValueCast(column);
+ }
+ return type.getQueryBindingWithValue(this.databaseDialect, column);
+ }
+
+ private String assembleCreateDatabaseSql(CatalogChanges catalogChanges) {
+ SqlStatementAssembler assembler = new SqlStatementAssembler();
+ assembler.appendSqlSlice("CREATE DATABASE IF NOT EXISTS ");
+ assembler.appendSqlSlice(((AbstractGeneralDatabaseDialect) databaseDialect).getQualifiedText(catalogChanges.getCatalog().getName()));
+ return assembler.build();
+ }
+
+ private String assembleDropDatabaseSql(CatalogChanges catalogChanges) {
+ SqlStatementAssembler assembler = new SqlStatementAssembler();
+ assembler.appendSqlSlice("DROP DATABASE IF EXISTS ");
+ assembler.appendSqlSlice(((AbstractGeneralDatabaseDialect) databaseDialect).getQualifiedText(catalogChanges.getCatalog().getName()));
+ return assembler.build();
+ }
+
+ private String assembleAlertDatabaseSql(CatalogChanges catalogChanges) {
+ SqlStatementAssembler assembler = new SqlStatementAssembler();
+ // todo
+ return assembler.build();
+ }
+
+ private String assembleCreateTableSql(CatalogChanges catalogChanges) {
+ SqlStatementAssembler assembler = new SqlStatementAssembler();
+ assembler.appendSqlSlice("CREATE TABLE IF NOT EXISTS ");
+ Table table = catalogChanges.getTable();
+ assembler.appendSqlSlice(((AbstractGeneralDatabaseDialect) databaseDialect).getQualifiedTableName(table.getTableId()));
+ assembler.appendSqlSlice(" (");
+ // assemble columns
+ List columns = catalogChanges.getColumns().stream().sorted(Comparator.comparingInt(Column::getOrder))
+ .collect(Collectors.toList());
+ List columnNames = columns.stream().map(item -> item.getName()).collect(Collectors.toList());
+ Map columnMap = columns.stream().collect(Collectors.toMap(Column::getName, item -> item));
+ assembler.appendSqlSliceLists(", ", columnNames, (columnName) -> {
+ StringBuilder builder = new StringBuilder();
+ // assemble column name
+ builder.append(((AbstractGeneralDatabaseDialect) databaseDialect).getQualifiedText(columnName));
+ // assemble column type
+ Column column = columnMap.get(columnName);
+ String typeName = this.databaseDialect.getTypeName(hibernateDialect, column);
+ builder.append(" ").append(typeName);
+
+ builder.append(" ").append(this.databaseDialect.getCharsetOrCollateFormatted(column));
+ if (Optional.ofNullable(table.getPrimaryKey().getColumnNames()).orElse(new ArrayList<>(0)).contains(columnName)) {
+ builder.append(" NOT NULL ");
+ if (column.isAutoIncremented()) {
+ builder.append(this.databaseDialect.getAutoIncrementFormatted(column));
+ }
+ } else {
+ if (column.isNotNull()) {
+ builder.append(" NOT NULL ");
+ }
+ }
+ addColumnDefaultValue(column, builder);
+ builder.append(" ").append(this.databaseDialect.getCommentFormatted(column));
+ // assemble column default value
+ return builder.toString();
+ });
+ // assemble primary key and others key
+ assembler.appendSqlSlice(", PRIMARY KEY(");
+ assembler.appendSqlSliceLists(",", catalogChanges.getTable().getPrimaryKey().getColumnNames(),
+ (columnName) -> ((AbstractGeneralDatabaseDialect) databaseDialect).getQualifiedText(columnName));
+ assembler.appendSqlSlice(")");
+ assembler.appendSqlSlice(")");
+ assembler.appendSqlSlice(this.databaseDialect.getTableOptionsFormatted(catalogChanges.getTable()));
+ return assembler.build();
+ }
+
+ private void addColumnDefaultValue(Column column, StringBuilder builder) {
+ if (column.isNotNull() && column.getDefaultValue() == null) {
+ return;
+ }
+ final String defaultValueFormatted = this.databaseDialect.getDefaultValueFormatted(column);
+ if (StringUtils.isNotEmpty(defaultValueFormatted)) {
+ builder.append(" DEFAULT ").append(defaultValueFormatted);
+ }
+ }
+
+ private String assembleDropTableSql(CatalogChanges catalogChanges) {
+ SqlStatementAssembler assembler = new SqlStatementAssembler();
+ assembler.appendSqlSlice("DROP TABLE IF EXISTS ");
+ assembler.appendSqlSlice(
+ ((AbstractGeneralDatabaseDialect) databaseDialect).getQualifiedTableName(catalogChanges.getTable().getTableId()));
+ return assembler.build();
+ }
+
+ private String assembleAlertTableSql(CatalogChanges catalogChanges) {
+ SqlStatementAssembler assembler = new SqlStatementAssembler();
+ return assembler.build();
+ }
+}
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/handle/SinkRecordHandler.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/handle/SinkRecordHandler.java
new file mode 100644
index 0000000000..fa6a5b841c
--- /dev/null
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/handle/SinkRecordHandler.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eventmesh.connector.jdbc.sink.handle;
+
+import org.apache.eventmesh.connector.jdbc.JdbcConnectData;
+
+/**
+ * This interface represents a schema change handler.
+ */
+public interface SinkRecordHandler {
+
+ /**
+ * Handles a schema change using the specified JDBC connection data.
+ *
+ * @param connectData the JDBC connection data
+ */
+ void handle(JdbcConnectData connectData) throws Exception;
+
+}
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/hibernate/DruidConnectionProvider.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/hibernate/DruidConnectionProvider.java
new file mode 100644
index 0000000000..e261978b7a
--- /dev/null
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/hibernate/DruidConnectionProvider.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eventmesh.connector.jdbc.sink.hibernate;
+
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.util.Map;
+
+import org.hibernate.HibernateException;
+import org.hibernate.cfg.Environment;
+import org.hibernate.engine.jdbc.connections.spi.ConnectionProvider;
+import org.hibernate.service.spi.Configurable;
+
+import com.alibaba.druid.pool.DruidDataSource;
+import com.alibaba.druid.pool.DruidDataSourceFactory;
+
+public class DruidConnectionProvider implements ConnectionProvider, Configurable {
+
+ private DruidDataSource dataSource = new DruidDataSource();
+
+ /**
+ * Obtains a connection for Hibernate use according to the underlying strategy of this provider.
+ *
+ * @return The obtained JDBC connection
+ * @throws SQLException Indicates a problem opening a connection
+ * @throws HibernateException Indicates a problem otherwise obtaining a connection.
+ */
+ @Override
+ public Connection getConnection() throws SQLException {
+ return dataSource.getConnection();
+ }
+
+ /**
+ * Release a connection from Hibernate use.
+ *
+ * @param conn The JDBC connection to release
+ * @throws SQLException Indicates a problem closing the connection
+ * @throws HibernateException Indicates a problem otherwise releasing a connection.
+ */
+ @Override
+ public void closeConnection(Connection conn) throws SQLException {
+ conn.close();
+ }
+
+ /**
+ *
+ * Does this connection provider support aggressive release of JDBC connections and re-acquisition of those connections (if need be) later?
+ *
+ *
+ * This is used in conjunction with {@link Environment#RELEASE_CONNECTIONS} to aggressively release JDBC connections. However, the configured
+ * ConnectionProvider must support re-acquisition of the same underlying connection for that semantic to work.
+ *
+ * Typically, this is only true in managed environments where a container tracks connections by transaction or thread.
+ *
+ * Note that JTA semantic depends on the fact that the underlying connection provider does support aggressive release.
+ *
+ * @return {@code true} if aggressive releasing is supported; {@code false} otherwise.
+ */
+ @Override
+ public boolean supportsAggressiveRelease() {
+ return false;
+ }
+
+ /**
+ * Configure the service.
+ *
+ * @param configurationValues The configuration properties.
+ */
+ @Override
+ public void configure(Map configurationValues) {
+ try {
+ DruidDataSourceFactory.config(dataSource, configurationValues);
+ } catch (SQLException e) {
+ throw new IllegalArgumentException("Config druid error", e);
+ }
+ }
+
+ /**
+ * Can this wrapped service be unwrapped as the indicated type?
+ *
+ * @param unwrapType The type to check.
+ * @return True/false.
+ */
+ @Override
+ public boolean isUnwrappableAs(Class unwrapType) {
+ return dataSource.isWrapperFor(unwrapType);
+ }
+
+ /**
+ * Unproxy the service proxy
+ *
+ * @param unwrapType The java type as which to unwrap this instance.
+ * @return The unwrapped reference
+ */
+ @Override
+ public T unwrap(Class unwrapType) {
+ return dataSource.unwrap(unwrapType);
+ }
+}
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/hibernate/HibernateConfiguration.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/hibernate/HibernateConfiguration.java
new file mode 100644
index 0000000000..fd5d49de74
--- /dev/null
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/hibernate/HibernateConfiguration.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eventmesh.connector.jdbc.sink.hibernate;
+
+import org.hibernate.SessionFactory;
+import org.hibernate.boot.registry.StandardServiceRegistryBuilder;
+import org.hibernate.cfg.Configuration;
+import org.hibernate.service.ServiceRegistry;
+
+public final class HibernateConfiguration {
+
+ public static HibernateConfigurationBuilder newBuilder() {
+ return new HibernateConfigurationBuilder();
+ }
+
+ public static class HibernateConfigurationBuilder {
+
+ private Configuration configuration;
+
+ public HibernateConfigurationBuilder() {
+ this.configuration = new Configuration();
+ this.configuration.setProperty("hibernate.connection.provider_class", DruidConnectionProvider.class.getName());
+ }
+
+ public HibernateConfigurationBuilder withUser(String username) {
+ configuration.setProperty("username", username);
+ return this;
+ }
+
+ public HibernateConfigurationBuilder withPassword(String password) {
+ configuration.setProperty("password", password);
+ return this;
+ }
+
+ public HibernateConfigurationBuilder withUrl(String url) {
+ configuration.setProperty("url", url);
+ return this;
+ }
+
+ public HibernateConfigurationBuilder withDruidMaxActive(String maxActive) {
+ configuration.setProperty("maxActive", maxActive);
+ return this;
+ }
+
+ public HibernateConfigurationBuilder withShowSql(boolean showSql) {
+ configuration.setProperty("hibernate.show_sql", Boolean.toString(showSql));
+ return this;
+ }
+
+ public SessionFactory build() {
+ ServiceRegistry serviceRegistry = new StandardServiceRegistryBuilder()
+ .applySettings(configuration.getProperties())
+ .build();
+ return configuration.buildSessionFactory(serviceRegistry);
+ }
+
+ }
+}
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/mysql/MysqlDialectAssemblyLine.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/mysql/MysqlDialectAssemblyLine.java
new file mode 100644
index 0000000000..670a2d0258
--- /dev/null
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/mysql/MysqlDialectAssemblyLine.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eventmesh.connector.jdbc.sink.mysql;
+
+import org.apache.eventmesh.connector.jdbc.Field;
+import org.apache.eventmesh.connector.jdbc.Schema;
+import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialect;
+import org.apache.eventmesh.connector.jdbc.dialect.SqlStatementAssembler;
+import org.apache.eventmesh.connector.jdbc.sink.handle.GeneralDialectAssemblyLine;
+import org.apache.eventmesh.connector.jdbc.source.SourceMateData;
+import org.apache.eventmesh.connector.jdbc.table.catalog.Column;
+import org.apache.eventmesh.connector.jdbc.utils.JdbcStringUtils;
+
+import java.util.Comparator;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import org.hibernate.dialect.Dialect;
+
+public class MysqlDialectAssemblyLine extends GeneralDialectAssemblyLine {
+
+ public MysqlDialectAssemblyLine(DatabaseDialect databaseDialect, Dialect hibernateDialect) {
+ super(databaseDialect, hibernateDialect);
+ }
+
+ /**
+ * Generates an upsert statement using the given sourceMateData, schema, and originStatement.
+ *
+ * @param sourceMateData The metadata of the data source.
+ * @param schema The schema to upsert into.
+ * @param originStatement The original upsert statement.
+ * @return The upsert statement as a string.
+ */
+ @Override
+ public String getUpsertStatement(SourceMateData sourceMateData, Schema schema, String originStatement) {
+ final SqlStatementAssembler sqlStatementAssembler = new SqlStatementAssembler();
+ sqlStatementAssembler.appendSqlSlice(getInsertStatement(sourceMateData, schema, originStatement));
+ Field afterField = schema.getFields().get(0);
+ List> columns = afterField.getFields().stream().map(item -> item.getColumn()).sorted(Comparator.comparingInt(Column::getOrder))
+ .collect(Collectors.toList());
+ if (JdbcStringUtils.compareVersion(getDatabaseDialect().getJdbcDriverMetaData().getDatabaseProductVersion(), "8.0.20") >= 0) {
+ // mysql doc:https://dev.mysql.com/doc/refman/8.0/en/insert-on-duplicate.html
+ // Beginning with MySQL 8.0.20, an INSERT ... SELECT ... ON DUPLICATE KEY UPDATE statement that uses VALUES() in the UPDATE clause
+ sqlStatementAssembler.appendSqlSlice("AS new ON DUPLICATE KEY UPDATE ");
+ sqlStatementAssembler.appendSqlSliceOfColumns(",", columns, column -> {
+ final String columnName = column.getName();
+ return columnName + "=new." + columnName;
+ });
+
+ } else {
+ sqlStatementAssembler.appendSqlSlice(" ON DUPLICATE KEY UPDATE ");
+ sqlStatementAssembler.appendSqlSliceOfColumns(",", columns, column -> {
+ final String columnName = column.getName();
+ return columnName + "=VALUES(" + columnName + ")";
+ });
+
+ }
+ return sqlStatementAssembler.build();
+ }
+}
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/AbstractEngine.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/AbstractEngine.java
index 2d1080e05b..08c3823725 100644
--- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/AbstractEngine.java
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/AbstractEngine.java
@@ -18,7 +18,7 @@
package org.apache.eventmesh.connector.jdbc.source;
import org.apache.eventmesh.common.ThreadWrapper;
-import org.apache.eventmesh.connector.jdbc.DatabaseDialect;
+import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialect;
import org.apache.eventmesh.connector.jdbc.source.config.JdbcSourceConfig;
import org.apache.eventmesh.connector.jdbc.source.config.SourceConnectorConfig;
import org.apache.eventmesh.connector.jdbc.table.catalog.TableId;
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/JdbcAllFactoryLoader.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/JdbcAllFactoryLoader.java
index f3c09adb53..2529d9a5a8 100644
--- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/JdbcAllFactoryLoader.java
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/JdbcAllFactoryLoader.java
@@ -19,7 +19,7 @@
import static com.google.common.base.Preconditions.checkNotNull;
-import org.apache.eventmesh.connector.jdbc.source.dialect.DatabaseDialectFactory;
+import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialectFactory;
import org.apache.eventmesh.connector.jdbc.source.dialect.cdc.CdcEngineFactory;
import org.apache.eventmesh.connector.jdbc.source.dialect.snapshot.SnapshotEngineFactory;
import org.apache.eventmesh.spi.EventMeshExtensionFactory;
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/JdbcSourceConnector.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/JdbcSourceConnector.java
index 202d7b9ca4..8d7d9cb66d 100644
--- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/JdbcSourceConnector.java
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/JdbcSourceConnector.java
@@ -17,10 +17,10 @@
package org.apache.eventmesh.connector.jdbc.source;
-import org.apache.eventmesh.connector.jdbc.DatabaseDialect;
+import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialect;
+import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialectFactory;
import org.apache.eventmesh.connector.jdbc.event.Event;
import org.apache.eventmesh.connector.jdbc.source.config.JdbcSourceConfig;
-import org.apache.eventmesh.connector.jdbc.source.dialect.DatabaseDialectFactory;
import org.apache.eventmesh.connector.jdbc.source.dialect.cdc.CdcEngine;
import org.apache.eventmesh.connector.jdbc.source.dialect.cdc.CdcEngineFactory;
import org.apache.eventmesh.connector.jdbc.source.dialect.snapshot.SnapshotEngine;
@@ -111,7 +111,7 @@ private void doInit() {
// Get the database dialect factory and create the database dialect.
final DatabaseDialectFactory databaseDialectFactory = JdbcAllFactoryLoader.getDatabaseDialectFactory(databaseType);
- this.databaseDialect = databaseDialectFactory.createDatabaseDialect(sourceConfig);
+ this.databaseDialect = databaseDialectFactory.createDatabaseDialect(this.sourceConfig.getSourceConnectorConfig().getJdbcConfig());
this.databaseDialect.init();
// Get the snapshot engine factory and create the snapshot engine
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/SourceJdbcTaskManager.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/SourceJdbcTaskManager.java
index 49e9c53a15..be38ffb75a 100644
--- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/SourceJdbcTaskManager.java
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/SourceJdbcTaskManager.java
@@ -64,7 +64,7 @@ public void init() {
}
private void doHandleEvent(Event event) {
- if (null == event) {
+ if (event == null) {
return;
}
JdbcConnectData jdbcConnectData = event.getJdbcConnectData();
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/SourceMateData.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/SourceMateData.java
index e1f6f28fd7..bf12a36d97 100644
--- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/SourceMateData.java
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/SourceMateData.java
@@ -17,12 +17,20 @@
package org.apache.eventmesh.connector.jdbc.source;
+import org.apache.eventmesh.connector.jdbc.source.dialect.mysql.MysqlSourceMateData;
+import org.apache.eventmesh.connector.jdbc.table.catalog.TableId;
+
+import com.fasterxml.jackson.annotation.JsonSubTypes;
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+
import lombok.Data;
/**
* Represents metadata related to a data source.
*/
@Data
+@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "connector")
+@JsonSubTypes({@JsonSubTypes.Type(value = MysqlSourceMateData.class, name = "mysql")})
public class SourceMateData {
/**
@@ -60,4 +68,13 @@ public class SourceMateData {
*/
private String tableName;
+ /**
+ * This method returns the TableId object with the specified catalog name, schema name, and table name.
+ *
+ * @return the TableId object
+ */
+ public TableId ofTableId() {
+ return new TableId(catalogName, schemaName, tableName);
+ }
+
}
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/TaskManagerCoordinator.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/TaskManagerCoordinator.java
index a1572787d7..c299fbc531 100644
--- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/TaskManagerCoordinator.java
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/TaskManagerCoordinator.java
@@ -17,6 +17,7 @@
package org.apache.eventmesh.connector.jdbc.source;
+import org.apache.eventmesh.common.utils.JsonUtils;
import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord;
import org.apache.commons.collections4.CollectionUtils;
@@ -102,6 +103,9 @@ public List poll() {
if (Objects.isNull(record)) {
break;
}
+ if (log.isDebugEnabled()) {
+ log.debug("record:{}", JsonUtils.toJSONString(record));
+ }
records.add(record);
} catch (InterruptedException e) {
break;
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/AbstractGeneralDatabaseDialect.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/AbstractGeneralDatabaseDialect.java
deleted file mode 100644
index ae497fce5c..0000000000
--- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/AbstractGeneralDatabaseDialect.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.eventmesh.connector.jdbc.source.dialect;
-
-import org.apache.eventmesh.connector.jdbc.DatabaseDialect;
-import org.apache.eventmesh.connector.jdbc.connection.JdbcConnection;
-import org.apache.eventmesh.connector.jdbc.exception.JdbcConnectionException;
-import org.apache.eventmesh.connector.jdbc.source.config.SourceConnectorConfig;
-
-import java.sql.Connection;
-import java.sql.PreparedStatement;
-import java.sql.SQLException;
-
-public abstract class AbstractGeneralDatabaseDialect implements DatabaseDialect {
-
- private static final int DEFAULT_BATCH_MAX_ROWS = 20;
-
- private SourceConnectorConfig config;
-
- private int batchMaxRows = DEFAULT_BATCH_MAX_ROWS;
-
- public AbstractGeneralDatabaseDialect(SourceConnectorConfig config) {
- this.config = config;
- }
-
- @Override
- public boolean isValid(Connection connection, int timeout) throws JdbcConnectionException, SQLException {
- return connection == null ? false : connection.isValid(timeout);
- }
-
- @Override
- public PreparedStatement createPreparedStatement(Connection connection, String sql) throws SQLException {
- PreparedStatement preparedStatement = connection.prepareStatement(sql);
- if (batchMaxRows > 0) {
- preparedStatement.setFetchSize(batchMaxRows);
- }
- return preparedStatement;
- }
-
-}
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/antlr4/mysql/MysqlAntlr4DdlParser.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/antlr4/mysql/MysqlAntlr4DdlParser.java
index 1cf69bde2f..9bb110815f 100644
--- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/antlr4/mysql/MysqlAntlr4DdlParser.java
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/antlr4/mysql/MysqlAntlr4DdlParser.java
@@ -20,12 +20,15 @@
import org.apache.eventmesh.connector.jdbc.antlr4.Antlr4DdlParser;
import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlLexer;
import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser;
+import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.CharsetNameContext;
+import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.CollationNameContext;
import org.apache.eventmesh.connector.jdbc.antlr4.listener.Antlr4DdlParserListener;
import org.apache.eventmesh.connector.jdbc.ddl.DdlParserCallback;
import org.apache.eventmesh.connector.jdbc.event.Event;
import org.apache.eventmesh.connector.jdbc.source.config.JdbcSourceConfig;
import org.apache.eventmesh.connector.jdbc.source.dialect.antlr4.mysql.listener.MySqlAntlr4DdlParserListener;
import org.apache.eventmesh.connector.jdbc.table.catalog.TableId;
+import org.apache.eventmesh.connector.jdbc.utils.JdbcStringUtils;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
@@ -55,6 +58,10 @@ public MysqlAntlr4DdlParser(boolean skipViews, boolean skipComments, JdbcSourceC
this(skipViews, skipComments, new HashSet<>(), sourceConfig);
}
+ public MysqlAntlr4DdlParser(boolean skipViews, boolean skipComments) {
+ this(skipViews, skipComments, new HashSet<>(), null);
+ }
+
@Override
protected MySqlLexer buildLexerInstance(CharStream charStreams) {
return new MySqlLexer(charStreams);
@@ -149,4 +156,25 @@ public void addTableIdSet(Set tableIdSet) {
public JdbcSourceConfig getSourceConfig() {
return sourceConfig;
}
+
+ public String parseCharset(CharsetNameContext charsetNameContext) {
+ String charsetName = null;
+ if (charsetNameContext != null && charsetNameContext.getText() != null) {
+ charsetName = JdbcStringUtils.withoutWrapper(charsetNameContext.getText());
+ }
+ return charsetName;
+ }
+
+ public String parseCollation(CollationNameContext collationNameContext) {
+ String collationName = null;
+ if (collationNameContext != null && collationNameContext.getText() != null) {
+ collationName = JdbcStringUtils.withoutWrapper(collationNameContext.getText()).toLowerCase();
+ /*
+ * for (int index = 0; index < CharsetMapping.MAP_SIZE; index++) { if
+ * (collationName.equals(CharsetMapping.getStaticCollationNameForCollationIndex(index))) { collationName =
+ * CharsetMapping.getStaticMysqlCharsetNameForCollationIndex(index); break; } }
+ */
+ }
+ return collationName;
+ }
}
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/antlr4/mysql/listener/ColumnDefinitionParserListener.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/antlr4/mysql/listener/ColumnDefinitionParserListener.java
index 9be9b42ddd..23e3728d47 100644
--- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/antlr4/mysql/listener/ColumnDefinitionParserListener.java
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/antlr4/mysql/listener/ColumnDefinitionParserListener.java
@@ -19,29 +19,46 @@
import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.AutoIncrementColumnConstraintContext;
import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.CollateColumnConstraintContext;
+import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.CollectionDataTypeContext;
import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.ColumnDefinitionContext;
import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.CommentColumnConstraintContext;
+import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.DataTypeContext;
+import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.DecimalLiteralContext;
+import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.DimensionDataTypeContext;
+import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.LongVarcharDataTypeContext;
+import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.NationalStringDataTypeContext;
+import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.NationalVaryingStringDataTypeContext;
import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.NullNotnullContext;
import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.PrimaryKeyColumnConstraintContext;
+import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.SimpleDataTypeContext;
+import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.SpatialDataTypeContext;
+import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.StringDataTypeContext;
import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.UniqueKeyColumnConstraintContext;
import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParserBaseListener;
import org.apache.eventmesh.connector.jdbc.source.dialect.antlr4.mysql.MysqlAntlr4DdlParser;
import org.apache.eventmesh.connector.jdbc.source.dialect.mysql.MysqlDataTypeConvertor;
import org.apache.eventmesh.connector.jdbc.table.catalog.TableEditor;
import org.apache.eventmesh.connector.jdbc.table.catalog.mysql.MysqlColumnEditor;
+import org.apache.eventmesh.connector.jdbc.table.catalog.mysql.MysqlOptions.MysqlColumnOptions;
import org.apache.eventmesh.connector.jdbc.table.type.EventMeshDataType;
import org.apache.eventmesh.connector.jdbc.utils.JdbcStringUtils;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+
import java.util.List;
import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
import org.antlr.v4.runtime.tree.ParseTreeListener;
import lombok.Getter;
import lombok.Setter;
+import lombok.extern.slf4j.Slf4j;
@Getter
@Setter
+@Slf4j
public class ColumnDefinitionParserListener extends MySqlParserBaseListener {
private DefaultValueParserListener defaultValueParserListener;
@@ -72,11 +89,108 @@ public ColumnDefinitionParserListener(List listeners, TableEd
public void enterColumnDefinition(ColumnDefinitionContext ctx) {
// parse Column data type
this.parser.runIfAllNotNull(() -> {
- String dataTypeString = ctx.dataType().getText();
- EventMeshDataType eventMeshType = this.dataTypeConvertor.toEventMeshType(dataTypeString);
- this.columnEditor.withEventMeshType(eventMeshType);
- this.columnEditor.withJdbcType(this.dataTypeConvertor.toJDBCType(dataTypeString));
- this.columnEditor.withType(dataTypeString);
+ DataTypeContext dataTypeContext = ctx.dataType();
+ String dataTypeString = null;
+ if (dataTypeContext instanceof StringDataTypeContext) {
+ StringDataTypeContext stringDataTypeCtx = (StringDataTypeContext) dataTypeContext;
+ dataTypeString = stringDataTypeCtx.typeName.getText();
+ // parse data type length
+ if (stringDataTypeCtx.lengthOneDimension() != null) {
+ this.columnEditor.length(Integer.parseInt(stringDataTypeCtx.lengthOneDimension().decimalLiteral().getText()));
+ }
+ // parse data type charset and collation
+ String charsetName = parser.parseCharset(stringDataTypeCtx.charsetName());
+ String collationName = parser.parseCollation(stringDataTypeCtx.collationName());
+ columnEditor.charsetName(charsetName);
+ columnEditor.collation(collationName);
+ } else if (dataTypeContext instanceof NationalStringDataTypeContext) {
+ NationalStringDataTypeContext nationalStringDataTypeCtx = (NationalStringDataTypeContext) dataTypeContext;
+ dataTypeString = nationalStringDataTypeCtx.typeName.getText();
+ if (nationalStringDataTypeCtx.lengthOneDimension() != null) {
+ this.columnEditor.length(Integer.parseInt(nationalStringDataTypeCtx.lengthOneDimension().decimalLiteral().getText()));
+ }
+ } else if (dataTypeContext instanceof NationalVaryingStringDataTypeContext) {
+ NationalVaryingStringDataTypeContext nationalVaryingStringDataTypeCtx = (NationalVaryingStringDataTypeContext) dataTypeContext;
+ dataTypeString = nationalVaryingStringDataTypeCtx.typeName.getText();
+ if (nationalVaryingStringDataTypeCtx.lengthOneDimension() != null) {
+ this.columnEditor.length(Integer.parseInt(nationalVaryingStringDataTypeCtx.lengthOneDimension().decimalLiteral().getText()));
+ }
+ } else if (dataTypeContext instanceof DimensionDataTypeContext) {
+ DimensionDataTypeContext dimensionDataTypeCtx = (DimensionDataTypeContext) dataTypeContext;
+ dataTypeString = dimensionDataTypeCtx.typeName.getText();
+ // parse column length
+ if (dimensionDataTypeCtx.lengthOneDimension() != null) {
+ this.columnEditor.length(Integer.parseInt(dimensionDataTypeCtx.lengthOneDimension().decimalLiteral().getText()));
+ }
+ // parse column scale if has scale
+ if (dimensionDataTypeCtx.lengthTwoDimension() != null) {
+ List decimalLiteralContexts = dimensionDataTypeCtx.lengthTwoDimension().decimalLiteral();
+ this.columnEditor.length(Integer.parseInt(decimalLiteralContexts.get(0).getText()));
+ this.columnEditor.scale(Integer.parseInt(decimalLiteralContexts.get(1).getText()));
+ }
+
+ if (dimensionDataTypeCtx.lengthTwoOptionalDimension() != null) {
+ List decimalLiteralContexts = dimensionDataTypeCtx.lengthTwoOptionalDimension().decimalLiteral();
+ if (decimalLiteralContexts.get(0).REAL_LITERAL() != null) {
+ String[] digits = decimalLiteralContexts.get(0).getText().split(".");
+ if (StringUtils.isBlank(digits[0]) || Integer.valueOf(digits[0]) == 0) {
+ this.columnEditor.length(10);
+ } else {
+ this.columnEditor.length(Integer.valueOf(digits[0]));
+ }
+ } else {
+ this.columnEditor.length(Integer.parseInt(decimalLiteralContexts.get(0).getText()));
+ }
+ if (decimalLiteralContexts.size() > 1) {
+ this.columnEditor.scale(Integer.parseInt(decimalLiteralContexts.get(1).getText()));
+ }
+ }
+ if (CollectionUtils.isNotEmpty(dimensionDataTypeCtx.SIGNED())) {
+ this.columnEditor.withOption(MysqlColumnOptions.SIGNED, dimensionDataTypeCtx.SIGNED().get(0).getText());
+ }
+ if (CollectionUtils.isNotEmpty(dimensionDataTypeCtx.UNSIGNED())) {
+ this.columnEditor.withOption(MysqlColumnOptions.UNSIGNED, dimensionDataTypeCtx.UNSIGNED().get(0).getText());
+ }
+ if (CollectionUtils.isNotEmpty(dimensionDataTypeCtx.ZEROFILL())) {
+ this.columnEditor.withOption(MysqlColumnOptions.ZEROFILL, dimensionDataTypeCtx.ZEROFILL().get(0).getText());
+ }
+ } else if (dataTypeContext instanceof SimpleDataTypeContext) {
+ // Do nothing for example: DATE, TINYBLOB, etc.
+ SimpleDataTypeContext simpleDataTypeCtx = (SimpleDataTypeContext) dataTypeContext;
+ dataTypeString = simpleDataTypeCtx.typeName.getText();
+ } else if (dataTypeContext instanceof CollectionDataTypeContext) {
+ CollectionDataTypeContext collectionDataTypeContext = (CollectionDataTypeContext) dataTypeContext;
+ dataTypeString = collectionDataTypeContext.typeName.getText();
+ if (collectionDataTypeContext.charsetName() != null) {
+ String charsetName = collectionDataTypeContext.charsetName().getText();
+ columnEditor.charsetName(charsetName);
+ }
+ } else if (dataTypeContext instanceof SpatialDataTypeContext) {
+ // do nothing
+ SpatialDataTypeContext spatialDataTypeCtx = (SpatialDataTypeContext) dataTypeContext;
+ dataTypeString = spatialDataTypeCtx.typeName.getText();
+ } else if (dataTypeContext instanceof LongVarcharDataTypeContext) {
+ LongVarcharDataTypeContext longVarcharDataTypeCtx = (LongVarcharDataTypeContext) dataTypeContext;
+ dataTypeString = longVarcharDataTypeCtx.typeName.getText();
+ String charsetName = parser.parseCharset(longVarcharDataTypeCtx.charsetName());
+ String collationName = parser.parseCollation(longVarcharDataTypeCtx.collationName());
+ columnEditor.charsetName(charsetName);
+ columnEditor.collation(collationName);
+ }
+ // handle enum and set type values
+ if (StringUtils.equalsAnyIgnoreCase(dataTypeString, "ENUM", "SET")) {
+ CollectionDataTypeContext collectionDataTypeContext = (CollectionDataTypeContext) dataTypeContext;
+ List values = collectionDataTypeContext.collectionOptions().STRING_LITERAL().stream()
+ .map(node -> JdbcStringUtils.withoutWrapper(node.getText())).collect(Collectors.toList());
+ columnEditor.enumValues(values);
+ }
+
+ if (StringUtils.isNotBlank(dataTypeString)) {
+ EventMeshDataType eventMeshType = this.dataTypeConvertor.toEventMeshType(dataTypeString);
+ this.columnEditor.withEventMeshType(eventMeshType);
+ this.columnEditor.withJdbcType(this.dataTypeConvertor.toJDBCType(dataTypeString));
+ this.columnEditor.withType(dataTypeString);
+ }
}, columnEditor);
this.parser.runIfAllNotNull(() -> {
@@ -140,7 +254,7 @@ public void exitColumnDefinition(ColumnDefinitionContext ctx) {
@Override
public void enterCollateColumnConstraint(CollateColumnConstraintContext ctx) {
if (ctx.COLLATE() != null) {
- columnEditor.collate(ctx.collationName().getText());
+ columnEditor.collation(ctx.collationName().getText());
}
super.enterCollateColumnConstraint(ctx);
}
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/antlr4/mysql/listener/CreateTableParserListener.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/antlr4/mysql/listener/CreateTableParserListener.java
index 5a72c4449b..4e30da93f9 100644
--- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/antlr4/mysql/listener/CreateTableParserListener.java
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/antlr4/mysql/listener/CreateTableParserListener.java
@@ -21,7 +21,9 @@
import org.apache.eventmesh.connector.jdbc.Payload;
import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.ColumnCreateTableContext;
import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.CopyCreateTableContext;
+import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.DecimalLiteralContext;
import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.QueryCreateTableContext;
+import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.TableOptionAutoIncrementContext;
import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.TableOptionCharsetContext;
import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.TableOptionCollateContext;
import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.TableOptionEngineContext;
@@ -32,9 +34,10 @@
import org.apache.eventmesh.connector.jdbc.source.dialect.mysql.MysqlSourceMateData;
import org.apache.eventmesh.connector.jdbc.table.catalog.Table;
import org.apache.eventmesh.connector.jdbc.table.catalog.TableId;
+import org.apache.eventmesh.connector.jdbc.table.catalog.mysql.MysqlOptions.MysqlTableOptions;
import org.apache.eventmesh.connector.jdbc.table.catalog.mysql.MysqlTableEditor;
-import org.apache.eventmesh.connector.jdbc.table.catalog.mysql.MysqlTableOptions;
import org.apache.eventmesh.connector.jdbc.table.catalog.mysql.MysqlTableSchema;
+import org.apache.eventmesh.connector.jdbc.utils.Antlr4Utils;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
@@ -98,7 +101,7 @@ public void enterColumnCreateTable(ColumnCreateTableContext ctx) {
@Override
public void exitColumnCreateTable(ColumnCreateTableContext ctx) {
- String ddl = ctx.getText();
+ String ddl = Antlr4Utils.getText(ctx);
parser.runIfAllNotNull(() -> {
listeners.remove(columnDefinitionListener);
// help JVM GC
@@ -114,7 +117,12 @@ public void exitColumnCreateTable(ColumnCreateTableContext ctx) {
.catalogName(currentDatabase)
.serverId(sourceConnectorConfig.getMysqlConfig().getServerId())
.build();
- Table table = new Table(tableSchema.getSimpleName(), tableSchema.getPrimaryKey(), tableSchema.getUniqueKeys(), tableSchema.getComment());
+ Table table = Table.newBuilder().withTableId(tableSchema.getTableId())
+ .withPrimaryKey(tableSchema.getPrimaryKey())
+ .withUniqueKeys(tableSchema.getUniqueKeys())
+ .withComment(tableSchema.getComment())
+ .withOptions(tableSchema.getTableOptions())
+ .build();
CatalogChanges changes = CatalogChanges.newBuilder().operationType(SchemaChangeEventType.TABLE_CREATE).table(table)
.columns(tableSchema.getColumns()).build();
payload.withSource(sourceMateData).withDdl(ddl).withCatalogChanges(changes);
@@ -136,7 +144,7 @@ private MysqlTableEditor createTableEditor(String tableName) {
@Override
public void enterTableOptionEngine(TableOptionEngineContext ctx) {
if (ctx.ENGINE() != null) {
- this.tableEditor.withOption(MysqlTableOptions.ENGINE, ctx.ENGINE().getText());
+ this.tableEditor.withOption(MysqlTableOptions.ENGINE, ctx.engineName().getText());
}
super.enterTableOptionEngine(ctx);
}
@@ -155,6 +163,16 @@ public void enterTableOptionCharset(TableOptionCharsetContext ctx) {
super.enterTableOptionCharset(ctx);
}
+ @Override
+ public void enterTableOptionAutoIncrement(TableOptionAutoIncrementContext ctx) {
+ DecimalLiteralContext decimalLiteralContext = ctx.decimalLiteral();
+ if (decimalLiteralContext != null) {
+ String autoIncrementNumber = Antlr4Utils.getText(decimalLiteralContext);
+ this.tableEditor.withOption(MysqlTableOptions.AUTO_INCREMENT, autoIncrementNumber);
+ }
+ super.enterTableOptionAutoIncrement(ctx);
+ }
+
@Override
public void enterTableOptionCollate(TableOptionCollateContext ctx) {
if (ctx.COLLATE() != null) {
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/antlr4/mysql/listener/DefaultValueParserListener.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/antlr4/mysql/listener/DefaultValueParserListener.java
index aad1dc65ad..51bb5fe579 100644
--- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/antlr4/mysql/listener/DefaultValueParserListener.java
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/antlr4/mysql/listener/DefaultValueParserListener.java
@@ -83,7 +83,7 @@ public void enterDefaultValue(DefaultValueContext ctx) {
if (stringLiteralContext.COLLATE() == null) {
columnEditor.defaultValueExpression(sign + unquote(stringLiteralContext.getText()));
} else {
- columnEditor.collate(sign + unquote(stringLiteralContext.STRING_LITERAL(0).getText()));
+ columnEditor.collation(sign + unquote(stringLiteralContext.STRING_LITERAL(0).getText()));
}
} else if (ctx.constant().decimalLiteral() != null) {
columnEditor.defaultValueExpression(sign + ctx.constant().decimalLiteral().getText());
@@ -117,14 +117,14 @@ public void enterDefaultValue(DefaultValueContext ctx) {
* ;
*/
List currentTimestampContexts = ctx.currentTimestamp();
- if (currentTimestampContexts.size() > 1 || (ctx.ON() == null && ctx.UPDATE() == null)) {
+ if (currentTimestampContexts.size() > 1 && (ctx.ON() != null && ctx.UPDATE() != null)) {
+ StringBuilder builder = new StringBuilder();
+ builder.append(currentTimestampContexts.get(0).getText()).append(" ").append(ctx.ON().getText()).append(" ").append(ctx.UPDATE())
+ .append(" ").append(currentTimestampContexts.get(1).getText());
+ columnEditor.defaultValueExpression(builder.toString());
+ } else if (currentTimestampContexts.size() == 1) {
CurrentTimestampContext currentTimestampContext = currentTimestampContexts.get(0);
- //
- if (currentTimestampContext.CURRENT_TIMESTAMP() != null || currentTimestampContext.NOW() != null) {
- columnEditor.defaultValueExpression("1970-01-01 00:00:00");
- } else {
- columnEditor.defaultValueExpression(currentTimestampContext.getText());
- }
+ columnEditor.defaultValueExpression(currentTimestampContext.getText());
}
} else if (ctx.expression() != null) {
// e.g. CREATE TABLE t2 (b BLOB DEFAULT ('abc'));
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/AbstractCdcEngine.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/AbstractCdcEngine.java
index fc9d2f3724..1fb1a95579 100644
--- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/AbstractCdcEngine.java
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/AbstractCdcEngine.java
@@ -18,9 +18,9 @@
package org.apache.eventmesh.connector.jdbc.source.dialect.cdc;
import org.apache.eventmesh.common.ThreadWrapper;
-import org.apache.eventmesh.connector.jdbc.DatabaseDialect;
import org.apache.eventmesh.connector.jdbc.JdbcContext;
import org.apache.eventmesh.connector.jdbc.ddl.DdlParser;
+import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialect;
import org.apache.eventmesh.connector.jdbc.source.config.JdbcSourceConfig;
import org.apache.eventmesh.connector.jdbc.source.config.SourceConnectorConfig;
import org.apache.eventmesh.connector.jdbc.table.catalog.TableId;
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/CdcEngineFactory.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/CdcEngineFactory.java
index 33e5f555b6..e08e1c8216 100644
--- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/CdcEngineFactory.java
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/CdcEngineFactory.java
@@ -17,7 +17,7 @@
package org.apache.eventmesh.connector.jdbc.source.dialect.cdc;
-import org.apache.eventmesh.connector.jdbc.DatabaseDialect;
+import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialect;
import org.apache.eventmesh.openconnect.api.config.SourceConfig;
import org.apache.eventmesh.spi.EventMeshExtensionType;
import org.apache.eventmesh.spi.EventMeshSPI;
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/mysql/MysqlCdcEngine.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/mysql/MysqlCdcEngine.java
index 67c837caf7..5650c3d0cc 100644
--- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/mysql/MysqlCdcEngine.java
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/mysql/MysqlCdcEngine.java
@@ -18,6 +18,7 @@
package org.apache.eventmesh.connector.jdbc.source.dialect.cdc.mysql;
import org.apache.eventmesh.common.EventMeshThreadFactory;
+import org.apache.eventmesh.connector.jdbc.CatalogChanges;
import org.apache.eventmesh.connector.jdbc.DataChanges;
import org.apache.eventmesh.connector.jdbc.DataChanges.Builder;
import org.apache.eventmesh.connector.jdbc.Field;
@@ -25,10 +26,12 @@
import org.apache.eventmesh.connector.jdbc.Schema;
import org.apache.eventmesh.connector.jdbc.config.JdbcConfig;
import org.apache.eventmesh.connector.jdbc.connection.mysql.MysqlJdbcConnection;
+import org.apache.eventmesh.connector.jdbc.dialect.mysql.MysqlDatabaseDialect;
import org.apache.eventmesh.connector.jdbc.event.DeleteDataEvent;
import org.apache.eventmesh.connector.jdbc.event.EventConsumer;
import org.apache.eventmesh.connector.jdbc.event.GeneralDataChangeEvent;
import org.apache.eventmesh.connector.jdbc.event.InsertDataEvent;
+import org.apache.eventmesh.connector.jdbc.event.SchemaChangeEventType;
import org.apache.eventmesh.connector.jdbc.event.UpdateDataEvent;
import org.apache.eventmesh.connector.jdbc.source.config.JdbcSourceConfig;
import org.apache.eventmesh.connector.jdbc.source.config.MysqlConfig;
@@ -40,12 +43,13 @@
import org.apache.eventmesh.connector.jdbc.source.dialect.mysql.EventDataDeserializationExceptionData;
import org.apache.eventmesh.connector.jdbc.source.dialect.mysql.EventMeshGtidSet;
import org.apache.eventmesh.connector.jdbc.source.dialect.mysql.MysqlConstants;
-import org.apache.eventmesh.connector.jdbc.source.dialect.mysql.MysqlDatabaseDialect;
import org.apache.eventmesh.connector.jdbc.source.dialect.mysql.MysqlJdbcContext;
import org.apache.eventmesh.connector.jdbc.source.dialect.mysql.MysqlSourceMateData;
import org.apache.eventmesh.connector.jdbc.table.catalog.Column;
+import org.apache.eventmesh.connector.jdbc.table.catalog.DefaultValueConvertor;
import org.apache.eventmesh.connector.jdbc.table.catalog.TableId;
import org.apache.eventmesh.connector.jdbc.table.catalog.TableSchema;
+import org.apache.eventmesh.connector.jdbc.table.catalog.mysql.MysqlDefaultValueConvertorImpl;
import org.apache.eventmesh.connector.jdbc.table.type.Pair;
import org.apache.eventmesh.openconnect.api.config.Config;
@@ -113,6 +117,8 @@ public class MysqlCdcEngine extends AbstractCdcEngine ignoreEvent(context, ignore)).accept(event);
@@ -386,7 +392,7 @@ private void enableGtidHandle() {
EventMeshGtidSet purgedServerEventMeshGtidSet = new EventMeshGtidSet(purgedServerGtid);
EventMeshGtidSet filteredEventMeshGtidSet = filterGtidSet(context, executedEventMeshGtidSet, purgedServerEventMeshGtidSet);
- if (null != filteredEventMeshGtidSet) {
+ if (filteredEventMeshGtidSet != null) {
client.setGtidSet(filteredEventMeshGtidSet.toString());
this.context.completedGtidSet(filteredEventMeshGtidSet.toString());
localGtidSet = new com.github.shyiko.mysql.binlog.GtidSet(filteredEventMeshGtidSet.toString());
@@ -509,6 +515,7 @@ protected void handleQueryEvent(MysqlJdbcContext context, Event event) {
// set current parse database to Ddl parser
ddlParser.setCurrentDatabase(queryEventData.getDatabase());
+ ddlParser.setCatalogTableSet(context.getCatalogTableSet());
ddlParser.parse(sql, this::handleDdlEvent);
}
@@ -516,6 +523,17 @@ private void handleDdlEvent(org.apache.eventmesh.connector.jdbc.event.Event even
if (event == null) {
return;
}
+ // handle default value expression
+ if (event.getJdbcConnectData().isSchemaChanges()) {
+ CatalogChanges catalogChanges = event.getJdbcConnectData().getPayload().getCatalogChanges();
+ SchemaChangeEventType schemaChangeEventType = SchemaChangeEventType.ofSchemaChangeEventType(catalogChanges.getType(),
+ catalogChanges.getOperationType());
+ if (SchemaChangeEventType.TABLE_CREATE == schemaChangeEventType || SchemaChangeEventType.TABLE_ALERT == schemaChangeEventType) {
+ catalogChanges.getColumns().forEach(column -> {
+ column.setDefaultValue(defaultValueConvertor.parseDefaultValue(column, column.getDefaultValueExpression()));
+ });
+ }
+ }
event.getJdbcConnectData().getPayload().ofSourceMateData().setSnapshot(false);
consumers.stream().forEach(consumer -> consumer.accept(event));
}
@@ -583,7 +601,7 @@ private MysqlSourceMateData buildMysqlSourceMateData(MysqlJdbcContext context, E
return sourceMateData;
}
- private enum CdcDmlType {
+ public enum CdcDmlType {
INSERT,
UPDATE,
DELETE
@@ -606,23 +624,28 @@ private void handleCdcDmlData(MysqlJdbcContext context, MysqlSourceMateData sour
List, Pair>> rows, CdcDmlType type) {
TableSchema tableSchema = context.getCatalogTableSet().getTableSchema(tableId);
- Map orderColumnMap = tableSchema.getOrderColumnMap();
- List columns = tableSchema.getColumns();
+ Map> orderColumnMap = tableSchema.getOrderColumnMap();
+ List> columns = tableSchema.getColumns();
List fields = null;
Builder builder = DataChanges.newBuilder();
if (CollectionUtils.isNotEmpty(columns)) {
fields = columns.stream()
- .map(col -> new Field(col.getDataType().getName(), col.isNotNull(), col.getName(), tableId.toString()))
- .collect(Collectors.toList());
+ .map(col -> {
+ Column rebuild = Column.newBuilder().withName(col.getName()).withDataType(col.getDataType()).withJdbcType(col.getJdbcType())
+ .withNativeType(col.getNativeType()).withOrder(col.getOrder()).build();
+ return new Field(rebuild, col.isNotNull(), col.getName(), tableId.toString());
+ }).collect(Collectors.toList());
}
int columnsSize = orderColumnMap.size();
for (Pair, Pair> pair : rows) {
GeneralDataChangeEvent dataEvent = buildEvent(type, tableId);
- Payload payload = dataEvent.getJdbcConnectData().getPayload();
+ builder.withType(dataEvent.getDataChangeEventType().ofCode());
Schema schema = new Schema();
+ // set primary key
+ schema.addKeys(tableSchema.getPrimaryKey().getColumnNames());
Pair beforePair = Optional.ofNullable(pair.getLeft()).orElse(new Pair<>());
Serializable[] beforeRows = beforePair.getLeft();
- if (null != beforeRows && beforeRows.length != 0) {
+ if (beforeRows != null && beforeRows.length != 0) {
BitSet includedColumns = beforePair.getRight();
Map beforeValues = new HashMap<>(beforeRows.length);
for (int index = 0; index < columnsSize; ++index) {
@@ -633,14 +656,14 @@ private void handleCdcDmlData(MysqlJdbcContext context, MysqlSourceMateData sour
beforeValues.put(orderColumnMap.get(index + 1).getName(), beforeRows[index]);
}
builder.withBefore(beforeValues);
- Field beforeField = new Field().withField(Payload.BEFORE_FIELD).withType("field").withName("payload.before").withRequired(false);
+ Field beforeField = new Field().withField(Payload.BEFORE_FIELD).withName(Payload.PAYLOAD_BEFORE).withRequired(false);
beforeField.withRequired(true).withFields(fields);
schema.add(beforeField);
}
Pair afterPair = Optional.ofNullable(pair.getRight()).orElse(new Pair<>());
Serializable[] afterRows = afterPair.getLeft();
- if (null != afterRows && afterRows.length != 0) {
+ if (afterRows != null && afterRows.length != 0) {
BitSet includedColumns = afterPair.getRight();
Map afterValues = new HashMap<>(afterRows.length);
for (int index = 0; index < columnsSize; ++index) {
@@ -650,15 +673,15 @@ private void handleCdcDmlData(MysqlJdbcContext context, MysqlSourceMateData sour
}
afterValues.put(orderColumnMap.get(index + 1).getName(), afterRows[index]);
}
- builder.withBefore(afterValues);
- Field afterField = new Field().withField(Payload.AFTER_FIELD).withType("field").withName("payload.after").withRequired(false);
+ builder.withAfter(afterValues);
+ Field afterField = new Field().withField(Payload.AFTER_FIELD).withName(Payload.PAYLOAD_AFTER).withRequired(false);
afterField.withRequired(true).withFields(fields);
schema.add(afterField);
}
+ Payload payload = dataEvent.getJdbcConnectData().getPayload();
payload.withSource(sourceMateData).withDataChanges(builder.build());
dataEvent.getJdbcConnectData().setSchema(schema);
consumers.stream().forEach(consumer -> consumer.accept(dataEvent));
-
}
}
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/mysql/MysqlCdcEngineFactory.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/mysql/MysqlCdcEngineFactory.java
index 60c3c7b7c1..35e722fe12 100644
--- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/mysql/MysqlCdcEngineFactory.java
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/mysql/MysqlCdcEngineFactory.java
@@ -17,11 +17,11 @@
package org.apache.eventmesh.connector.jdbc.source.dialect.cdc.mysql;
-import org.apache.eventmesh.connector.jdbc.DatabaseDialect;
+import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialect;
+import org.apache.eventmesh.connector.jdbc.dialect.mysql.MysqlDatabaseDialect;
import org.apache.eventmesh.connector.jdbc.source.dialect.cdc.AbstractCdcEngineFactory;
import org.apache.eventmesh.connector.jdbc.source.dialect.cdc.CdcEngine;
import org.apache.eventmesh.connector.jdbc.source.dialect.cdc.CdcEngineFactory;
-import org.apache.eventmesh.connector.jdbc.source.dialect.mysql.MysqlDatabaseDialect;
import org.apache.eventmesh.openconnect.api.config.SourceConfig;
import org.apache.commons.lang3.StringUtils;
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/mysql/RowDeserializers.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/mysql/RowDeserializers.java
index f605e95bde..e23839563f 100644
--- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/mysql/RowDeserializers.java
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/mysql/RowDeserializers.java
@@ -23,9 +23,9 @@
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.LocalTime;
-import java.time.Year;
import java.time.ZoneOffset;
import java.time.ZonedDateTime;
+import java.util.BitSet;
import java.util.Map;
import com.github.shyiko.mysql.binlog.event.TableMapEventData;
@@ -99,6 +99,11 @@ protected Serializable deserializeTimestampV2(int meta, ByteArrayInputStream inp
protected Serializable deserializeYear(ByteArrayInputStream inputStream) throws IOException {
return RowDeserializers.deserializeYear(inputStream);
}
+
+ @Override
+ protected Serializable deserializeBit(int meta, ByteArrayInputStream inputStream) throws IOException {
+ return ((BitSet) super.deserializeBit(meta, inputStream)).toByteArray();
+ }
}
public static class UpdateRowsEventMeshDeserializer extends UpdateRowsEventDataDeserializer {
@@ -156,6 +161,12 @@ protected Serializable deserializeTimestampV2(int meta, ByteArrayInputStream inp
protected Serializable deserializeYear(ByteArrayInputStream inputStream) throws IOException {
return RowDeserializers.deserializeYear(inputStream);
}
+
+ @Override
+ protected Serializable deserializeBit(int meta, ByteArrayInputStream inputStream) throws IOException {
+ return ((BitSet) super.deserializeBit(meta, inputStream)).toByteArray();
+
+ }
}
public static class DeleteRowsEventMeshDeserializer extends DeleteRowsEventDataDeserializer {
@@ -213,6 +224,10 @@ protected Serializable deserializeTimestampV2(int meta, ByteArrayInputStream inp
protected Serializable deserializeYear(ByteArrayInputStream inputStream) throws IOException {
return RowDeserializers.deserializeYear(inputStream);
}
+
+ protected Serializable deserializeBit(int meta, ByteArrayInputStream inputStream) throws IOException {
+ return ((BitSet) super.deserializeBit(meta, inputStream)).toByteArray();
+ }
}
protected static Serializable deserializeTimestamp(ByteArrayInputStream inputStream) throws IOException {
@@ -228,7 +243,7 @@ protected static Serializable deserializeTimestampV2(int meta, ByteArrayInputStr
}
protected static Serializable deserializeYear(ByteArrayInputStream inputStream) throws IOException {
- return Year.of(1900 + inputStream.readInteger(1));
+ return LocalDate.parse(String.format("%d-01-01", 1900 + inputStream.readInteger(1)));
}
/**
@@ -448,4 +463,8 @@ protected static int deserializeFractionalSecondsInNanos(int fsp, ByteArrayInput
return 0;
}
+ protected static byte[] deserializeBit(int meta, ByteArrayInputStream inputStream) throws IOException {
+ return inputStream.read(meta);
+ }
+
}
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/mysql/MysqlDataTypeConvertor.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/mysql/MysqlDataTypeConvertor.java
index 33b26ebe72..322e4f5f18 100644
--- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/mysql/MysqlDataTypeConvertor.java
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/mysql/MysqlDataTypeConvertor.java
@@ -19,12 +19,23 @@
import org.apache.eventmesh.connector.jdbc.DataTypeConvertor;
import org.apache.eventmesh.connector.jdbc.exception.DataTypeConvertException;
-import org.apache.eventmesh.connector.jdbc.table.type.CalendarType;
-import org.apache.eventmesh.connector.jdbc.table.type.DecimalType;
import org.apache.eventmesh.connector.jdbc.table.type.EventMeshDataType;
-import org.apache.eventmesh.connector.jdbc.table.type.PrimitiveByteArrayType;
-import org.apache.eventmesh.connector.jdbc.table.type.PrimitiveType;
import org.apache.eventmesh.connector.jdbc.table.type.SQLType;
+import org.apache.eventmesh.connector.jdbc.type.eventmesh.BooleanEventMeshDataType;
+import org.apache.eventmesh.connector.jdbc.type.eventmesh.BytesEventMeshDataType;
+import org.apache.eventmesh.connector.jdbc.type.eventmesh.DateEventMeshDataType;
+import org.apache.eventmesh.connector.jdbc.type.eventmesh.DateTimeEventMeshDataType;
+import org.apache.eventmesh.connector.jdbc.type.eventmesh.DecimalEventMeshDataType;
+import org.apache.eventmesh.connector.jdbc.type.eventmesh.Float32EventMeshDataType;
+import org.apache.eventmesh.connector.jdbc.type.eventmesh.Float64EventMeshDataType;
+import org.apache.eventmesh.connector.jdbc.type.eventmesh.Int16EventMeshDataType;
+import org.apache.eventmesh.connector.jdbc.type.eventmesh.Int32EventMeshDataType;
+import org.apache.eventmesh.connector.jdbc.type.eventmesh.Int64EventMeshDataType;
+import org.apache.eventmesh.connector.jdbc.type.eventmesh.Int8EventMeshDataType;
+import org.apache.eventmesh.connector.jdbc.type.eventmesh.NullEventMeshDataType;
+import org.apache.eventmesh.connector.jdbc.type.eventmesh.StringEventMeshDataType;
+import org.apache.eventmesh.connector.jdbc.type.eventmesh.TimeEventMeshDataType;
+import org.apache.eventmesh.connector.jdbc.type.eventmesh.YearEventMeshDataType;
import java.sql.JDBCType;
import java.util.Map;
@@ -98,49 +109,50 @@ public EventMeshDataType toEventMeshType(MysqlType connectorDataType, MapMysql doc
*/
- if (null == dataTypeProperties) {
- return PrimitiveByteArrayType.BYTES_TYPE;
+ if (dataTypeProperties == null) {
+ return BytesEventMeshDataType.INSTANCE;
}
Integer precision = (Integer) dataTypeProperties.get(MysqlDataTypeConvertor.PRECISION);
if (precision != null && precision == 1) {
- return PrimitiveType.BOOLEAN_TYPE;
+ return BooleanEventMeshDataType.INSTANCE;
}
- return PrimitiveByteArrayType.BYTES_TYPE;
+ return BytesEventMeshDataType.INSTANCE;
}
case TINYINT:
- return PrimitiveType.BYTE_TYPE;
+ return Int8EventMeshDataType.INSTANCE;
case TINYINT_UNSIGNED:
case SMALLINT:
- return PrimitiveType.SHORT_TYPE;
+ return Int16EventMeshDataType.INSTANCE;
case SMALLINT_UNSIGNED:
case INT:
case MEDIUMINT:
case MEDIUMINT_UNSIGNED:
- return PrimitiveType.INT_TYPE;
+ return Int32EventMeshDataType.INSTANCE;
case INT_UNSIGNED:
case BIGINT:
- return PrimitiveType.LONG_TYPE;
+ return Int64EventMeshDataType.INSTANCE;
case FLOAT:
case FLOAT_UNSIGNED:
- return PrimitiveType.FLOAT_TYPE;
+ return Float32EventMeshDataType.INSTANCE;
case DOUBLE:
case DOUBLE_UNSIGNED:
- return PrimitiveType.DOUBLE_TYPE;
+ return Float64EventMeshDataType.INSTANCE;
case TIME:
- return CalendarType.LOCAL_TIME_TYPE;
+ return TimeEventMeshDataType.INSTANCE;
case YEAR:
+ return YearEventMeshDataType.INSTANCE;
case DATE:
- return CalendarType.LOCAL_DATE_TYPE;
+ return DateEventMeshDataType.INSTANCE;
case TIMESTAMP:
case DATETIME:
- return CalendarType.LOCAL_DATE_TIME_TYPE;
+ return DateTimeEventMeshDataType.INSTANCE;
case CHAR:
case VARCHAR:
case TINYTEXT:
@@ -149,7 +161,8 @@ public EventMeshDataType toEventMeshType(MysqlType connectorDataType, Map toEventMeshType(MysqlType connectorDataType, Map toEventMeshType(MysqlType connectorDataType, MapMysql doc-DECIMAL, NUMERIC
*/
if (dataTypeProperties == null) {
- return new DecimalType(DEFAULT_PRECISION, DEFAULT_SCALE);
+ return new DecimalEventMeshDataType(DEFAULT_PRECISION, DEFAULT_SCALE);
}
Integer precision = (Integer) dataTypeProperties.getOrDefault(PRECISION, DEFAULT_PRECISION);
Integer scale = (Integer) dataTypeProperties.getOrDefault(SCALE, DEFAULT_SCALE);
- return new DecimalType(precision, scale);
+ return new DecimalEventMeshDataType(precision, scale);
}
default:
throw new DataTypeConvertException(String.format("%s type is not supported", connectorDataType.getName()));
@@ -218,8 +231,6 @@ public MysqlType toConnectorType(EventMeshDataType eventMeshDataType, Map snapshot
private Callable createSnapshotDataEvent4TableCallable(Jc context, SnapshotContext snapshotContext,
Queue connectionPool, String sql, TableId tableId) {
UniversalJdbcContext universalJdbcContext = (UniversalJdbcContext) context;
- universalJdbcContext.withTableId(tableId);
+ //universalJdbcContext.withTableId(tableId);
return () -> {
JdbcConnection connection = connectionPool.poll();
- MysqlSourceMateData sourceMateData = MysqlSourceMateData.newBuilder()
- .name(sourceConnectorConfig.getName())
- .snapshot(true)
- .withTableId(tableId)
- .serverId(sourceConnectorConfig.getMysqlConfig().getServerId())
- .build();
+ SourceMateData sourceMateData = buildSourceMateData(context, snapshotContext, tableId);
TableSchema tableSchema = universalJdbcContext.getCatalogTableSet().getTableSchema(tableId);
- Field field = new Field().withField("after").withType("field").withName("payload.after").withRequired(false);
+ Field field = new Field().withField("after").withName("payload.after").withRequired(false);
List columns = tableSchema.getColumns();
if (CollectionUtils.isNotEmpty(columns)) {
- List fields = columns.stream()
- .map(col -> new Field(col.getDataType().getName(), col.isNotNull(), col.getName(), tableId.toString()))
- .collect(Collectors.toList());
+ List fields = columns.stream().map(col -> {
+ Column rebuild = Column.newBuilder().withName(col.getName()).withDataType(col.getDataType())
+ .withJdbcType(col.getJdbcType()).withNativeType(col.getNativeType()).withOrder(col.getOrder()).build();
+ return new Field(rebuild, col.isNotNull(), col.getName(), tableId.toString());
+ }).collect(Collectors.toList());
field.withRequired(true).withFields(fields);
}
try (Statement statement = connection.createStatement(jdbcSourceConfig.getSourceConnectorConfig().getSnapshotFetchSize(), 100)) {
@@ -239,13 +236,14 @@ private Callable createSnapshotDataEvent4TableCallable(Jc context, Snapsho
while (resultSet.next()) {
int columnCount = resultSet.getMetaData().getColumnCount();
InsertDataEvent event = new InsertDataEvent(tableId);
- Payload payload = event.getJdbcConnectData().getPayload();
Map values = new HashMap<>(columnCount);
for (int index = 1; index <= columnCount; ++index) {
values.put(resultSet.getMetaData().getColumnName(index), resultSet.getObject(index));
}
Builder builder = DataChanges.newBuilder();
builder.withAfter(values);
+ builder.withType(event.getDataChangeEventType().ofCode());
+ final Payload payload = event.getJdbcConnectData().getPayload();
payload.withDataChanges(builder.build());
payload.withSource(sourceMateData);
event.getJdbcConnectData().setSchema(new Schema(Collections.singletonList(field)));
@@ -273,6 +271,16 @@ private Queue createConnectionPool(final SnapshotContext snapshotContext, TableId tableId);
+
/**
* Pre-snapshot preparations.
*
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/SnapshotEngineFactory.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/SnapshotEngineFactory.java
index 164b5b97b4..d573d7081a 100644
--- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/SnapshotEngineFactory.java
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/SnapshotEngineFactory.java
@@ -17,8 +17,8 @@
package org.apache.eventmesh.connector.jdbc.source.dialect.snapshot;
-import org.apache.eventmesh.connector.jdbc.DatabaseDialect;
import org.apache.eventmesh.connector.jdbc.JdbcContext;
+import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialect;
import org.apache.eventmesh.connector.jdbc.source.config.JdbcSourceConfig;
import org.apache.eventmesh.spi.EventMeshExtensionType;
import org.apache.eventmesh.spi.EventMeshSPI;
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/mysql/MysqlSnapshotEngine.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/mysql/MysqlSnapshotEngine.java
index 2d4a7c2d31..e76ab49a3e 100644
--- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/mysql/MysqlSnapshotEngine.java
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/mysql/MysqlSnapshotEngine.java
@@ -17,19 +17,25 @@
package org.apache.eventmesh.connector.jdbc.source.dialect.snapshot.mysql;
+import org.apache.eventmesh.connector.jdbc.CatalogChanges;
import org.apache.eventmesh.connector.jdbc.connection.mysql.MysqlJdbcConnection;
import org.apache.eventmesh.connector.jdbc.context.mysql.MysqlOffsetContext;
import org.apache.eventmesh.connector.jdbc.context.mysql.MysqlPartition;
+import org.apache.eventmesh.connector.jdbc.dialect.mysql.MysqlDatabaseDialect;
import org.apache.eventmesh.connector.jdbc.event.Event;
import org.apache.eventmesh.connector.jdbc.event.EventConsumer;
+import org.apache.eventmesh.connector.jdbc.event.SchemaChangeEventType;
+import org.apache.eventmesh.connector.jdbc.source.SourceMateData;
import org.apache.eventmesh.connector.jdbc.source.config.JdbcSourceConfig;
import org.apache.eventmesh.connector.jdbc.source.config.MysqlConfig;
import org.apache.eventmesh.connector.jdbc.source.dialect.mysql.MysqlConstants;
-import org.apache.eventmesh.connector.jdbc.source.dialect.mysql.MysqlDatabaseDialect;
import org.apache.eventmesh.connector.jdbc.source.dialect.mysql.MysqlDialectSql;
import org.apache.eventmesh.connector.jdbc.source.dialect.mysql.MysqlJdbcContext;
+import org.apache.eventmesh.connector.jdbc.source.dialect.mysql.MysqlSourceMateData;
import org.apache.eventmesh.connector.jdbc.source.dialect.snapshot.AbstractSnapshotEngine;
+import org.apache.eventmesh.connector.jdbc.table.catalog.DefaultValueConvertor;
import org.apache.eventmesh.connector.jdbc.table.catalog.TableId;
+import org.apache.eventmesh.connector.jdbc.table.catalog.mysql.MysqlDefaultValueConvertorImpl;
import org.apache.eventmesh.connector.jdbc.utils.MysqlUtils;
import org.apache.commons.collections4.CollectionUtils;
@@ -60,6 +66,8 @@ public class MysqlSnapshotEngine extends
private MysqlJdbcConnection connection;
+ private DefaultValueConvertor defaultValueConvertor = new MysqlDefaultValueConvertorImpl();
+
public MysqlSnapshotEngine(JdbcSourceConfig jdbcSourceConfig, MysqlDatabaseDialect databaseDialect, MysqlJdbcContext jdbcContext) {
super(jdbcSourceConfig, databaseDialect, jdbcContext, jdbcContext.getPartition(), jdbcContext.getOffsetContext());
this.connection = databaseDialect.getConnection();
@@ -76,6 +84,29 @@ public void close() throws Exception {
shutdown();
}
+ /**
+ * Builds the source metadata.
+ *
+ * @param context The context.
+ * @param snapshotContext The snapshot context.
+ * @param tableId The table id
+ * @return The source metadata.
+ */
+ @Override
+ protected SourceMateData buildSourceMateData(MysqlJdbcContext context, SnapshotContext snapshotContext,
+ TableId tableId) {
+
+ MysqlSourceMateData sourceMateData = MysqlSourceMateData.newBuilder()
+ .name(sourceConnectorConfig.getName())
+ .withTableId(tableId)
+ .serverId(sourceConnectorConfig.getMysqlConfig().getServerId())
+ .snapshot(true)
+ .position(context.getSourceInfo().getCurrentBinlogPosition())
+ .build();
+
+ return sourceMateData;
+ }
+
@Override
protected void preSnapshot(MysqlJdbcContext jdbcContext, SnapshotContext snapshotContext) {
// nothing to do
@@ -185,7 +216,17 @@ private void addParseDdlAndEvent(MysqlJdbcContext jdbcContext, String ddl, Table
if (event == null) {
return;
}
- event.getJdbcConnectData().getPayload().ofSourceMateData().setSnapshot(true);
+ // handle default value expression
+ if (event.getJdbcConnectData().isSchemaChanges()) {
+ CatalogChanges catalogChanges = event.getJdbcConnectData().getPayload().getCatalogChanges();
+ SchemaChangeEventType schemaChangeEventType = SchemaChangeEventType.ofSchemaChangeEventType(catalogChanges.getType(),
+ catalogChanges.getOperationType());
+ if (SchemaChangeEventType.TABLE_CREATE == schemaChangeEventType || SchemaChangeEventType.TABLE_ALERT == schemaChangeEventType) {
+ catalogChanges.getColumns().forEach(
+ column -> column.setDefaultValue(defaultValueConvertor.parseDefaultValue(column, column.getDefaultValueExpression())));
+ }
+ }
+ event.getJdbcConnectData().getPayload().withDdl(ddl).ofSourceMateData().setSnapshot(true);
eventQueue.put(event);
} catch (InterruptedException e) {
throw new RuntimeException(e);
@@ -270,7 +311,7 @@ public void run() {
while (isRunning) {
try {
Event event = eventQueue.poll(5, TimeUnit.SECONDS);
- if (null == event) {
+ if (event == null) {
continue;
}
consumers.forEach(consumer -> consumer.accept(event));
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/mysql/MysqlSnapshotEngineFactory.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/mysql/MysqlSnapshotEngineFactory.java
index 53c5f993da..210ded0edf 100644
--- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/mysql/MysqlSnapshotEngineFactory.java
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/mysql/MysqlSnapshotEngineFactory.java
@@ -17,10 +17,10 @@
package org.apache.eventmesh.connector.jdbc.source.dialect.snapshot.mysql;
-import org.apache.eventmesh.connector.jdbc.DatabaseDialect;
+import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialect;
+import org.apache.eventmesh.connector.jdbc.dialect.mysql.MysqlDatabaseDialect;
import org.apache.eventmesh.connector.jdbc.source.config.JdbcSourceConfig;
import org.apache.eventmesh.connector.jdbc.source.dialect.antlr4.mysql.MysqlAntlr4DdlParser;
-import org.apache.eventmesh.connector.jdbc.source.dialect.mysql.MysqlDatabaseDialect;
import org.apache.eventmesh.connector.jdbc.source.dialect.mysql.MysqlJdbcContext;
import org.apache.eventmesh.connector.jdbc.source.dialect.snapshot.SnapshotEngine;
import org.apache.eventmesh.connector.jdbc.source.dialect.snapshot.SnapshotEngineFactory;
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/AbstractColumnEditorImpl.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/AbstractColumnEditorImpl.java
index 146d232a2d..a3bcc97f8e 100644
--- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/AbstractColumnEditorImpl.java
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/AbstractColumnEditorImpl.java
@@ -20,6 +20,7 @@
import org.apache.eventmesh.connector.jdbc.table.type.EventMeshDataType;
import java.sql.JDBCType;
+import java.util.List;
public abstract class AbstractColumnEditorImpl implements ColumnEditor {
@@ -36,7 +37,7 @@ public abstract class AbstractColumnEditorImpl enumValues;
+
+ private Options options;
+
public AbstractColumnEditorImpl(String name) {
this.name = name;
}
@@ -132,7 +146,7 @@ public CE withJdbcType(JDBCType jdbcType) {
*/
@SuppressWarnings("unchecked")
@Override
- public CE withEventMeshType(EventMeshDataType eventMeshType) {
+ public CE withEventMeshType(EventMeshDataType eventMeshType) {
this.eventMeshDataType = eventMeshType;
return (CE) this;
}
@@ -158,7 +172,7 @@ public CE withOrder(int order) {
*/
@SuppressWarnings("unchecked")
@Override
- public CE length(int length) {
+ public CE length(long length) {
this.columnLength = length;
return (CE) this;
}
@@ -241,11 +255,47 @@ public CE notNull(boolean notNull) {
return (CE) this;
}
- public EventMeshDataType ofEventMeshDataType() {
+ @SuppressWarnings("unchecked")
+ @Override
+ public CE charsetName(String charsetName) {
+ this.charsetName = charsetName;
+ return (CE) this;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public CE enumValues(List enumValues) {
+ this.enumValues = enumValues;
+ return (CE) this;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public CE withOption(String key, Object value) {
+ if (options == null) {
+ this.options = new Options();
+ }
+ this.options.put(key, value);
+ return (CE) this;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public CE withOptions(Options options) {
+ if (options != null) {
+ if (this.options == null) {
+ this.options = new Options();
+ }
+ this.options.putAll(options);
+ }
+ return (CE) this;
+ }
+
+ public EventMeshDataType ofEventMeshDataType() {
return eventMeshDataType;
}
- public Integer ofColumnLength() {
+ public Long ofColumnLength() {
return columnLength;
}
@@ -284,4 +334,16 @@ public boolean isOptional() {
public int ofOrder() {
return this.order;
}
+
+ public String ofCharsetName() {
+ return this.charsetName;
+ }
+
+ public List ofEnumValues() {
+ return this.enumValues;
+ }
+
+ public Options ofOptions() {
+ return this.options;
+ }
}
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/AbstractTableEditorImpl.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/AbstractTableEditorImpl.java
index 61a7c7bf3f..4d26b3cb85 100644
--- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/AbstractTableEditorImpl.java
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/AbstractTableEditorImpl.java
@@ -174,11 +174,6 @@ public TE withUniqueKeys(UniqueKey... uniqueKeys) {
return (TE) this;
}
- /**
- * @param key
- * @param value
- * @return
- */
@Override
@SuppressWarnings("unchecked")
public TE withOption(String key, Object value) {
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/CatalogTableSet.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/CatalogTableSet.java
index ed2b6dc6e4..16c083f06a 100644
--- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/CatalogTableSet.java
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/CatalogTableSet.java
@@ -51,6 +51,7 @@ public void overrideTable(TableSchema tableSchema) {
return;
}
tableSchemaMap.putTableSchema(tableSchema);
+ tableIdSet.addTableId(tableSchema.getTableId());
}
public TableSchema getTableSchema(TableId tableId) {
@@ -65,6 +66,10 @@ public TableIdSet() {
values = new HashSet<>(32);
}
+ public void addTableId(TableId tableId) {
+ values.add(tableId);
+ }
+
public void removeDatabase(String catalogName, String schemaName) {
values.removeIf(
entry -> StringUtils.equals(entry.getCatalogName(), catalogName) && StringUtils.equals(entry.getSchemaName(), schemaName));
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/Column.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/Column.java
index 94ed25f24e..363a50c147 100644
--- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/Column.java
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/Column.java
@@ -22,6 +22,12 @@
import java.io.Serializable;
import java.sql.JDBCType;
import java.sql.Types;
+import java.util.List;
+import java.util.Optional;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import lombok.AllArgsConstructor;
import lombok.Data;
@@ -33,7 +39,7 @@
@Data
@AllArgsConstructor
@NoArgsConstructor
-public abstract class Column implements Serializable {
+public class Column implements Serializable {
/**
* Name of the column
@@ -43,7 +49,9 @@ public abstract class Column implements Serializable {
/**
* Data type of the column
*/
- protected EventMeshDataType dataType;
+ @JsonSerialize(using = EventMeshDataTypeJsonSerializer.class)
+ @JsonDeserialize(using = EventMeshDataTypeJsonDeserializer.class)
+ protected EventMeshDataType dataType;
/**
* {@link Types JDBC type}
@@ -53,7 +61,7 @@ public abstract class Column implements Serializable {
/**
* Length of the column
*/
- protected Integer columnLength;
+ protected Long columnLength;
/**
* Decimal point of the column
@@ -63,7 +71,7 @@ public abstract class Column implements Serializable {
/**
* Indicates if the column can be null or not
*/
- protected boolean notNull;
+ protected boolean notNull = false;
/**
* Comment for the column
@@ -75,15 +83,202 @@ public abstract class Column implements Serializable {
*/
protected Object defaultValue;
+ @JsonIgnore
protected String defaultValueExpression;
- protected int order;
+ // order of the column in the table
+ protected int order = 1;
+
+ protected String charsetName;
+
+ // Use wrapper types to reduce data transmission during serialization
+ protected Boolean autoIncremented;
+
+ protected Boolean generated;
+
+ protected String collationName;
+
+ protected List enumValues;
+
+ // for mysql: varchar or json
+ protected String nativeType;
+
+ protected Options options;
+
+ public Column(String name, EventMeshDataType dataType, JDBCType jdbcType, Long columnLength, Integer decimal, boolean notNull, String comment,
+ Object defaultValue, String defaultValueExpression, int order, String charsetName, boolean autoIncremented, boolean generated,
+ String collationName) {
+ this.name = name;
+ this.dataType = dataType;
+ this.jdbcType = jdbcType;
+ this.columnLength = columnLength;
+ this.decimal = decimal;
+ this.notNull = notNull;
+ this.comment = comment;
+ this.defaultValue = defaultValue;
+ this.defaultValueExpression = defaultValueExpression;
+ this.order = order;
+ this.charsetName = charsetName;
+ this.autoIncremented = autoIncremented;
+ this.generated = generated;
+ this.collationName = collationName;
+ }
+
+ private Column(Builder builder) {
+ this.name = builder.name;
+ this.dataType = builder.dataType;
+ this.jdbcType = builder.jdbcType;
+ this.columnLength = builder.columnLength;
+ this.decimal = builder.decimal;
+ this.notNull = builder.notNull;
+ this.comment = builder.comment;
+ this.defaultValue = builder.defaultValue;
+ this.defaultValueExpression = builder.defaultValueExpression;
+ this.order = builder.order;
+ this.charsetName = builder.charsetName;
+ this.autoIncremented = builder.autoIncremented;
+ this.generated = builder.generated;
+ this.collationName = builder.collationName;
+ this.enumValues = builder.enumValues;
+ this.nativeType = builder.nativeType;
+ this.options = builder.options;
+ }
+
+ public boolean isAutoIncremented() {
+ return Optional.ofNullable(this.autoIncremented).orElse(false);
+ }
+
+ public static Builder newBuilder() {
+ return new Builder();
+ }
/**
* creates a clone of the Column
*
* @return clone of column
*/
- public abstract Col clone();
+ public Col clone() {
+ return null;
+ }
+
+ /**
+ * Builder for the Column class
+ */
+ public static class Builder {
+
+ protected String name;
+ protected EventMeshDataType dataType;
+ protected JDBCType jdbcType;
+ protected Long columnLength;
+ protected Integer decimal;
+ protected boolean notNull = false;
+ protected String comment;
+ protected Object defaultValue;
+ protected String defaultValueExpression;
+ protected int order = 1;
+ protected String charsetName;
+ protected Boolean autoIncremented;
+ protected Boolean generated;
+ protected String collationName;
+ protected List enumValues;
+ // for mysql: varchar or json
+ protected String nativeType;
+
+ protected Options options;
+
+ public Builder withName(String name) {
+ this.name = name;
+ return this;
+ }
+
+ public Builder withDataType(EventMeshDataType dataType) {
+ this.dataType = dataType;
+ return this;
+ }
+
+ public Builder withJdbcType(JDBCType jdbcType) {
+ this.jdbcType = jdbcType;
+ return this;
+ }
+
+ public Builder withColumnLength(Long columnLength) {
+ this.columnLength = columnLength;
+ return this;
+ }
+
+ public Builder withDecimal(Integer decimal) {
+ this.decimal = decimal;
+ return this;
+ }
+
+ public Builder withNotNull(boolean notNull) {
+ this.notNull = notNull;
+ return this;
+ }
+
+ public Builder withComment(String comment) {
+ this.comment = comment;
+ return this;
+ }
+
+ public Builder withDefaultValue(Object defaultValue) {
+ this.defaultValue = defaultValue;
+ return this;
+ }
+
+ public Builder withDefaultValueExpression(String defaultValueExpression) {
+ this.defaultValueExpression = defaultValueExpression;
+ return this;
+ }
+
+ public Builder withOrder(int order) {
+ this.order = order;
+ return this;
+ }
+
+ public Builder withCharsetName(String charsetName) {
+ this.charsetName = charsetName;
+ return this;
+ }
+
+ public Builder withAutoIncremented(boolean autoIncremented) {
+ this.autoIncremented = autoIncremented;
+ return this;
+ }
+
+ public Builder withGenerated(boolean generated) {
+ this.generated = generated;
+ return this;
+ }
+
+ public Builder withCollationName(String collationName) {
+ this.collationName = collationName;
+ return this;
+ }
+
+ public Builder withEnumValues(List enumValues) {
+ this.enumValues = enumValues;
+ return this;
+ }
+
+ public Builder withNativeType(String nativeType) {
+ this.nativeType = nativeType;
+ return this;
+ }
+
+ public Builder withOptions(Options options) {
+ this.options = options;
+ return this;
+ }
+
+ /**
+ * Builds the Column instance.
+ *
+ * @return Column instance
+ */
+ public Column build() {
+ return new Column(this);
+ }
+ }
}
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/ColumnEditor.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/ColumnEditor.java
index 2661c7db27..e99abaccd8 100644
--- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/ColumnEditor.java
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/ColumnEditor.java
@@ -20,6 +20,7 @@
import org.apache.eventmesh.connector.jdbc.table.type.EventMeshDataType;
import java.sql.JDBCType;
+import java.util.List;
/**
* An interface for building and configuring columns in a database table.
@@ -66,7 +67,7 @@ public interface ColumnEditor {
* @param eventMeshType The EventMesh data type of the column.
* @return The column editor instance.
*/
- CE withEventMeshType(EventMeshDataType eventMeshType);
+ CE withEventMeshType(EventMeshDataType eventMeshType);
/**
* Sets the order or position of the column within a table.
@@ -82,7 +83,7 @@ public interface ColumnEditor {
* @param length The length of the column.
* @return The column editor instance.
*/
- CE length(int length);
+ CE length(long length);
/**
* Sets the scale of the column (if applicable).
@@ -132,10 +133,19 @@ public interface ColumnEditor {
*/
CE notNull(boolean notNull);
+ CE charsetName(String charsetName);
+
+ CE enumValues(List enumValues);
+
+ CE withOption(String key, Object value);
+
+ CE withOptions(Options options);
+
/**
* Builds and returns the configured column.
*
* @return The configured column.
*/
Col build();
+
}
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/DefaultColumn.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/DefaultColumn.java
index f682ea9438..e0dae1e617 100644
--- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/DefaultColumn.java
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/DefaultColumn.java
@@ -28,26 +28,34 @@
@NoArgsConstructor
public class DefaultColumn extends Column {
- public DefaultColumn(String name, EventMeshDataType dataType, JDBCType jdbcType, Integer columnLength, Integer decimal, boolean notNull,
- String comment, Object defaultValue, String defaultValueExpression) {
- super(name, dataType, jdbcType, columnLength, decimal, notNull, comment, defaultValue, defaultValueExpression, 0);
+ public DefaultColumn(String name, EventMeshDataType dataType, JDBCType jdbcType, Long columnLength, Integer decimal, boolean notNull,
+ String comment, Object defaultValue, String defaultValueExpression, String charsetName, boolean autoIncremented, boolean generated,
+ String collationName) {
+ super(name, dataType, jdbcType, columnLength, decimal, notNull, comment, defaultValue, defaultValueExpression, 0, charsetName,
+ autoIncremented, generated, collationName);
}
- public DefaultColumn(String name, EventMeshDataType dataType, JDBCType jdbcType, Integer columnLength, Integer decimal, boolean notNull,
- String comment, Object defaultValue, String defaultValueExpression, int order) {
- super(name, dataType, jdbcType, columnLength, decimal, notNull, comment, defaultValue, defaultValueExpression, order);
+ public DefaultColumn(String name, EventMeshDataType dataType, JDBCType jdbcType, Long columnLength, Integer decimal, boolean notNull,
+ String comment, Object defaultValue, String defaultValueExpression, int order, String charsetName, boolean autoIncremented, boolean generated,
+ String collationName) {
+ super(name, dataType, jdbcType, columnLength, decimal, notNull, comment, defaultValue, defaultValueExpression, order, charsetName,
+ autoIncremented, generated, collationName);
}
public static DefaultColumn of(
- String name, EventMeshDataType dataType, JDBCType jdbcType, Integer columnLength, Integer decimal, boolean notNull,
- String comment, Object defaultValue, String defaultValueExpression) {
- return new DefaultColumn(name, dataType, jdbcType, columnLength, decimal, notNull, comment, defaultValue, defaultValueExpression);
+ String name, EventMeshDataType dataType, JDBCType jdbcType, Long columnLength, Integer decimal, boolean notNull,
+ String comment, Object defaultValue, String defaultValueExpression, String charsetName, boolean autoIncremented, boolean generated,
+ String collationName) {
+ return new DefaultColumn(name, dataType, jdbcType, columnLength, decimal, notNull, comment, defaultValue, defaultValueExpression,
+ charsetName, autoIncremented, generated, collationName);
}
public static DefaultColumn of(
- String name, EventMeshDataType dataType, JDBCType jdbcType, Integer columnLength, Integer decimal, boolean notNull,
- String comment, Object defaultValue, String defaultValueExpression, int order) {
- return new DefaultColumn(name, dataType, jdbcType, columnLength, decimal, notNull, comment, defaultValue, defaultValueExpression, order);
+ String name, EventMeshDataType dataType, JDBCType jdbcType, Long columnLength, Integer decimal, boolean notNull,
+ String comment, Object defaultValue, String defaultValueExpression, int order, String charsetName, boolean autoIncremented, boolean generated,
+ String collationName) {
+ return new DefaultColumn(name, dataType, jdbcType, columnLength, decimal, notNull, comment, defaultValue, defaultValueExpression, order,
+ charsetName, autoIncremented, generated, collationName);
}
/**
@@ -57,6 +65,7 @@ public static DefaultColumn of(
*/
@Override
public DefaultColumn clone() {
- return DefaultColumn.of(name, dataType, jdbcType, columnLength, decimal, notNull, comment, defaultValue, defaultValueExpression, order);
+ return DefaultColumn.of(name, dataType, jdbcType, columnLength, decimal, notNull, comment, defaultValue, defaultValueExpression, order,
+ charsetName, autoIncremented, generated, collationName);
}
}
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/DefaultValueConvertor.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/DefaultValueConvertor.java
new file mode 100644
index 0000000000..9c3e6d4185
--- /dev/null
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/DefaultValueConvertor.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eventmesh.connector.jdbc.table.catalog;
+
+/**
+ * Functional interface for converting default values.
+ */
+@FunctionalInterface
+public interface DefaultValueConvertor {
+
+ /**
+ * Parses the default value expression for a column.
+ *
+ * @param column The column for which the default value is being parsed.
+ * @param defaultValueExpression The expression representing the default value.
+ * @return The parsed default value.
+ */
+ Object parseDefaultValue(Column column, String defaultValueExpression);
+}
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/EventMeshDataTypeJsonDeserializer.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/EventMeshDataTypeJsonDeserializer.java
new file mode 100644
index 0000000000..86b5530614
--- /dev/null
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/EventMeshDataTypeJsonDeserializer.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eventmesh.connector.jdbc.table.catalog;
+
+import org.apache.eventmesh.connector.jdbc.table.type.EventMeshDataType;
+import org.apache.eventmesh.connector.jdbc.table.type.EventMeshTypeNameConverter;
+import org.apache.eventmesh.connector.jdbc.type.eventmesh.NullEventMeshDataType;
+
+import org.apache.commons.lang3.StringUtils;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import com.fasterxml.jackson.core.JacksonException;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonDeserializer;
+import com.fasterxml.jackson.databind.JsonNode;
+
+public class EventMeshDataTypeJsonDeserializer extends JsonDeserializer {
+
+ @Override
+ public EventMeshDataType deserialize(JsonParser jsonParser, DeserializationContext deserializationContext)
+ throws IOException, JacksonException {
+ JsonNode treeNode = jsonParser.readValueAsTree();
+ Iterator> fields = treeNode.fields();
+ while (fields.hasNext()) {
+ Map.Entry field = fields.next();
+ if (StringUtils.equals("eventMeshDataType", field.getKey())) {
+ String value = field.getValue().asText();
+ return EventMeshTypeNameConverter.ofEventMeshDataType(value);
+ }
+ }
+ return NullEventMeshDataType.INSTANCE;
+ }
+}
\ No newline at end of file
diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/utils/HttpExchangeUtils.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/EventMeshDataTypeJsonSerializer.java
similarity index 56%
rename from eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/utils/HttpExchangeUtils.java
rename to eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/EventMeshDataTypeJsonSerializer.java
index 67479c8c2f..68fd9a6954 100644
--- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/utils/HttpExchangeUtils.java
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/EventMeshDataTypeJsonSerializer.java
@@ -15,27 +15,22 @@
* limitations under the License.
*/
-package org.apache.eventmesh.runtime.admin.utils;
+package org.apache.eventmesh.connector.jdbc.table.catalog;
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.nio.charset.StandardCharsets;
-
-public class HttpExchangeUtils {
+import org.apache.eventmesh.connector.jdbc.table.type.EventMeshDataType;
- public static String streamToString(InputStream stream) throws IOException {
- try (BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) {
+import java.io.IOException;
- StringBuilder buffer = new StringBuilder();
- int b;
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.databind.JsonSerializer;
+import com.fasterxml.jackson.databind.SerializerProvider;
- while ((b = bufferedReader.read()) != -1) {
- buffer.append((char) b);
- }
+public class EventMeshDataTypeJsonSerializer extends JsonSerializer {
- return buffer.toString();
- }
+ @Override
+ public void serialize(EventMeshDataType value, JsonGenerator gen, SerializerProvider serializers) throws IOException {
+ gen.writeStartObject();
+ gen.writeStringField("eventMeshDataType", value.getName());
+ gen.writeEndObject();
}
}
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/Index.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/Index.java
new file mode 100644
index 0000000000..751d26cb13
--- /dev/null
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/Index.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eventmesh.connector.jdbc.table.catalog;
+
+import java.io.Serializable;
+import java.util.List;
+
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+
+/**
+ * This class represents an Index object with attributes such as name, column names, index type, index method, and comment.
+ * It provides a Builder pattern for creating Index objects.
+ */
+@Getter
+@Setter
+@NoArgsConstructor
+public class Index implements Serializable {
+
+ // Name of the index
+ private String name;
+
+ // List of column names included in the index
+ private List columnNames;
+
+ // Type of the index (e.g., unique, normal, etc.)
+ private String indexType;
+
+ // Method used for the index (e.g., B-tree, Hash, etc.)
+ private String indexMethod;
+
+ // Comment associated with the index
+ private String comment;
+
+ protected Index(Builder builder) {
+ this.name = builder.name;
+ this.columnNames = builder.columnNames;
+ this.indexType = builder.indexType;
+ this.indexMethod = builder.indexMethod;
+ this.comment = builder.comment;
+ }
+
+ public Index(List columnNames) {
+ this.columnNames = columnNames;
+ }
+
+ public Index(String name, List columnNames) {
+ this.name = name;
+ this.columnNames = columnNames;
+ }
+
+ public Index(String name, List columnNames, String comment) {
+ this.name = name;
+ this.columnNames = columnNames;
+ this.comment = comment;
+ }
+
+ public void addColumnNames(String... columnNames) {
+ if (columnNames != null && columnNames.length > 0) {
+ for (String columnName : columnNames) {
+ this.columnNames.add(columnName);
+ }
+ }
+ }
+
+ public Index(String name, List columnNames, String indexType, String indexMethod, String comment) {
+ this.name = name;
+ this.columnNames = columnNames;
+ this.indexType = indexType;
+ this.indexMethod = indexMethod;
+ this.comment = comment;
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ public static class Builder {
+
+ protected String name;
+ protected List columnNames;
+ protected String indexType;
+ protected String indexMethod;
+ protected String comment;
+
+ public static Builder newIndex() {
+ return new Builder();
+ }
+
+ public Builder withName(String name) {
+ this.name = name;
+ return this;
+ }
+
+ public Builder withColumnNames(List columnNames) {
+ this.columnNames = columnNames;
+ return this;
+ }
+
+ public Builder withIndexType(String indexType) {
+ this.indexType = indexType;
+ return this;
+ }
+
+ public Builder withIndexMethod(String indexMethod) {
+ this.indexMethod = indexMethod;
+ return this;
+ }
+
+ public Builder withComment(String comment) {
+ this.comment = comment;
+ return this;
+ }
+
+ public Index build() {
+ return new Index(this);
+ }
+ }
+}
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/PrimaryKey.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/PrimaryKey.java
index 2095b30429..48fa27d936 100644
--- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/PrimaryKey.java
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/PrimaryKey.java
@@ -22,6 +22,7 @@
public class PrimaryKey extends UniqueKey {
public PrimaryKey() {
+
}
public PrimaryKey(String name, List columnNames, String comment) {
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/Table.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/Table.java
index 34a44eedff..158ffc81da 100644
--- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/Table.java
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/Table.java
@@ -19,16 +19,14 @@
import java.util.List;
-import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
-@AllArgsConstructor
@NoArgsConstructor
public class Table {
- private String name;
+ private TableId tableId;
private PrimaryKey primaryKey;
@@ -36,4 +34,72 @@ public class Table {
private String comment;
+ private Options options = new Options();
+
+ public Table(TableId tableId, PrimaryKey primaryKey, List uniqueKeys, String comment) {
+ this.tableId = tableId;
+ this.primaryKey = primaryKey;
+ this.uniqueKeys = uniqueKeys;
+ this.comment = comment;
+ }
+
+ public Table(TableId tableId, PrimaryKey primaryKey, List uniqueKeys, String comment, Options options) {
+ this.tableId = tableId;
+ this.primaryKey = primaryKey;
+ this.uniqueKeys = uniqueKeys;
+ this.comment = comment;
+ if (options != null) {
+ this.options.putAll(options);
+ }
+ }
+
+ public void put(String key, Object value) {
+ options.put(key, value);
+ }
+
+ public void putAll(Options options) {
+ this.options.putAll(options);
+ }
+
+ public static Builder newBuilder() {
+ return new Builder();
+ }
+
+ public static class Builder {
+
+ private TableId tableId;
+ private PrimaryKey primaryKey;
+ private List uniqueKeys;
+ private String comment;
+ private Options options;
+
+ public Builder withTableId(TableId tableId) {
+ this.tableId = tableId;
+ return this;
+ }
+
+ public Builder withPrimaryKey(PrimaryKey primaryKey) {
+ this.primaryKey = primaryKey;
+ return this;
+ }
+
+ public Builder withUniqueKeys(List uniqueKeys) {
+ this.uniqueKeys = uniqueKeys;
+ return this;
+ }
+
+ public Builder withComment(String comment) {
+ this.comment = comment;
+ return this;
+ }
+
+ public Builder withOptions(Options options) {
+ this.options = options;
+ return this;
+ }
+
+ public Table build() {
+ return new Table(tableId, primaryKey, uniqueKeys, comment, options);
+ }
+ }
}
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/TableSchema.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/TableSchema.java
index 3eadd78d56..fa4ed243d8 100644
--- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/TableSchema.java
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/TableSchema.java
@@ -40,14 +40,14 @@ public class TableSchema implements Serializable {
/**
* A map of column names to their respective column objects.
*/
- private Map columnMap;
+ private Map> columnMap;
/**
* A list of columns in the table.
*/
- private List columns;
+ private List> columns;
- private Map orderColumnMap;
+ private Map> orderColumnMap;
/**
* The primary key of the table.
@@ -58,8 +58,8 @@ public class TableSchema implements Serializable {
private String comment;
- public TableSchema(TableId tableId, Map columnMap, List columns,
- Map orderColumnMap, PrimaryKey primaryKey, List uniqueKeys, String comment) {
+ public TableSchema(TableId tableId, Map> columnMap, List> columns,
+ Map> orderColumnMap, PrimaryKey primaryKey, List uniqueKeys, String comment) {
this.tableId = tableId;
this.columnMap = columnMap;
this.columns = columns;
@@ -88,9 +88,9 @@ public static TableSchemaBuilder newTableSchemaBuilder() {
public static class TableSchemaBuilder {
private TableId tableId;
- private Map columnMap;
- private Map orderColumnMap;
- private List columns;
+ private Map> columnMap;
+ private Map> orderColumnMap;
+ private List> columns;
private PrimaryKey primaryKey;
private List uniqueKeys;
private String comment;
@@ -104,12 +104,12 @@ public TableSchemaBuilder withTableId(TableId tableId) {
return this;
}
- public TableSchemaBuilder withColumns(Map columnMap) {
+ public TableSchemaBuilder withColumns(Map> columnMap) {
this.columnMap = columnMap;
return this;
}
- public TableSchemaBuilder withColumns(List columns) {
+ public TableSchemaBuilder withColumns(List> columns) {
this.columns = columns;
return this;
}
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/UniqueKey.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/UniqueKey.java
index f44ed561f4..4677d43c62 100644
--- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/UniqueKey.java
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/UniqueKey.java
@@ -17,7 +17,6 @@
package org.apache.eventmesh.connector.jdbc.table.catalog;
-import java.io.Serializable;
import java.util.List;
import lombok.Getter;
@@ -29,44 +28,26 @@
*/
@Setter
@Getter
-public class UniqueKey implements Serializable {
+public class UniqueKey extends Index {
- // The name of the unique key, if specified.
- private String name;
-
- // The list of column names that make up the unique/primary key.
- private List columnNames;
-
- // An optional comment or description for the unique/primary key.
- private String comment;
+ private static final String INDEX_TYPE = "UNIQUE";
public UniqueKey() {
}
public UniqueKey(String name, List columnNames, String comment) {
- this.name = name;
- this.columnNames = columnNames;
- this.comment = comment;
+ super(name, columnNames, INDEX_TYPE, null, comment);
}
public UniqueKey(String name, List columnNames) {
- this.name = name;
- this.columnNames = columnNames;
+ super(name, columnNames);
}
public UniqueKey(List columnNames) {
- this.columnNames = columnNames;
+ super(columnNames);
}
public UniqueKey copy() {
- return new UniqueKey(name, columnNames, comment);
- }
-
- public void addColumnNames(String... columnNames) {
- if (columnNames != null && columnNames.length > 0) {
- for (String columnName : columnNames) {
- this.columnNames.add(columnName);
- }
- }
+ return new UniqueKey(getName(), getColumnNames(), getComment());
}
}
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/mysql/MysqlColumn.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/mysql/MysqlColumn.java
index c0440a4d14..424f70f3a6 100644
--- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/mysql/MysqlColumn.java
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/mysql/MysqlColumn.java
@@ -18,36 +18,34 @@
package org.apache.eventmesh.connector.jdbc.table.catalog.mysql;
import org.apache.eventmesh.connector.jdbc.table.catalog.Column;
+import org.apache.eventmesh.connector.jdbc.table.catalog.Options;
import org.apache.eventmesh.connector.jdbc.table.type.EventMeshDataType;
import java.sql.JDBCType;
+import java.util.List;
+
+import lombok.Data;
+import lombok.EqualsAndHashCode;
/**
* Represents a MySQL column in a database table.
*/
+@Data
+@EqualsAndHashCode(callSuper = true)
public class MysqlColumn extends Column {
- private boolean autoIncremented;
-
- private boolean generated;
-
- private String collationName;
-
- public MysqlColumn(String name, EventMeshDataType dataType, JDBCType jdbcType, Integer columnLength, Integer decimal, boolean notNull,
- String comment, Object defaultValue, String defaultValueExpression, boolean autoIncremented, boolean generated, String collationName) {
- super(name, dataType, jdbcType, columnLength, decimal, notNull, comment, defaultValue, defaultValueExpression, 0);
- this.autoIncremented = autoIncremented;
- this.generated = generated;
- this.collationName = collationName;
+ public MysqlColumn(String name, EventMeshDataType dataType, JDBCType jdbcType, Long columnLength, Integer decimal, boolean notNull,
+ String comment, Object defaultValue, String defaultValueExpression, boolean autoIncremented, boolean generated, String collationName,
+ String charsetName, List enumValues, String nativeType, Options options) {
+ super(name, dataType, jdbcType, columnLength, decimal, notNull, comment, defaultValue, defaultValueExpression, 0, charsetName,
+ autoIncremented, generated, collationName, enumValues, nativeType, options);
}
- public MysqlColumn(String name, EventMeshDataType dataType, JDBCType jdbcType, Integer columnLength, Integer decimal, boolean notNull,
+ public MysqlColumn(String name, EventMeshDataType dataType, JDBCType jdbcType, Long columnLength, Integer decimal, boolean notNull,
String comment, Object defaultValue, String defaultValueExpression, boolean autoIncremented, boolean generated, String collationName,
- int order) {
- super(name, dataType, jdbcType, columnLength, decimal, notNull, comment, defaultValue, defaultValueExpression, order);
- this.autoIncremented = autoIncremented;
- this.generated = generated;
- this.collationName = collationName;
+ int order, String charsetName, List enumValues, String nativeType, Options options) {
+ super(name, dataType, jdbcType, columnLength, decimal, notNull, comment, defaultValue, defaultValueExpression, order, charsetName,
+ autoIncremented, generated, collationName, enumValues, nativeType, options);
}
public MysqlColumn() {
@@ -55,17 +53,19 @@ public MysqlColumn() {
}
public static MysqlColumn of(
- String name, EventMeshDataType dataType, JDBCType jdbcType, Integer columnLength, Integer decimal, boolean notNull,
- String comment, Object defaultValue, String defaultValueExpression, boolean autoIncremented, boolean generated, String collationName) {
+ String name, EventMeshDataType dataType, JDBCType jdbcType, Long columnLength, Integer decimal, boolean notNull,
+ String comment, Object defaultValue, String defaultValueExpression, boolean autoIncremented, boolean generated, String collationName,
+ String charsetName, List enumValues, String nativeType, Options options) {
return new MysqlColumn(name, dataType, jdbcType, columnLength, decimal, notNull, comment, defaultValue, defaultValueExpression,
- autoIncremented, generated, collationName);
+ autoIncremented, generated, collationName, charsetName, enumValues, nativeType, options);
}
public static MysqlColumn of(
- String name, EventMeshDataType dataType, JDBCType jdbcType, Integer columnLength, Integer decimal, boolean notNull, String comment,
- Object defaultValue, String defaultValueExpression, boolean autoIncremented, boolean generated, String collationName, int order) {
+ String name, EventMeshDataType dataType, JDBCType jdbcType, Long columnLength, Integer decimal, boolean notNull, String comment,
+ Object defaultValue, String defaultValueExpression, boolean autoIncremented, boolean generated, String collationName, int order,
+ String charsetName, List enumValues, String nativeType, Options options) {
return new MysqlColumn(name, dataType, jdbcType, columnLength, decimal, notNull, comment, defaultValue, defaultValueExpression,
- autoIncremented, generated, collationName, order);
+ autoIncremented, generated, collationName, order, charsetName, enumValues, nativeType, options);
}
/**
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/mysql/MysqlColumnEditor.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/mysql/MysqlColumnEditor.java
index cea5ea9ceb..470e31affd 100644
--- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/mysql/MysqlColumnEditor.java
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/mysql/MysqlColumnEditor.java
@@ -65,6 +65,6 @@ static MysqlColumnEditor ofEditor() {
* @param collationName The name of the collation to set.
* @return The column editor with the collation set.
*/
- MysqlColumnEditor collate(String collationName);
+ MysqlColumnEditor collation(String collationName);
}
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/mysql/MysqlColumnEditorImpl.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/mysql/MysqlColumnEditorImpl.java
index c7b7648af2..9f6ac950d0 100644
--- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/mysql/MysqlColumnEditorImpl.java
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/mysql/MysqlColumnEditorImpl.java
@@ -17,7 +17,12 @@
package org.apache.eventmesh.connector.jdbc.table.catalog.mysql;
+import org.apache.eventmesh.connector.jdbc.source.dialect.mysql.MysqlDataTypeConvertor;
import org.apache.eventmesh.connector.jdbc.table.catalog.AbstractColumnEditorImpl;
+import org.apache.eventmesh.connector.jdbc.table.type.EventMeshDataType;
+
+import java.util.HashMap;
+import java.util.Map;
public class MysqlColumnEditorImpl extends AbstractColumnEditorImpl implements MysqlColumnEditor {
@@ -27,6 +32,8 @@ public class MysqlColumnEditorImpl extends AbstractColumnEditorImpl dataTypeProperties = new HashMap<>();
+ if (ofColumnLength() != null) {
+ dataTypeProperties.put(MysqlDataTypeConvertor.PRECISION, ofColumnLength().intValue());
+ }
+ dataTypeProperties.put(MysqlDataTypeConvertor.SCALE, ofScale());
+ EventMeshDataType eventMeshType = convertor.toEventMeshType(ofJdbcType(), dataTypeProperties);
+ withEventMeshType(eventMeshType);
+
return MysqlColumn.of(ofName(), ofEventMeshDataType(), ofJdbcType(), ofColumnLength(), ofScale(), isNotNull(), ofComment(), ofDefaultValue(),
- ofDefaultValueExpression(), autoIncremented, generated, collationName, ofOrder());
+ ofDefaultValueExpression(), autoIncremented, generated, collationName, ofOrder(), ofCharsetName(), ofEnumValues(), ofTypeName(),
+ ofOptions());
}
}
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/mysql/MysqlDefaultValueConvertorImpl.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/mysql/MysqlDefaultValueConvertorImpl.java
new file mode 100644
index 0000000000..1832f17d77
--- /dev/null
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/mysql/MysqlDefaultValueConvertorImpl.java
@@ -0,0 +1,218 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eventmesh.connector.jdbc.table.catalog.mysql;
+
+import org.apache.eventmesh.common.Constants;
+import org.apache.eventmesh.connector.jdbc.table.catalog.Column;
+import org.apache.eventmesh.connector.jdbc.table.catalog.DefaultValueConvertor;
+import org.apache.eventmesh.connector.jdbc.utils.ByteArrayUtils;
+
+import org.apache.commons.lang3.StringUtils;
+
+import java.math.BigDecimal;
+import java.math.RoundingMode;
+import java.sql.JDBCType;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.LocalTime;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeFormatterBuilder;
+import java.time.temporal.ChronoField;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Optional;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class MysqlDefaultValueConvertorImpl implements DefaultValueConvertor {
+
+ private static final String EPOCH_DATE = "1970-01-01";
+
+ // Time The range is '-838:59:59.000000' to '838:59:59.000000'
+ private static final Pattern TIME_PATTERN = Pattern.compile("(\\-?[0-9]*):([0-9]*)(:([0-9]*))?(\\.([0-9]*))?");
+
+ private static final Set NUMBER_DATA_TYPES = Collections.unmodifiableSet(new HashSet<>(
+ Arrays.asList(JDBCType.TINYINT, JDBCType.INTEGER, JDBCType.DATE, JDBCType.TIMESTAMP, JDBCType.TIMESTAMP_WITH_TIMEZONE, JDBCType.TIME,
+ JDBCType.BOOLEAN, JDBCType.BIT, JDBCType.NUMERIC, JDBCType.DECIMAL, JDBCType.FLOAT, JDBCType.DOUBLE, JDBCType.REAL)));
+
+ private static final Set BINARY_DATA_TYPES = Collections.unmodifiableSet(new HashSet<>(
+ Arrays.asList(JDBCType.BINARY, JDBCType.VARBINARY)));
+
+ @Override
+ public Object parseDefaultValue(Column column, String defaultValueExpression) {
+ if (defaultValueExpression == null) {
+ return null;
+ }
+ defaultValueExpression = defaultValueExpression.trim();
+
+ if (NUMBER_DATA_TYPES.contains(column.getJdbcType()) && StringUtils.equalsAnyIgnoreCase(defaultValueExpression, Boolean.TRUE.toString(),
+ Boolean.FALSE.toString())) {
+ /*
+ * These types are synonyms for DECIMAL: DEC[(M[,D])] [UNSIGNED] [ZEROFILL], NUMERIC[(M[,D])] [UNSIGNED] [ZEROFILL], FIXED[(M[,D])]
+ * [UNSIGNED] [ZEROFILL]
+ */
+ if (column.getJdbcType() == JDBCType.DECIMAL || column.getJdbcType() == JDBCType.NUMERIC) {
+ return convert2Decimal(column, defaultValueExpression);
+ }
+ return StringUtils.equalsIgnoreCase(Boolean.TRUE.toString(), defaultValueExpression) ? 1 : 0;
+ }
+
+ if (BINARY_DATA_TYPES.contains(column.getJdbcType()) && column.getDefaultValueExpression() != null) {
+ // https://dev.mysql.com/doc/refman/8.0/en/binary-varbinary.html
+ String cleanedDefaultValueExpression = StringUtils.replace(column.getDefaultValueExpression(), "\\0", "");
+ return ByteArrayUtils.bytesToHexString(cleanedDefaultValueExpression.getBytes(Constants.DEFAULT_CHARSET));
+ }
+
+ switch (column.getDataType().getSQLType()) {
+ case DATE:
+ return convert2LocalDate(column, defaultValueExpression);
+ case TIMESTAMP:
+ return convertToLocalDateTime(column, defaultValueExpression);
+ case TIMESTAMP_WITH_TIMEZONE:
+ return convertToTimestamp(column, defaultValueExpression);
+ case TIME:
+ return convertToLocalTime(column, defaultValueExpression);
+ case BOOLEAN:
+ return convert2Boolean(column, defaultValueExpression);
+ case BIT:
+ return convertToBits(column, defaultValueExpression);
+
+ case NUMERIC:
+ case DECIMAL:
+ return convert2Decimal(column, defaultValueExpression);
+
+ case FLOAT:
+ case DOUBLE:
+ case REAL:
+ return Double.parseDouble(defaultValueExpression);
+ default:
+ }
+ return defaultValueExpression;
+ }
+
+ private Object convert2Boolean(Column column, String value) {
+ // value maybe is numeric or string
+ if (StringUtils.isNumeric(value)) {
+ return Integer.parseInt(value) != 0;
+ }
+ return Boolean.parseBoolean(value);
+ }
+
+ private Object convert2Decimal(Column column, String value) {
+ return Optional.ofNullable(column.getDecimal()).isPresent() ? new BigDecimal(value).setScale(column.getDecimal(), RoundingMode.HALF_UP)
+ : new BigDecimal(value);
+ }
+
+ private Object convertToBits(Column column, String value) {
+ // value: '101010111'
+ if (column.getColumnLength() > 1) {
+ int nums = value.length() / Byte.SIZE + (value.length() % Byte.SIZE == 0 ? 0 : 1);
+ byte[] bytes = new byte[nums];
+ int length = value.length();
+ for (int i = 0; i < nums; i++) {
+ int size = value.length() - Byte.SIZE < 0 ? 0 : value.length() - Byte.SIZE;
+ bytes[nums - i - 1] = (byte) Integer.parseInt(value.substring(size, length), 2);
+ value = value.substring(0, size);
+ }
+ return bytes;
+ }
+
+ // value: '1' or '0' parse to boolean
+ return Short.parseShort(value) != 0;
+ }
+
+ private Object convertToLocalTime(Column column, String value) {
+
+ Matcher matcher = TIME_PATTERN.matcher(value);
+ if (!matcher.matches()) {
+ throw new IllegalArgumentException("Unexpected format for TIME column: " + value);
+ }
+
+ final int hours = Integer.parseInt(matcher.group(1));
+ final int minutes = Integer.parseInt(matcher.group(2));
+ final String secondsGroup = matcher.group(4);
+ int seconds = 0;
+ int nanoSeconds = 0;
+
+ if (secondsGroup != null) {
+ seconds = Integer.parseInt(secondsGroup);
+ String microSecondsString = matcher.group(6);
+ if (microSecondsString != null) {
+ nanoSeconds = Integer.parseInt(microSecondsString) * 1000;
+ }
+ }
+ return LocalTime.of(hours, minutes, seconds, nanoSeconds);
+ }
+
+ private Object convertToTimestamp(Column column, String value) {
+ // Mysql not support
+ return null;
+ }
+
+ private Object convertToLocalDateTime(Column column, String value) {
+ if (StringUtils.containsAny(value, "CURRENT_TIMESTAMP", "current_timestamp")) {
+ return value;
+ }
+ // The TIMESTAMP data type is used for values that contain both date and time parts.
+ // TIMESTAMP has a range of '1970-01-01 00:00:01' UTC to '2038-01-19 03:14:07' UTC.
+ return LocalDateTime.from(timestampFormat(Optional.ofNullable(column.getColumnLength()).orElse(0L).intValue()).parse(value));
+ }
+
+ private Object convert2LocalDate(Column column, String value) {
+ // The DATE type is used for values with a date part but no time part.
+ // MySQL retrieves and displays DATE values in 'YYYY-MM-DD' format.
+ // The supported range is '1000-01-01' to '9999-12-31'.
+
+ try {
+ if (StringUtils.contains(value, "-")) {
+ return LocalDate.parse(value);
+ }
+ // maybe is year, e.g. 2020
+ if (StringUtils.isNumeric(value)) {
+ return LocalDate.parse(value + "-01-01");
+ }
+ // format: 20200101
+ return LocalDate.from(dateFormat().parse(value));
+ } catch (Exception e) {
+ log.warn("Convert date error[value={}]", value);
+ return LocalDate.parse(EPOCH_DATE);
+ }
+ }
+
+ private DateTimeFormatter timestampFormat(int length) {
+ final DateTimeFormatterBuilder dtf = new DateTimeFormatterBuilder().appendPattern("yyyy-MM-dd").optionalStart().appendLiteral(" ")
+ .append(DateTimeFormatter.ISO_LOCAL_TIME).optionalEnd().parseDefaulting(ChronoField.HOUR_OF_DAY, 0)
+ .parseDefaulting(ChronoField.MINUTE_OF_HOUR, 0).parseDefaulting(ChronoField.SECOND_OF_MINUTE, 0);
+ if (length > 0) {
+ dtf.appendFraction(ChronoField.MICRO_OF_SECOND, 0, length, true);
+ }
+ return dtf.toFormatter();
+ }
+
+ private DateTimeFormatter dateFormat() {
+ final DateTimeFormatterBuilder dtf = new DateTimeFormatterBuilder().appendValue(ChronoField.YEAR, 4).appendValue(ChronoField.MONTH_OF_YEAR, 2)
+ .optionalStart().appendValue(ChronoField.DAY_OF_YEAR, 2);
+ return dtf.toFormatter();
+ }
+
+}
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/mysql/MysqlOptions.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/mysql/MysqlOptions.java
new file mode 100644
index 0000000000..dc82e6ee51
--- /dev/null
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/mysql/MysqlOptions.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eventmesh.connector.jdbc.table.catalog.mysql;
+
+/**
+ * The MysqlOptions class provides the constants for configuring options in MySQL tables and columns.
+ */
+public class MysqlOptions {
+
+ public static final class MysqlTableOptions {
+
+ public static String ENGINE = "ENGINE";
+
+ public static String AUTO_INCREMENT = "AUTO_INCREMENT";
+
+ public static String CHARSET = "CHARSET";
+
+ public static String COLLATE = "COLLATE";
+
+ public static String COMMENT = "COMMENT";
+ }
+
+ public static final class MysqlColumnOptions {
+
+ public static String SIGNED = "SIGNED";
+
+ public static String UNSIGNED = "UNSIGNED";
+
+ public static String ZEROFILL = "ZEROFILL";
+ }
+
+}
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/mysql/MysqlTableSchema.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/mysql/MysqlTableSchema.java
index c26ca5c3ce..4cddc03a65 100644
--- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/mysql/MysqlTableSchema.java
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/mysql/MysqlTableSchema.java
@@ -26,8 +26,11 @@
import java.util.List;
import java.util.Map;
+import lombok.Getter;
+
public class MysqlTableSchema extends TableSchema {
+ @Getter
private Options tableOptions;
public MysqlTableSchema(TableId tableId, Map columnMap, List columns, Map orderColumnMap,
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/CalendarType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/CalendarType.java
deleted file mode 100644
index 788b85ecd7..0000000000
--- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/CalendarType.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.eventmesh.connector.jdbc.table.type;
-
-import java.time.LocalDate;
-import java.time.LocalDateTime;
-import java.time.LocalTime;
-import java.time.temporal.Temporal;
-import java.util.Objects;
-
-public class CalendarType implements EventMeshDataType {
-
- // Constants for LocalDate, LocalTime, and LocalDateTime types
- public static final CalendarType LOCAL_DATE_TYPE = new CalendarType<>(LocalDate.class, SQLType.DATE);
-
- public static final CalendarType LOCAL_TIME_TYPE = new CalendarType<>(LocalTime.class, SQLType.TIME);
-
- public static final CalendarType LOCAL_DATE_TIME_TYPE = new CalendarType<>(LocalDateTime.class, SQLType.TIMESTAMP);
-
- private final Class typeClass;
- private final SQLType sqlType;
-
- private CalendarType(Class typeClass, SQLType sqlType) {
- this.typeClass = typeClass;
- this.sqlType = sqlType;
- }
-
- /**
- * Returns the type class of the data.
- *
- * @return the type class of the data.
- */
- @Override
- public Class getTypeClass() {
- return typeClass;
- }
-
- /**
- * Returns the SQL type of the data.
- *
- * @return the SQL type of the data.
- */
- @Override
- public SQLType getSQLType() {
- return sqlType;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (!(o instanceof CalendarType)) {
- return false;
- }
- CalendarType that = (CalendarType) o;
- return Objects.equals(getTypeClass(), that.getTypeClass()) && sqlType == that.sqlType;
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(getTypeClass(), sqlType);
- }
-
- @Override
- public String toString() {
- return typeClass.getName();
- }
-}
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/EventMeshDataType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/EventMeshDataType.java
index 07da72bea7..110ef499df 100644
--- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/EventMeshDataType.java
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/EventMeshDataType.java
@@ -17,22 +17,26 @@
package org.apache.eventmesh.connector.jdbc.table.type;
+import org.apache.eventmesh.connector.jdbc.type.Type;
+
/**
- * Defines Event Mesh data type with methods to get the type class and SQL type of the data.
+ * An interface representing a data type used in an EventMesh.
+ *
+ * @param The type of the data.
*/
-public interface EventMeshDataType {
+public interface EventMeshDataType extends Type {
/**
- * Gets the type class of the data.
+ * Gets the class representing the type of the data.
*
- * @return the type class of the data.
+ * @return The class representing the type of the data.
*/
Class getTypeClass();
/**
* Gets the SQL type of the data.
*
- * @return the SQL type of the data.
+ * @return The SQL type of the data.
*/
SQLType getSQLType();
@@ -41,7 +45,5 @@ public interface EventMeshDataType {
*
* @return The name of the data type.
*/
- default String getName() {
- return EventMeshTypeNameConverter.ofName(this);
- }
+ String getName();
}
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/EventMeshTypeNameConverter.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/EventMeshTypeNameConverter.java
index 181e735a91..f78386d37e 100644
--- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/EventMeshTypeNameConverter.java
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/EventMeshTypeNameConverter.java
@@ -17,45 +17,45 @@
package org.apache.eventmesh.connector.jdbc.table.type;
+import org.apache.eventmesh.connector.jdbc.type.eventmesh.BooleanEventMeshDataType;
+import org.apache.eventmesh.connector.jdbc.type.eventmesh.BytesEventMeshDataType;
+import org.apache.eventmesh.connector.jdbc.type.eventmesh.DateEventMeshDataType;
+import org.apache.eventmesh.connector.jdbc.type.eventmesh.DateTimeEventMeshDataType;
+import org.apache.eventmesh.connector.jdbc.type.eventmesh.DecimalEventMeshDataType;
+import org.apache.eventmesh.connector.jdbc.type.eventmesh.Float32EventMeshDataType;
+import org.apache.eventmesh.connector.jdbc.type.eventmesh.Float64EventMeshDataType;
+import org.apache.eventmesh.connector.jdbc.type.eventmesh.Int16EventMeshDataType;
+import org.apache.eventmesh.connector.jdbc.type.eventmesh.Int32EventMeshDataType;
+import org.apache.eventmesh.connector.jdbc.type.eventmesh.Int64EventMeshDataType;
+import org.apache.eventmesh.connector.jdbc.type.eventmesh.Int8EventMeshDataType;
+import org.apache.eventmesh.connector.jdbc.type.eventmesh.StringEventMeshDataType;
+import org.apache.eventmesh.connector.jdbc.type.eventmesh.TimeEventMeshDataType;
+
import java.util.HashMap;
import java.util.Map;
public final class EventMeshTypeNameConverter {
- private static Map, String> PRIMITIVE_TYPE_MAP = new HashMap<>(32);
+ private static Map PRIMITIVE_TYPE_MAP = new HashMap<>(32);
static {
- PRIMITIVE_TYPE_MAP.put(PrimitiveType.STRING_TYPE, "string");
- PRIMITIVE_TYPE_MAP.put(PrimitiveType.BOOLEAN_TYPE, "bool");
- PRIMITIVE_TYPE_MAP.put(PrimitiveType.BYTE_TYPE, "byte");
- PRIMITIVE_TYPE_MAP.put(PrimitiveType.SHORT_TYPE, "short");
- PRIMITIVE_TYPE_MAP.put(PrimitiveType.INT_TYPE, "int");
- PRIMITIVE_TYPE_MAP.put(PrimitiveType.LONG_TYPE, "long");
- PRIMITIVE_TYPE_MAP.put(PrimitiveType.FLOAT_TYPE, "float");
- PRIMITIVE_TYPE_MAP.put(PrimitiveType.DOUBLE_TYPE, "double");
- PRIMITIVE_TYPE_MAP.put(PrimitiveType.VOID_TYPE, "void");
- PRIMITIVE_TYPE_MAP.put(CalendarType.LOCAL_DATE_TYPE, "LocalDate");
- PRIMITIVE_TYPE_MAP.put(CalendarType.LOCAL_TIME_TYPE, "LocalTime");
- PRIMITIVE_TYPE_MAP.put(CalendarType.LOCAL_DATE_TIME_TYPE, "LocalDateTime");
- PRIMITIVE_TYPE_MAP.put(PrimitiveByteArrayType.BYTES_TYPE, "bytes");
-
- PRIMITIVE_TYPE_MAP.put(PrimitiveArrayType.STRING_ARRAY_TYPE, "string-array");
- PRIMITIVE_TYPE_MAP.put(PrimitiveArrayType.BOOLEAN_ARRAY_TYPE, "bool-array");
- PRIMITIVE_TYPE_MAP.put(PrimitiveArrayType.BYTE_ARRAY_TYPE, "byte-array");
- PRIMITIVE_TYPE_MAP.put(PrimitiveArrayType.SHORT_ARRAY_TYPE, "short-array");
- PRIMITIVE_TYPE_MAP.put(PrimitiveArrayType.INT_ARRAY_TYPE, "int-array");
- PRIMITIVE_TYPE_MAP.put(PrimitiveArrayType.LONG_ARRAY_TYPE, "long-array");
- PRIMITIVE_TYPE_MAP.put(PrimitiveArrayType.FLOAT_ARRAY_TYPE, "float-array");
- PRIMITIVE_TYPE_MAP.put(PrimitiveArrayType.DOUBLE_ARRAY_TYPE, "double-array");
+ PRIMITIVE_TYPE_MAP.put(BooleanEventMeshDataType.INSTANCE.getName(), BooleanEventMeshDataType.INSTANCE);
+ PRIMITIVE_TYPE_MAP.put(Float32EventMeshDataType.INSTANCE.getName(), Float32EventMeshDataType.INSTANCE);
+ PRIMITIVE_TYPE_MAP.put(Float64EventMeshDataType.INSTANCE.getName(), Float64EventMeshDataType.INSTANCE);
+ PRIMITIVE_TYPE_MAP.put(Int8EventMeshDataType.INSTANCE.getName(), Int8EventMeshDataType.INSTANCE);
+ PRIMITIVE_TYPE_MAP.put(Int16EventMeshDataType.INSTANCE.getName(), Int16EventMeshDataType.INSTANCE);
+ PRIMITIVE_TYPE_MAP.put(Int32EventMeshDataType.INSTANCE.getName(), Int32EventMeshDataType.INSTANCE);
+ PRIMITIVE_TYPE_MAP.put(Int64EventMeshDataType.INSTANCE.getName(), Int64EventMeshDataType.INSTANCE);
+ PRIMITIVE_TYPE_MAP.put(StringEventMeshDataType.INSTANCE.getName(), StringEventMeshDataType.INSTANCE);
+ PRIMITIVE_TYPE_MAP.put(BytesEventMeshDataType.INSTANCE.getName(), BytesEventMeshDataType.INSTANCE);
+ PRIMITIVE_TYPE_MAP.put(DateEventMeshDataType.INSTANCE.getName(), DateEventMeshDataType.INSTANCE);
+ PRIMITIVE_TYPE_MAP.put(TimeEventMeshDataType.INSTANCE.getName(), TimeEventMeshDataType.INSTANCE);
+ PRIMITIVE_TYPE_MAP.put(DateTimeEventMeshDataType.INSTANCE.getName(), DateTimeEventMeshDataType.INSTANCE);
+ PRIMITIVE_TYPE_MAP.put(DecimalEventMeshDataType.INSTANCE.getName(), DecimalEventMeshDataType.INSTANCE);
}
- public static String ofName(EventMeshDataType type) {
- String typeName = PRIMITIVE_TYPE_MAP.get(type);
- if (typeName == null && (type instanceof DecimalType)) {
- DecimalType decimalType = (DecimalType) type;
- return String.format("decimal(%s,%s)", decimalType.getScale(), decimalType.getPrecision());
- }
- return typeName;
+ public static EventMeshDataType ofEventMeshDataType(String dataType) {
+ return PRIMITIVE_TYPE_MAP.get(dataType);
}
}
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/MapType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/MapType.java
index 5e33d818bf..f8a5fb530f 100644
--- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/MapType.java
+++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/MapType.java
@@ -19,10 +19,9 @@
import java.util.Arrays;
import java.util.List;
-import java.util.Map;
import java.util.Objects;
-public class MapType implements EventMeshDataType> {
+public class MapType {
private static final List SUPPORTED_KEY_TYPES =
Arrays.asList(
@@ -40,11 +39,11 @@ public class MapType implements EventMeshDataType> {
SQLType.STRING,
SQLType.DECIMAL);
- private final EventMeshDataType keyType;
+ private final EventMeshDataType keyType;
- private final EventMeshDataType valueType;
+ private final EventMeshDataType valueType;
- public MapType(EventMeshDataType keyType, EventMeshDataType valueType) {
+ public MapType(EventMeshDataType keyType, EventMeshDataType valueType) {
Objects.requireNonNull(keyType, "The key type is required.");
Objects.requireNonNull(valueType, "The value type is required.");
@@ -56,27 +55,6 @@ public MapType(EventMeshDataType keyType, EventMeshDataType valueType) {
this.valueType = valueType;
}
- /**
- * Returns the type class of the data.
- *
- * @return the type class of the data.
- */
- @SuppressWarnings("unchecked")
- @Override
- public Class> getTypeClass() {
- return (Class>) (Class) Map.class;
- }
-
- /**
- * Returns the SQL type of the data.
- *
- * @return the SQL type of the data.
- */
- @Override
- public SQLType getSQLType() {
- return SQLType.MAP;
- }
-
@Override
public boolean equals(Object o) {
if (this == o) {
@@ -94,11 +72,11 @@ public int hashCode() {
return Objects.hash(keyType, valueType);
}
- public EventMeshDataType keyType() {
+ public EventMeshDataType keyType() {
return this.keyType;
}
- public EventMeshDataType valueType() {
+ public EventMeshDataType valueType() {
return this.valueType;
}
}
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/PrimitiveArrayType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/PrimitiveArrayType.java
deleted file mode 100644
index d7f9abeb00..0000000000
--- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/PrimitiveArrayType.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.eventmesh.connector.jdbc.table.type;
-
-import java.util.Objects;
-
-public class PrimitiveArrayType implements EventMeshDataType {
-
- @SuppressWarnings("unchecked")
- public static final PrimitiveArrayType STRING_ARRAY_TYPE = new PrimitiveArrayType(String[].class, PrimitiveType.STRING_TYPE);
-
- @SuppressWarnings("unchecked")
- public static final PrimitiveArrayType BOOLEAN_ARRAY_TYPE = new PrimitiveArrayType(Boolean[].class,
- PrimitiveType.BOOLEAN_TYPE);
-
- @SuppressWarnings("unchecked")
- public static final PrimitiveArrayType BYTE_ARRAY_TYPE = new PrimitiveArrayType(Byte[].class, PrimitiveType.BYTE_TYPE);
-
- @SuppressWarnings("unchecked")
- public static final PrimitiveArrayType SHORT_ARRAY_TYPE = new PrimitiveArrayType(Short[].class, PrimitiveType.SHORT_TYPE);
-
- @SuppressWarnings("unchecked")
- public static final PrimitiveArrayType INT_ARRAY_TYPE = new PrimitiveArrayType(Integer[].class, PrimitiveType.INT_TYPE);
-
- @SuppressWarnings("unchecked")
- public static final PrimitiveArrayType LONG_ARRAY_TYPE = new PrimitiveArrayType(Long[].class, PrimitiveType.LONG_TYPE);
-
- @SuppressWarnings("unchecked")
- public static final PrimitiveArrayType FLOAT_ARRAY_TYPE = new PrimitiveArrayType(Float[].class, PrimitiveType.FLOAT_TYPE);
-
- @SuppressWarnings("unchecked")
- public static final PrimitiveArrayType DOUBLE_ARRAY_TYPE = new PrimitiveArrayType(Double[].class, PrimitiveType.DOUBLE_TYPE);
-
- private final Class typeClass;
-
- private final PrimitiveType sqlType;
-
- private PrimitiveArrayType(Class arrayClass, PrimitiveType elementType) {
- this.typeClass = arrayClass;
- this.sqlType = elementType;
- }
-
- /**
- * Returns the type class of the data.
- *
- * @return the type class of the data.
- */
- @Override
- public Class getTypeClass() {
- return typeClass;
- }
-
- /**
- * Returns the SQL type of the data.
- *
- * @return the SQL type of the data.
- */
- @Override
- public SQLType getSQLType() {
- return SQLType.ARRAY;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (!(o instanceof PrimitiveArrayType)) {
- return false;
- }
- PrimitiveArrayType that = (PrimitiveArrayType) o;
- return Objects.equals(getTypeClass(), that.getTypeClass()) && Objects.equals(sqlType, that.sqlType);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(getTypeClass(), sqlType);
- }
-
- @Override
- public String toString() {
- return typeClass.getName();
- }
-}
diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/PrimitiveType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/PrimitiveType.java
deleted file mode 100644
index d6e8bdd514..0000000000
--- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/PrimitiveType.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.eventmesh.connector.jdbc.table.type;
-
-import java.util.Objects;
-
-public class PrimitiveType implements EventMeshDataType