Skip to content

Commit

Permalink
[ISSUE apache#5039]support full sync of mysql (apache#5038)
Browse files Browse the repository at this point in the history
* use conf instead of resource

* prepare for full and check

* more and more

* mysql type for read

* mysql type for read

* close to finish full read and begin full write

* do full write

* prepare for full write

* close to finish full read and begin full write

* close to finish full read and begin full write

* close to finish full read and begin full write

* fix commit execute

* fix checkstyle and license

* fix checkstyle and license

* fix checkstyle and license
  • Loading branch information
sodaRyCN authored Jul 23, 2024
1 parent 56f893c commit 3b770f8
Show file tree
Hide file tree
Showing 50 changed files with 3,035 additions and 142 deletions.
47 changes: 45 additions & 2 deletions build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -210,6 +210,48 @@ tasks.register('dist') {
}
}

tasks.register('dist-admin') {
subprojects.forEach { subProject ->
dependsOn("${subProject.path}:jar")
}
def includedProjects =
[
"eventmesh-admin-server",
"eventmesh-common",
"eventmesh-spi",
"eventmesh-registry:eventmesh-registry-api",
"eventmesh-registry:eventmesh-registry-nacos",
"eventmesh-openconnect:eventmesh-openconnect-offsetmgmt-plugin:eventmesh-openconnect-offsetmgmt-api"
]
doLast {
includedProjects.each {
def subProject = findProject(it)
copy {
from subProject.jar.archivePath
into rootProject.file('dist/apps')
}
copy {
from subProject.configurations.runtimeClasspath
into rootProject.file('dist/lib')
exclude 'eventmesh-*'
}
copy {
from subProject.file('bin')
into rootProject.file('dist/bin')
}
copy {
from subProject.file('conf')
from subProject.sourceSets.main.resources.srcDirs
into rootProject.file('dist/conf')
duplicatesStrategy = DuplicatesStrategy.EXCLUDE
exclude 'META-INF'
}

}
}

}

tasks.register('installPlugin') {
var pluginProjects = subprojects.findAll {
it.file('gradle.properties').exists()
Expand Down Expand Up @@ -754,11 +796,12 @@ subprojects {
dependency "software.amazon.awssdk:s3:2.26.3"
dependency "com.github.rholder:guava-retrying:2.0.0"

dependency "org.mybatis.spring.boot:mybatis-spring-boot-starter:2.3.2"
dependency "com.alibaba:druid-spring-boot-starter:1.2.23"
dependency "com.baomidou:mybatis-plus-boot-starter:3.5.5"
dependency "org.springframework.boot:spring-boot-starter-jetty:2.7.18"
dependency "com.baomidou:mybatis-plus:3.5.7"
dependency "com.mysql:mysql-connector-j:8.4.0"
dependency "org.springframework.boot:spring-boot-starter-jetty:2.7.10"
dependency "org.locationtech.jts:jts-core:1.19.0"
}
}
}
23 changes: 15 additions & 8 deletions eventmesh-admin-server/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -31,18 +31,25 @@ dependencies {
implementation "io.grpc:grpc-stub"
implementation "io.grpc:grpc-netty-shaded"

// https://mvnrepository.com/artifact/com.baomidou/mybatis-plus-boot-starter
implementation 'com.baomidou:mybatis-plus-boot-starter:3.5.7'
implementation "org.reflections:reflections:0.10.2"
// https://mvnrepository.com/artifact/com.baomidou/mybatis-plus-boot-starter
implementation "com.baomidou:mybatis-plus-boot-starter"

// https://mvnrepository.com/artifact/com.alibaba/druid-spring-boot-starter
implementation "com.alibaba:druid-spring-boot-starter"
compileOnly 'com.mysql:mysql-connector-j'
compileOnly 'org.projectlombok:lombok'
annotationProcessor 'org.projectlombok:lombok'
// https://mvnrepository.com/artifact/com.alibaba/druid-spring-boot-starter
implementation "com.alibaba:druid-spring-boot-starter"
implementation 'com.mysql:mysql-connector-j'
compileOnly 'org.projectlombok:lombok'
annotationProcessor 'org.projectlombok:lombok'
}

configurations.implementation {
exclude group: "org.springframework.boot", module: "spring-boot-starter-logging"
}

sourceSets {
main {
resources {
srcDirs = ['src/main/resources', 'conf']
}
}
}

File renamed without changes.
108 changes: 108 additions & 0 deletions eventmesh-admin-server/conf/log4j2.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->

<Configuration status="debug" monitorInterval="60">
<Appenders>
<Console name="console" target="SYSTEM_OUT">
<PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p [%t] %c{1}(%F:%L) - %m%n"/>
</Console>
</Appenders>

<Loggers>
<AsyncRoot level="debug" includeLocation="true">
<AppenderRef ref="console"/>
</AsyncRoot>

<AsyncLogger name="retry" level="info" additivity="false" includeLocation="true">
<AppenderRef ref="retryAppender"/>
</AsyncLogger>

<AsyncLogger name="httpMonitor" level="debug" additivity="false" includeLocation="true">
<AppenderRef ref="console"/>
</AsyncLogger>

<AsyncLogger name="tcpMonitor" level="debug" additivity="false" includeLocation="true">
<AppenderRef ref="console"/>
</AsyncLogger>

<AsyncLogger name="appMonitor" level="debug" additivity="false" includeLocation="true">
<AppenderRef ref="console"/>
</AsyncLogger>

<AsyncLogger name="cmd" level="debug" additivity="false" includeLocation="true">
<AppenderRef ref="console"/>
</AsyncLogger>

<AsyncLogger name="message" level="debug" additivity="false" includeLocation="true">
<AppenderRef ref="console"/>
</AsyncLogger>

<AsyncLogger name="batchMessage" level="debug" additivity="false" includeLocation="true">
<AppenderRef ref="console"/>
</AsyncLogger>

<AsyncLogger name="http" level="debug" additivity="false" includeLocation="true">
<AppenderRef ref="console"/>
</AsyncLogger>

<AsyncLogger name="metaStorage" level="debug" additivity="false" includeLocation="true">
<AppenderRef ref="console"/>
</AsyncLogger>

<AsyncLogger name="acl" level="debug" additivity="false" includeLocation="true">
<AppenderRef ref="console"/>
</AsyncLogger>

<AsyncLogger name="org.apache.eventmesh.runtime" level="debug" additivity="false" includeLocation="true">
<AppenderRef ref="console"/>
</AsyncLogger>

<AsyncLogger name="org.apache.rocketmq" level="INFO" additivity="false" includeLocation="true">
<AppenderRef ref="console"/>
</AsyncLogger>

<AsyncLogger name="RocketmqClient" level="INFO" additivity="false" includeLocation="true">
<AppenderRef ref="console"/>
</AsyncLogger>

<AsyncLogger name="RocketmqRemoting" level="INFO" additivity="false" includeLocation="true">
<AppenderRef ref="console"/>
</AsyncLogger>

<AsyncLogger name="RocketmqCommon" level="debug" additivity="false" includeLocation="true">
<AppenderRef ref="console"/>
</AsyncLogger>

<AsyncLogger name="sessionLogger" level="debug" additivity="false" includeLocation="true">
<AppenderRef ref="console"/>
</AsyncLogger>

<AsyncLogger name="subscribeLogger" level="info" additivity="false" includeLocation="true">
<AppenderRef ref="console"/>
</AsyncLogger>

<AsyncLogger name="com.alipay.sofa.jraft" level="info" additivity="false" includeLocation="true">
<AppenderRef ref="console"/>
</AsyncLogger>

<AsyncLogger name="io.grpc.netty" level="info" additivity="false" includeLocation="true">
<AppenderRef ref="console"/>
</AsyncLogger>

</Loggers>
</Configuration>
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

package org.apache.eventmesh.admin.server;

import org.apache.eventmesh.common.ComponentLifeCycle;
import org.apache.eventmesh.common.remote.Task;
import org.apache.eventmesh.common.remote.request.ReportHeartBeatRequest;
import org.apache.eventmesh.common.utils.PagedList;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ public void start() {
}

@Override
public void destroy() {
public void stop() {
if (configuration.isEventMeshRegistryPluginEnabled()) {
registryService.unRegister(adminServeInfo);
try {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;

@SpringBootApplication
@SpringBootApplication()
public class ExampleAdminServer {

public static void main(String[] args) throws Exception {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@

package org.apache.eventmesh.admin.server.web;

import org.apache.eventmesh.admin.server.ComponentLifeCycle;
import org.apache.eventmesh.common.ComponentLifeCycle;
import org.apache.eventmesh.common.remote.payload.PayloadFactory;

import javax.annotation.PostConstruct;
Expand All @@ -40,9 +40,9 @@ public void init() throws Exception {
}

@PreDestroy
public void shutdown() {
public void shutdown() throws Exception {
log.info("[{}] server will destroy", this.getClass().getSimpleName());
destroy();
stop();
log.info("[{}] server has be destroy", this.getClass().getSimpleName());
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ public void start() throws Exception {
}

@Override
public void destroy() {
public void stop() {
try {
if (server != null) {
server.shutdown();
Expand Down
1 change: 1 addition & 0 deletions eventmesh-common/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ dependencies {
implementation "org.apache.httpcomponents:httpclient"

implementation "io.netty:netty-all"
compileOnly 'com.mysql:mysql-connector-j'

implementation "io.grpc:grpc-protobuf:${grpcVersion}"
implementation "io.grpc:grpc-stub:${grpcVersion}"
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.eventmesh.common;

import java.util.concurrent.atomic.AtomicBoolean;

import lombok.extern.slf4j.Slf4j;

@Slf4j
public abstract class AbstractComponent implements ComponentLifeCycle {
private final AtomicBoolean started = new AtomicBoolean(false);
private final AtomicBoolean stopped = new AtomicBoolean(false);

@Override
public void start() throws Exception {
if (!started.compareAndSet(false, true)) {
log.info("component [{}] has started", this.getClass());
return;
}
log.info("component [{}] will start", this.getClass());
run();
log.info("component [{}] started successfully", this.getClass());
}

@Override
public void stop() throws Exception {
if (!stopped.compareAndSet(false, true)) {
log.info("component [{}] has stopped", this.getClass());
return;
}
log.info("component [{}] will stop", this.getClass());
shutdown();
log.info("component [{}] stopped successfully", this.getClass());
}

protected abstract void run() throws Exception;

protected abstract void shutdown() throws Exception;
}
Original file line number Diff line number Diff line change
Expand Up @@ -15,14 +15,14 @@
* limitations under the License.
*/

package org.apache.eventmesh.admin.server;
package org.apache.eventmesh.common;

/**
* adminServer ComponentLifeCycle
* LifeCycle of EventMesh Component
*/
public interface ComponentLifeCycle {

void start() throws Exception;

void destroy();
void stop() throws Exception;
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.eventmesh.common.config.connector.rdb;

import org.apache.eventmesh.common.config.connector.rdb.canal.RdbDBDefinition;

import java.util.Set;

import lombok.Data;

@Data
public class JdbcConfig {
private String url;

private String dbAddress;

private int dbPort;

private String userName;

private String passWord;

private Set<RdbDBDefinition> databases;
}
Loading

0 comments on commit 3b770f8

Please sign in to comment.