diff --git a/.gitignore b/.gitignore index e75f1a1279..95bfd3fd59 100644 --- a/.gitignore +++ b/.gitignore @@ -47,4 +47,5 @@ bld/ [Bb]in/ [Oo]bj/ [Ll]og/ -[Ll]ogs/ \ No newline at end of file +[Ll]ogs/ +**/org/apache/eventmesh/connector/jdbc/antlr4/autogeneration/* diff --git a/build.gradle b/build.gradle index e368eb58cc..8f2293ae02 100644 --- a/build.gradle +++ b/build.gradle @@ -90,6 +90,7 @@ allprojects { .exclude('**/org/apache/eventmesh/common/protocol/grpc/cloudevents**') .exclude('**/org/apache/eventmesh/connector/openfunction/client/EventMeshGrpcService**') .exclude('**/org/apache/eventmesh/connector/openfunction/client/CallbackServiceGrpc**') + .exclude('**/org/apache/eventmesh/connector/jdbc/antlr**') dependencies { repositories { @@ -249,6 +250,9 @@ subprojects { rulesMinimumPriority = 5 ruleSets = ["category/java/errorprone.xml", "category/java/bestpractices.xml"] ignoreFailures = true + pmdMain { + excludes = ["**/org/apache/eventmesh/connector/jdbc/antlr4/autogeneration/**"] + } } jar { @@ -341,6 +345,7 @@ subprojects { javadoc { source = sourceSets.main.java destinationDir = reporting.file("javadoc") + options.encoding = "UTF-8" } task packageJavadoc(type: Jar, dependsOn: ['javadoc']) { diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/ResetCountDownLatch.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/ResetCountDownLatch.java new file mode 100644 index 0000000000..c04cc6cdec --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/ResetCountDownLatch.java @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common; + +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.AbstractQueuedSynchronizer; + +/** + * ResetCountDownLatch can reset + * + * @see java.util.concurrent.CountDownLatch + */ +public class ResetCountDownLatch { + + private final RestSync restSync; + + public ResetCountDownLatch(int count) { + this.restSync = new RestSync(count); + } + + + /** + * Causes the current thread to wait until the latch has counted down to zero, unless the thread is {@linkplain Thread#interrupt interrupted}. + * + *

If the current count is zero then this method returns immediately. + * + *

If the current count is greater than zero then the current + * thread becomes disabled for thread scheduling purposes and lies dormant until one of two things happen: + *

+ * + *

If the current thread: + *

+ * then {@link InterruptedException} is thrown and the current thread's + * interrupted status is cleared. + * + * @throws InterruptedException if the current thread is interrupted while waiting + */ + public void await() throws InterruptedException { + restSync.acquireSharedInterruptibly(1); + } + + /** + * Causes the current thread to wait until the latch has counted down to zero, unless the thread is {@linkplain Thread#interrupt interrupted}, or + * the specified waiting time elapses. + * + *

If the current count is zero then this method returns immediately + * with the value {@code true}. + * + *

If the current count is greater than zero then the current + * thread becomes disabled for thread scheduling purposes and lies dormant until one of three things happen: + *

+ * + *

If the count reaches zero then the method returns with the + * value {@code true}. + * + *

If the current thread: + *

+ * then {@link InterruptedException} is thrown and the current thread's + * interrupted status is cleared. + * + *

If the specified waiting time elapses then the value {@code false} + * is returned. If the time is less than or equal to zero, the method + * will not wait at all. + * + * @param timeout the maximum time to wait + * @param unit the time unit of the {@code timeout} argument + * @return {@code true} if the count reached zero and {@code false} if the waiting time elapsed before the count reached zero + * @throws InterruptedException if the current thread is interrupted while waiting + */ + public boolean await(long timeout, TimeUnit unit) + throws InterruptedException { + return restSync.tryAcquireSharedNanos(1, unit.toNanos(timeout)); + } + + + /** + * Decrements the count of the latch, releasing all waiting threads if the count reaches zero. + * + *

If the current count is greater than zero then it is decremented. + * If the new count is zero then all waiting threads are re-enabled for thread scheduling purposes. + * + *

If the current count equals zero then nothing happens. + */ + public void countDown() { + restSync.releaseShared(1); + } + + /** + * Returns the current count. + * + *

This method is typically used for debugging and testing purposes. + * + * @return the current count + */ + public int getCount() { + return restSync.getCount(); + } + + /** + * Reset the CountDownLatch + */ + public void reset() { + restSync.reset(); + } + + /** + * Synchronization control For ResetCountDownLatch. Uses AQS state to represent count. + */ + private static final class RestSync extends AbstractQueuedSynchronizer { + + private final int initCount; + + RestSync(int count) { + if (count < 0) { + throw new IllegalArgumentException("count must be greater than or equal to 0"); + } + this.initCount = count; + setState(count); + } + + protected void reset() { + setState(initCount); + } + + int getCount() { + return getState(); + } + + @Override + protected int tryAcquireShared(int acquires) { + return (getState() == 0) ? 1 : -1; + } + + @Override + protected boolean tryReleaseShared(int releases) { + for (; ; ) { + int count = getState(); + if (count == 0) { + return false; + } + int nextCount = count - 1; + if (compareAndSetState(count, nextCount)) { + return nextCount == 0; + } + } + } + } +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/ThreadWrapper.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/ThreadWrapper.java new file mode 100644 index 0000000000..0b67e69e9c --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/ThreadWrapper.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common; + +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public abstract class ThreadWrapper implements Runnable { + + private final AtomicBoolean started = new AtomicBoolean(false); + protected Thread thread; + protected final ResetCountDownLatch waiter = new ResetCountDownLatch(1); + protected volatile AtomicBoolean hasWakeup = new AtomicBoolean(false); + protected boolean isDaemon = false; + protected volatile boolean isRunning = false; + + public ThreadWrapper() { + + } + + public abstract String getThreadName(); + + public void start() { + + if (!started.compareAndSet(false, true)) { + log.warn("Start thread:{} fail", getThreadName()); + return; + } + this.thread = new Thread(this, getThreadName()); + this.thread.setDaemon(isDaemon); + this.thread.start(); + this.isRunning = true; + log.info("Start thread:{} success", getThreadName()); + } + + public void await() { + if (hasWakeup.compareAndSet(true, false)) { + return; + } + //reset count + waiter.reset(); + try { + waiter.await(); + } catch (InterruptedException e) { + log.error("Thread[{}] Interrupted", getThreadName(), e); + } finally { + hasWakeup.set(false); + } + } + + public void await(long timeout) { + await(timeout, TimeUnit.MILLISECONDS); + } + + public void await(long timeout, TimeUnit timeUnit) { + if (hasWakeup.compareAndSet(true, false)) { + return; + } + //reset count + waiter.reset(); + try { + waiter.await(timeout, timeUnit == null ? TimeUnit.MILLISECONDS : timeUnit); + } catch (InterruptedException e) { + log.error("Thread[{}] Interrupted", getThreadName(), e); + } finally { + hasWakeup.set(false); + } + } + + public void wakeup() { + if (hasWakeup.compareAndSet(false, true)) { + waiter.countDown(); + } + } + + public void shutdownImmediately() { + shutdown(true); + } + + public void shutdown() { + shutdown(false); + } + + private void shutdown(final boolean interruptThread) { + if (!started.compareAndSet(true, false)) { + return; + } + this.isRunning = false; + //wakeup the thread to run + wakeup(); + + try { + if (interruptThread) { + this.thread.interrupt(); + } + if (!this.isDaemon) { + //wait main thread to wait this thread finish + this.thread.join(TimeUnit.SECONDS.toMillis(60)); + } + } catch (InterruptedException e) { + log.error("Thread[{}] Interrupted", getThreadName(), e); + } + } + + public void setDaemon(boolean daemon) { + isDaemon = daemon; + } + + public boolean isStated() { + return this.started.get(); + } +} diff --git a/eventmesh-common/src/test/java/org/apache/eventmesh/common/ResetCountDownLatchTest.java b/eventmesh-common/src/test/java/org/apache/eventmesh/common/ResetCountDownLatchTest.java new file mode 100644 index 0000000000..abc3229a41 --- /dev/null +++ b/eventmesh-common/src/test/java/org/apache/eventmesh/common/ResetCountDownLatchTest.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common; + +import java.util.concurrent.TimeUnit; + +import org.junit.Assert; +import org.junit.Test; + +public class ResetCountDownLatchTest { + + @Test + public void testConstructorParameterError() { + try { + new ResetCountDownLatch(-1); + } catch (IllegalArgumentException e) { + Assert.assertEquals(e.getMessage(), "count must be greater than or equal to 0"); + } + ResetCountDownLatch resetCountDownLatch = new ResetCountDownLatch(1); + Assert.assertEquals(1, resetCountDownLatch.getCount()); + } + + @Test + public void testAwaitTimeout() throws InterruptedException { + ResetCountDownLatch latch = new ResetCountDownLatch(1); + boolean await = latch.await(5, TimeUnit.MILLISECONDS); + Assert.assertFalse(await); + latch.countDown(); + await = latch.await(5, TimeUnit.MILLISECONDS); + Assert.assertTrue(await); + } + + @Test(timeout = 1000) + public void testCountDownAndGetCount() throws InterruptedException { + int count = 2; + ResetCountDownLatch resetCountDownLatch = new ResetCountDownLatch(count); + Assert.assertEquals(count, resetCountDownLatch.getCount()); + resetCountDownLatch.countDown(); + Assert.assertEquals(count - 1, resetCountDownLatch.getCount()); + resetCountDownLatch.countDown(); + resetCountDownLatch.await(); + Assert.assertEquals(0, resetCountDownLatch.getCount()); + } + + @Test + public void testReset() throws InterruptedException { + int count = 2; + ResetCountDownLatch resetCountDownLatch = new ResetCountDownLatch(count); + resetCountDownLatch.countDown(); + Assert.assertEquals(count - 1, resetCountDownLatch.getCount()); + resetCountDownLatch.reset(); + Assert.assertEquals(count, resetCountDownLatch.getCount()); + resetCountDownLatch.countDown(); + resetCountDownLatch.countDown(); + resetCountDownLatch.await(); + Assert.assertEquals(0, resetCountDownLatch.getCount()); + resetCountDownLatch.countDown(); + Assert.assertEquals(0, resetCountDownLatch.getCount()); + resetCountDownLatch.reset(); + resetCountDownLatch.countDown(); + Assert.assertEquals(1, resetCountDownLatch.getCount()); + + } +} \ No newline at end of file diff --git a/eventmesh-common/src/test/java/org/apache/eventmesh/common/ThreadWrapperTest.java b/eventmesh-common/src/test/java/org/apache/eventmesh/common/ThreadWrapperTest.java new file mode 100644 index 0000000000..593bb35206 --- /dev/null +++ b/eventmesh-common/src/test/java/org/apache/eventmesh/common/ThreadWrapperTest.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common; + +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import org.junit.Assert; +import org.junit.Test; + +public class ThreadWrapperTest { + + @Test + public void getThreadName() { + ThreadWrapper wrapper = createThreadWrapper(false); + wrapper.start(); + Assert.assertEquals("EventMesh-Wrapper-mxsm", wrapper.thread.getName()); + } + + @Test + public void start() { + ThreadWrapper wrapper = createThreadWrapper(false); + wrapper.start(); + Assert.assertTrue(wrapper.isStated()); + Assert.assertTrue(wrapper.thread.isAlive()); + } + + @Test(timeout = 1000) + public void await() { + ThreadWrapper wrapper = createThreadWrapper(false); + wrapper.start(); + wrapper.await(1, TimeUnit.MILLISECONDS); + Assert.assertFalse(wrapper.hasWakeup.get()); + wrapper.wakeup(); + Assert.assertTrue(wrapper.hasWakeup.get()); + wrapper.await(); + Assert.assertFalse(wrapper.hasWakeup.get()); + wrapper.await(2, TimeUnit.MILLISECONDS); + + } + + @Test + public void wakeup() { + } + + @Test + public void shutdown() { + AtomicInteger counter = new AtomicInteger(); + ThreadWrapper wrapper = new ThreadWrapper() { + @Override + public String getThreadName() { + return "EventMesh-Wrapper-mxsm"; + } + + @Override + public void run() { + try { + TimeUnit.MILLISECONDS.sleep(500); + } catch (InterruptedException e) { + e.printStackTrace(); + } + counter.set(100); + } + }; + wrapper.start(); + wrapper.shutdown(); + Assert.assertEquals(100, counter.get()); + } + + @Test + public void shutdownImmediately() { + AtomicInteger counter = new AtomicInteger(); + ThreadWrapper wrapper = new ThreadWrapper() { + @Override + public String getThreadName() { + return "EventMesh-Wrapper-mxsm"; + } + + @Override + public void run() { + try { + TimeUnit.SECONDS.sleep(100); + } catch (InterruptedException e) { + return; + } + counter.set(100); + } + }; + wrapper.start(); + wrapper.shutdownImmediately(); + Assert.assertEquals(0, counter.get()); + } + + @Test + public void setDaemon() { + ThreadWrapper threadWrapper = createThreadWrapper(true); + threadWrapper.start(); + Assert.assertTrue(threadWrapper.thread.isDaemon()); + + ThreadWrapper threadWrapper1 = createThreadWrapper(false); + threadWrapper1.start(); + Assert.assertFalse(threadWrapper1.thread.isDaemon()); + } + + private ThreadWrapper createThreadWrapper(boolean daemon) { + ThreadWrapper wrapper = new ThreadWrapper() { + @Override + public String getThreadName() { + return "EventMesh-Wrapper-mxsm"; + } + + @Override + public void run() { + // nothing to do + } + }; + wrapper.setDaemon(daemon); + return wrapper; + } +} \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/build.gradle b/eventmesh-connectors/eventmesh-connector-jdbc/build.gradle new file mode 100644 index 0000000000..e4fac1705e --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/build.gradle @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + id 'java' + //id 'antlr' +} + +repositories { + mavenCentral() +} + +/*generateGrammarSource { + maxHeapSize = '64m' + arguments += ['-package', 'org.apache.eventmesh.connector.jdbc.antlr4.autogeneration', '-visitor'] + outputDirectory = file('src/main/java/org/apache/eventmesh/connector/jdbc/antlr4/autogeneration') +} + +packageSources { + dependsOn generateGrammarSource +}*/ + +dependencies { + //antlr("org.antlr:antlr4:4.13.0") + implementation 'org.antlr:antlr4-runtime:4.13.0' + implementation project(":eventmesh-common") + implementation project(":eventmesh-openconnect:eventmesh-openconnect-java") + implementation project(":eventmesh-spi") + implementation 'com.zendesk:mysql-binlog-connector-java:0.28.0' + implementation 'mysql:mysql-connector-java:8.0.32' + compileOnly 'org.projectlombok:lombok' + annotationProcessor 'org.projectlombok:lombok' + + testImplementation "org.assertj:assertj-core" + + testImplementation "org.mockito:mockito-core" + testImplementation "org.powermock:powermock-module-junit4" + testImplementation "org.powermock:powermock-api-mockito2" +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/AbstractPartition.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/AbstractPartition.java new file mode 100644 index 0000000000..516dac5cfe --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/AbstractPartition.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc; + +public abstract class AbstractPartition implements Partition { + // This class is intended to be a base class for partition implementations. + // It may provide common functionality or default implementations for methods declared in the Partition interface. + // Concrete partition classes will extend this class and provide their specific implementations. +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/CatalogChanges.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/CatalogChanges.java new file mode 100644 index 0000000000..a58d6c3057 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/CatalogChanges.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc; + +import org.apache.eventmesh.connector.jdbc.event.SchemaChangeEventType; +import org.apache.eventmesh.connector.jdbc.table.catalog.CatalogSchema; +import org.apache.eventmesh.connector.jdbc.table.catalog.Column; +import org.apache.eventmesh.connector.jdbc.table.catalog.Table; + +import java.util.List; + +import lombok.Data; + +@Data +/** + * Represents changes in a catalog, such as schema or table modifications. + */ +public class CatalogChanges { + + /** + * The type of change (e.g., "T(Table)" or "D(Database)"). + * + * {@link SchemaChangeEventType#type} + * + */ + private String type; + + /** + * The specific operation type (e.g., "C(Create)", "D(Drop)", "A(Alert)"). + * + * {@link SchemaChangeEventType#operationType} + * + */ + private String operationType; + // The catalog schema associated with the changes + private CatalogSchema catalog; + // The table associated with the changes + private Table table; + // The list of columns affected by the changes + private List columns; + + private CatalogChanges(String type, String operationType, CatalogSchema catalog, Table table, + List columns) { + this.type = type; + this.operationType = operationType; + this.catalog = catalog; + this.table = table; + this.columns = columns; + } + + /** + * Creates a new instance of the Builder for constructing CatalogChanges. + * + * @return The Builder instance. + */ + public static Builder newBuilder() { + return new Builder(); + } + + /** + * Builder class for constructing CatalogChanges. + */ + public static class Builder { + + private String type; + private String operationType; + private CatalogSchema catalog; + private Table table; + private List columns; + + /** + * Sets the operation type for the change. + * + * @param changeEventType The SchemaChangeEventType representing the change. + * @return The Builder instance. + */ + public Builder operationType(SchemaChangeEventType changeEventType) { + this.type = changeEventType.ofType(); + this.operationType = changeEventType.ofOperationType(); + return this; + } + + /** + * Sets the catalog schema associated with the changes. + * + * @param catalog The CatalogSchema instance. + * @return The Builder instance. + */ + public Builder catalog(CatalogSchema catalog) { + this.catalog = catalog; + return this; + } + + /** + * Sets the table associated with the changes. + * + * @param table The Table instance. + * @return The Builder instance. + */ + public Builder table(Table table) { + this.table = table; + return this; + } + + /** + * Sets the list of columns affected by the changes. + * + * @param columns The list of Column instances. + * @return The Builder instance. + */ + public Builder columns(List columns) { + this.columns = columns; + return this; + } + + /** + * Builds a new CatalogChanges instance. + * + * @return The constructed CatalogChanges instance. + */ + public CatalogChanges build() { + return new CatalogChanges(type, operationType, catalog, table, columns); + } + } +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/DataTypeConvertor.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/DataTypeConvertor.java new file mode 100644 index 0000000000..38dee7ea5b --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/DataTypeConvertor.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc; + +import org.apache.eventmesh.connector.jdbc.exception.DataTypeConvertException; +import org.apache.eventmesh.connector.jdbc.table.type.EventMeshDataType; + +import java.sql.JDBCType; +import java.util.Map; + +/** + * convert data types between EventMesh and connector. + */ +public interface DataTypeConvertor { + + /** + * Converts a string representation of a connector data type to the corresponding JDBCType. + * + * @param connectorDataType The string representation of the connector data type. + * @return The corresponding JDBCType, or null if the connector data type is not recognized. + */ + JDBCType toJDBCType(String connectorDataType); + + + /** + * Converts a connector data type to an EventMesh data type. + * + * @param connectorDataType The connector data type to be converted. + * @return The converted EventMesh data type. + * @throws DataTypeConvertException If the conversion fails. + */ + EventMeshDataType toEventMeshType(String connectorDataType) throws DataTypeConvertException; + + /** + * Converts JDBCType and dataTypeProperties to EventMeshDataType. + * + * @param jdbcType the JDBCType to be converted + * @param dataTypeProperties the properties of the data type + * @return the converted EventMeshDataType + * @throws DataTypeConvertException if there is an error during conversion + */ + EventMeshDataType toEventMeshType(JDBCType jdbcType, Map dataTypeProperties) throws DataTypeConvertException; + + /** + * Converts a connector data type to an EventMesh data type with additional data type properties. + * + * @param connectorDataType The connector data type to be converted. + * @param dataTypeProperties Additional data type properties. + * @return The converted EventMesh data type. + * @throws DataTypeConvertException If the conversion fails. + */ + EventMeshDataType toEventMeshType(T connectorDataType, Map dataTypeProperties) throws DataTypeConvertException; + + /** + * Converts an EventMesh data type to a connector data type with additional data type properties. + * + * @param eventMeshDataType The EventMesh data type to be converted. + * @param dataTypeProperties Additional data type properties. + * @return The converted connector data type. + * @throws DataTypeConvertException If the conversion fails. + */ + T toConnectorType(EventMeshDataType eventMeshDataType, Map dataTypeProperties) throws DataTypeConvertException; +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/DatabaseDialect.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/DatabaseDialect.java new file mode 100644 index 0000000000..76dbf5fab9 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/DatabaseDialect.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc; + +import org.apache.eventmesh.connector.jdbc.connection.JdbcConnection; +import org.apache.eventmesh.connector.jdbc.connection.JdbcConnectionProvider; +import org.apache.eventmesh.connector.jdbc.table.catalog.Catalog; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; + +/** + * Interface for a database dialect, which extends the ConnectionProvider and Catalog interfaces. + */ +public interface DatabaseDialect extends JdbcConnectionProvider, Catalog { + + /** + * Initializes the database dialect. + */ + void init(); + + /** + * Starts the database dialect. + */ + void start(); + + /** + * Retrieves the name of the database dialect. + * + * @return The name of the database dialect. + */ + String getName(); + + /** + * Creates a prepared statement for the given SQL query using the provided database connection. + * + * @param connection The database connection. + * @param sql The SQL query. + * @return The prepared statement. + * @throws SQLException If an error occurs while creating the prepared statement. + */ + PreparedStatement createPreparedStatement(Connection connection, String sql) throws SQLException; + + /** + * Retrieves the JDBC driver meta-data associated with the database dialect. + * + * @return The JDBC driver meta-data. + */ + JdbcDriverMetaData getJdbcDriverMetaData(); + + /** + * Retrieves the JDBC protocol associated with the database dialect. + * + * @return The JDBC protocol. + */ + String jdbcProtocol(); +} + diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/Field.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/Field.java new file mode 100644 index 0000000000..24dcb57747 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/Field.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc; + +public class Field { + + private String type; + private String optional; + private String name; +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/JdbcConnectData.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/JdbcConnectData.java new file mode 100644 index 0000000000..f708192129 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/JdbcConnectData.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc; + +public final class JdbcConnectData { + + private Payload payload = new Payload(); + + private Schema schema; + + public Payload getPayload() { + return payload; + } + + public void setPayload(Payload payload) { + this.payload = payload; + } + + public Schema getSchema() { + return schema; + } + + public void setSchema(Schema schema) { + this.schema = schema; + } +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/JdbcContext.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/JdbcContext.java new file mode 100644 index 0000000000..506e4baa2c --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/JdbcContext.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc; + +/** + * Interface representing a JDBC context. + */ +public interface JdbcContext { + + Part getPartition(); + + OffSetCtx getOffsetContext(); + +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/JdbcDriverMetaData.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/JdbcDriverMetaData.java new file mode 100644 index 0000000000..fdc1e0fcde --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/JdbcDriverMetaData.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.ToString; + +/** + * Represents metadata information about a JDBC driver + */ +@Data +@AllArgsConstructor +@ToString +public class JdbcDriverMetaData { + + // The major version number of the JDBC driver + private final int jdbcMajorVersion; + + // The minor version number of the JDBC driver + private final int jdbcMinorVersion; + + // The name of the JDBC driver + private final String jdbcDriverName; + + // The name of the database product + private final String databaseProductName; + + // The version of the database product + private final String databaseProductVersion; + +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/OffsetContext.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/OffsetContext.java new file mode 100644 index 0000000000..98bd2213d7 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/OffsetContext.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc; + +import java.util.Map; + +/** + * The OffsetContext interface represents the offset context for event processing. It provides methods to retrieve the offset map and check if a + * snapshot is currently running. + */ +public interface OffsetContext { + + /** + * Retrieves the offset map associated with the context. + * + * @return The offset map. + */ + Map getOffset(); + + /** + * Checks if a snapshot is currently running. + * + * @return True if a snapshot is running, false otherwise. + */ + boolean isSnapshotRunning(); + + boolean isSnapshotCompleted(); + + void markSnapshotRunning(); + + void markSnapshotCompleted(); + +} + diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/Partition.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/Partition.java new file mode 100644 index 0000000000..e1394a4f7c --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/Partition.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc; + +import java.util.Map; + +/** + * This interface represents a partition. + */ +public interface Partition { + + /** + *

Returns a map representing the partition.The keys of the map represent the partition keys, and the values represent the corresponding + * partition values.

+ * + * @return a map representing the partition + */ + Map getPartition(); + +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/PartitionOffSetContextPair.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/PartitionOffSetContextPair.java new file mode 100644 index 0000000000..30ec03d111 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/PartitionOffSetContextPair.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc; + +public final class PartitionOffSetContextPair { + + private Part partition; + + private Offset offsetContext; + + private PartitionOffSetContextPair(Part partition, Offset offsetContext) { + this.partition = partition; + this.offsetContext = offsetContext; + } + + public static PartitionOffSetContextPair of(Part partition, Offset offset) { + return new PartitionOffSetContextPair<>(partition, offset); + } + + public Part getPartition() { + return partition; + } + + public Offset getOffsetContext() { + return offsetContext; + } +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/Payload.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/Payload.java new file mode 100644 index 0000000000..2a9235bd9b --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/Payload.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc; + +import org.apache.eventmesh.connector.jdbc.source.SourceMateData; + +import java.util.HashMap; + +public final class Payload extends HashMap { + + public Payload withSource(SourceMateData source) { + super.put("source", source); + return this; + } + + public Payload withDdl(String ddl) { + super.put("ddl", ddl); + return this; + } + + public Payload withCatalogChanges(CatalogChanges catalogChanges) { + super.put("catalogChanges", catalogChanges); + return this; + } + + public SourceMateData ofSourceMateData() { + return (SourceMateData) super.get("source"); + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + + private final Payload payload; + + private Builder() { + payload = new Payload(); + } + + public Builder put(String key, Object value) { + payload.put(key, value); + return this; + } + + public Builder withSource(SourceMateData source) { + payload.put("source", source); + return this; + } + + public Payload build() { + return payload; + } + } + +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/Schema.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/Schema.java new file mode 100644 index 0000000000..7916a911cc --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/Schema.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc; + +import java.util.List; + +public class Schema { + + private boolean optional; + + private Field field; + + private List fields; + +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/UniversalJdbcContext.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/UniversalJdbcContext.java new file mode 100644 index 0000000000..4577ec5ab2 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/UniversalJdbcContext.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc; + +import org.apache.eventmesh.connector.jdbc.ddl.DdlParser; +import org.apache.eventmesh.connector.jdbc.table.catalog.CatalogTableSet; + +public abstract class UniversalJdbcContext implements + JdbcContext { + + private PartitionOffSetContextPair poCtx; + + private Parser parser; + + private CatalogTableSet tableSet = new CatalogTableSet(); + + public UniversalJdbcContext(PartitionOffSetContextPair poCtx, Parser parser) { + this.poCtx = poCtx; + this.parser = parser; + } + + public UniversalJdbcContext(Part part, OffSetCtx offSetCtx, Parser parser) { + this.poCtx = PartitionOffSetContextPair.of(part, offSetCtx); + this.parser = parser; + } + + + @Override + public Part getPartition() { + return poCtx.getPartition(); + } + + + @Override + public OffSetCtx getOffsetContext() { + return poCtx.getOffsetContext(); + } + + public Parser getParser() { + return parser; + } + + public CatalogTableSet getCatalogTableSet() { + return this.tableSet; + } +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/UniversalOffsetContext.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/UniversalOffsetContext.java new file mode 100644 index 0000000000..baa7ba3554 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/UniversalOffsetContext.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc; + +public abstract class UniversalOffsetContext implements OffsetContext { + +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/antlr4/Antlr4DdlParser.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/antlr4/Antlr4DdlParser.java new file mode 100644 index 0000000000..61e79eede2 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/antlr4/Antlr4DdlParser.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.antlr4; + +import org.apache.eventmesh.connector.jdbc.antlr4.listener.Antlr4DdlParserListener; +import org.apache.eventmesh.connector.jdbc.ddl.AbstractDdlParser; +import org.apache.eventmesh.connector.jdbc.ddl.DdlParserCallback; +import org.apache.eventmesh.connector.jdbc.table.catalog.CatalogTableSet; + +import org.antlr.v4.runtime.CharStream; +import org.antlr.v4.runtime.CharStreams; +import org.antlr.v4.runtime.CodePointCharStream; +import org.antlr.v4.runtime.CommonTokenStream; +import org.antlr.v4.runtime.Lexer; +import org.antlr.v4.runtime.Parser; +import org.antlr.v4.runtime.tree.ParseTree; +import org.antlr.v4.runtime.tree.ParseTreeWalker; + +public abstract class Antlr4DdlParser extends AbstractDdlParser { + + private Antlr4DdlParserListener antlr4DdlParserListener; + + private CatalogTableSet tableSet; + + public Antlr4DdlParser(boolean skipViews, boolean skipComments) { + super(skipViews, skipComments); + } + + /** + * Parses the given DDL SQL statement. + * + * @param ddlSql the DDL SQL statement to be parsed + */ + @Override + public void parse(String ddlSql, DdlParserCallback callback) { + CodePointCharStream ddlSqlStream = CharStreams.fromString(ddlSql); + L lexer = buildLexerInstance(ddlSqlStream); + P parser = buildParserInstance(new CommonTokenStream(lexer)); + ParseTree parseTree = parseTree(parser); + antlr4DdlParserListener = createParseTreeWalkerListener(callback); + ParseTreeWalker.DEFAULT.walk(antlr4DdlParserListener, parseTree); + } + + protected abstract L buildLexerInstance(CharStream charStreams); + + protected abstract P buildParserInstance(CommonTokenStream commonTokenStream); + + protected abstract ParseTree parseTree(P parser); + + protected abstract Antlr4DdlParserListener createParseTreeWalkerListener(DdlParserCallback callback); + + public void setCatalogTableSet(CatalogTableSet tableSet) { + this.tableSet = tableSet; + } + + public CatalogTableSet getCatalogTableSet() { + return this.tableSet; + } + +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/antlr4/listener/Antlr4DdlParserListener.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/antlr4/listener/Antlr4DdlParserListener.java new file mode 100644 index 0000000000..e3f3d3f539 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/antlr4/listener/Antlr4DdlParserListener.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.antlr4.listener; + +import org.antlr.v4.runtime.tree.ParseTreeListener; + +public interface Antlr4DdlParserListener extends ParseTreeListener { + +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/common/SourceInfo.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/common/SourceInfo.java new file mode 100644 index 0000000000..0bbe0354b9 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/common/SourceInfo.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.common; + +/** + * Interface representing source information. + */ +public interface SourceInfo { + +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/config/JdbcConfig.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/config/JdbcConfig.java new file mode 100644 index 0000000000..9c7558426c --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/config/JdbcConfig.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.config; + +import java.util.Properties; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * Represents the configuration for a JDBC connection. + */ +@AllArgsConstructor +@NoArgsConstructor +@Data +public class JdbcConfig { + + private String databaseName; + + // The hostname of the database server. + private String hostname; + + // The port number of the database server. + private int port; + + // The username for the database connection. + private String user; + + // The password for the database connection. + private String password; + + private String initialStatements; + + private int connectTimeout; + + /** + * Converts the JdbcConfig object to a Properties object containing the user and password. + * + * @return The Properties object representing the JdbcConfig. + */ + public Properties asProperties() { + Properties props = new Properties(); + props.put("user", this.user); + props.put("password", this.password); + return props; + } +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/config/JdbcServerConfig.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/config/JdbcServerConfig.java new file mode 100644 index 0000000000..451ecf71e9 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/config/JdbcServerConfig.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.config; + +import org.apache.eventmesh.openconnect.api.config.Config; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +/** + * configuration for the JDBC server. + */ +@Data +@EqualsAndHashCode(callSuper = true) +public class JdbcServerConfig extends Config { + + // Indicates whether the source is enabled or not + private boolean sourceEnable; + + // Indicates whether the sink is enabled or not + private boolean sinkEnable; + +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/connection/JdbcConnection.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/connection/JdbcConnection.java new file mode 100644 index 0000000000..373927b098 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/connection/JdbcConnection.java @@ -0,0 +1,596 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.connection; + +import org.apache.eventmesh.connector.jdbc.JdbcDriverMetaData; +import org.apache.eventmesh.connector.jdbc.config.JdbcConfig; + +import org.apache.commons.lang3.StringUtils; + +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; + +import javax.annotation.concurrent.ThreadSafe; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +/** + * JdbcConnection class representing a JDBC connection. + * Implements the AutoCloseable interface. + */ +public class JdbcConnection implements AutoCloseable { + + private static final int CONNECTION_VALID_CHECK_TIMEOUT_IN_SEC = 3; + + private static final String STATEMENT_DELIMITER = ";"; + + private final JdbcConfig jdbcConfig; + + private volatile Connection connection; + + private final InitialOperation initialOperation; + + private final ConnectionFactory connectionFactory; + + private JdbcDriverMetaData jdbcDriverMetaData; + + private boolean lazyConnection = true; + + public JdbcConnection(JdbcConfig jdbcConfig, InitialOperation initialOperation, ConnectionFactory connectionFactory) { + this(jdbcConfig, initialOperation, connectionFactory, true); + } + + public JdbcConnection(JdbcConfig jdbcConfig, InitialOperation initialOperation, ConnectionFactory connectionFactory, boolean lazyConnection) { + this.jdbcConfig = jdbcConfig; + this.initialOperation = initialOperation; + this.connectionFactory = connectionFactory; + this.lazyConnection = lazyConnection; + if (!this.lazyConnection) { + try { + connection(); + } catch (SQLException e) { + log.warn("Get Connection error", e); + throw new RuntimeException(e); + } + } + } + + /** + * Closes the JDBC connection. + * + * @throws Exception if an error occurs while closing the connection. + */ + @Override + public void close() throws Exception { + if (connection != null) { + connection.close(); + } + } + + /** + * Retrieves the JDBC configuration. + * + * @return The JDBC configuration. + */ + public JdbcConfig getJdbcConfig() { + return jdbcConfig; + } + + /** + * Sets the auto-commit mode for the connection. + * + * @param autoCommit The auto-commit mode. + * @return The JdbcConnection instance. + * @throws SQLException if a database access error occurs. + */ + public JdbcConnection setAutoCommit(boolean autoCommit) throws SQLException { + connection().setAutoCommit(autoCommit); + return this; + } + + /** + * Retrieves the JDBC connection. Creates a new connection if not already connected. + * + * @return The JDBC connection. + * @throws SQLException if a database access error occurs. + */ + public synchronized Connection connection() throws SQLException { + return connection(true); + } + + /** + * Retrieves the JDBC connection. Creates a new connection if not already connected. + * + * @param executeOnConnect Flag indicating whether to execute initial statements on connect. + * @return The JDBC connection. + * @throws SQLException if a database access error occurs. + */ + public synchronized Connection connection(boolean executeOnConnect) throws SQLException { + if (!isConnected()) { + connection = connectionFactory.connect(jdbcConfig); + if (!isConnected()) { + throw new SQLException("Unable to obtain a JDBC connection"); + } + + if (initialOperation != null) { + execute(initialOperation); + } + final String statements = jdbcConfig.getInitialStatements(); + if (StringUtils.isNotBlank(statements) && executeOnConnect) { + String[] split = statements.split(STATEMENT_DELIMITER); + execute(split); + } + jdbcDriverMetaData = createJdbcDriverInfo(); + } + return connection; + } + + /** + * Creates the JDBC driver metadata by retrieving information from the provided connection's database metadata. + * + * @return The JDBC driver metadata. + * @throws SQLException if a database access error occurs. + */ + private JdbcDriverMetaData createJdbcDriverInfo() throws SQLException { + DatabaseMetaData metadata = connection.getMetaData(); + + // Retrieve the JDBC driver information from the database metadata + int majorVersion = metadata.getJDBCMajorVersion(); + int minorVersion = metadata.getJDBCMinorVersion(); + String driverName = metadata.getDriverName(); + String productName = metadata.getDatabaseProductName(); + String productVersion = metadata.getDatabaseProductVersion(); + + // Create and return the JdbcDriverMetaData instance + return new JdbcDriverMetaData(majorVersion, minorVersion, driverName, productName, productVersion); + } + + + public JdbcDriverMetaData getJdbcDriverMetaData() { + return jdbcDriverMetaData; + } + + /** + * Executes SQL statements on the JDBC connection. + * + * @param sqlStatements The SQL statements to execute. + * @return The JdbcConnection instance. + * @throws SQLException if a database access error occurs. + */ + public JdbcConnection execute(String... sqlStatements) throws SQLException { + return execute(statement -> { + for (String sqlStatement : sqlStatements) { + if (sqlStatement != null) { + if (log.isDebugEnabled()) { + log.debug("Executing '{}'", sqlStatement); + } + statement.execute(sqlStatement); + } + } + }); + } + + /** + * Executes a custom initial operation on the JDBC connection. + * + * @param operation The initial operation to execute. + * @return The JdbcConnection instance. + * @throws SQLException if a database access error occurs. + */ + public JdbcConnection execute(InitialOperation operation) throws SQLException { + Connection conn = connection(); + try (Statement statement = conn.createStatement()) { + operation.apply(statement); + commit(); + } + return this; + } + + /** + * Execute the given SQL statements without committing the changes. + * + * @param sqlStatements The SQL statements to execute + * @return This JdbcConnection instance + * @throws SQLException If an SQL error occurs + */ + public JdbcConnection executeWithoutCommitting(String... sqlStatements) throws SQLException { + Connection conn = connection(); + if (conn.getAutoCommit()) { + throw new SQLException("Cannot execute without committing because auto-commit is enabled"); + } + + try (Statement statement = conn.createStatement()) { + for (String sqlStatement : sqlStatements) { + if (log.isDebugEnabled()) { + log.debug("Executing sql statement: {}", sqlStatement); + } + statement.execute(sqlStatement); + } + } + + return this; + } + + /** + * Checks if the JDBC connection is connected. + * + * @return true if the connection is connected, false otherwise. + * @throws SQLException if a database access error occurs. + */ + public synchronized boolean isConnected() throws SQLException { + if (connection == null) { + return false; + } + return !connection.isClosed(); + } + + /** + * Checks if the JDBC connection is valid. + * + * @return true if the connection is valid, false otherwise. + * @throws SQLException if a database access error occurs. + */ + public synchronized boolean isValid() throws SQLException { + return isConnected() && connection.isValid(CONNECTION_VALID_CHECK_TIMEOUT_IN_SEC); + } + + /** + * Commits the changes on the JDBC connection. + * + * @return The JdbcConnection instance. + * @throws SQLException if a database access error occurs. + */ + public JdbcConnection commit() throws SQLException { + Connection conn = connection(); + if (!conn.getAutoCommit()) { + conn.commit(); + } + return this; + } + + /** + * Executes a query on the JDBC connection and consumes the result set using the provided consumer. + * + * @param sql The SQL query to execute. + * @param resultConsumer The consumer to process the result set. + * @return The JdbcConnection instance. + * @throws SQLException if a database access error occurs. + */ + public JdbcConnection query(String sql, JdbcResultSetConsumer resultConsumer) throws SQLException { + // Check if the connection is connected and valid + if (isConnected() && isValid()) { + connection(); + } + return query(sql, Connection::createStatement, resultConsumer); + } + + /** + * Executes a query on the JDBC connection and consumes the result set using the provided consumer. + * + * @param sql The SQL query to execute. + * @param statementFactory The factory to create the statement. + * @param resultConsumer The consumer to process the result set. + * @return The JdbcConnection instance. + * @throws SQLException if a database access error occurs. + */ + public JdbcConnection query(String sql, StatementFactory statementFactory, JdbcResultSetConsumer resultConsumer) throws SQLException { + Connection conn = connection(); + try (Statement statement = statementFactory.createStatement(conn)) { + if (log.isDebugEnabled()) { + log.debug("Query sql '{}'", sql); + } + try (ResultSet resultSet = statement.executeQuery(sql)) { + if (resultConsumer != null) { + resultConsumer.accept(resultSet); + } + } + } + return this; + } + + /** + * Executes a query on the JDBC connection and maps the result set using the provided ResultSetMapper. + * + * @param sql The SQL query to execute. + * @param resultSetMapper The mapper to map the result set to an object. + * @return The mapped object. + * @throws SQLException if a database access error occurs. + */ + public T query(String sql, ResultSetMapper resultSetMapper) throws SQLException { + // Check if the connection is connected and valid + if (isConnected() && isValid()) { + connection(); + } + return query(sql, Connection::createStatement, resultSetMapper); + } + + /** + * Executes a query on the JDBC connection and maps the result set using the provided ResultSetMapper. + * + * @param sql The SQL query to execute. + * @param statementFactory The factory to create the statement. + * @param resultSetMapper The mapper to map the result set to an object. + * @return The mapped object. + * @throws SQLException if a database access error occurs. + */ + public T query(String sql, StatementFactory statementFactory, ResultSetMapper resultSetMapper) throws SQLException { + Connection conn = connection(); + try (Statement statement = statementFactory.createStatement(conn)) { + if (log.isDebugEnabled()) { + log.debug("Query sql '{}'", sql); + } + try (ResultSet resultSet = statement.executeQuery(sql)) { + if (resultSetMapper != null) { + return resultSetMapper.map(resultSet); + } + } + } + return null; + } + + /** + * Executes a prepared query on the JDBC connection and consumes the result set using the provided consumer. + * + * @param sql The SQL query to execute. + * @param resultConsumer The consumer to process the result set. + * @param preparedParameters The prepared parameters for the query. + * @return The JdbcConnection instance. + * @throws SQLException if a database access error occurs. + */ + public JdbcConnection preparedQuery(String sql, JdbcResultSetConsumer resultConsumer, PreparedParameter... preparedParameters) + throws SQLException { + // Check if the connection is connected and valid + if (isConnected() && isValid()) { + connection(); + } + return preparedQuery(sql, (conn, statement) -> conn.prepareStatement(sql), resultConsumer, preparedParameters); + } + + /** + * Executes a prepared query on the JDBC connection and consumes the result set using the provided consumer. + * + * @param sql The SQL query to execute. + * @param preparedStatementFactory The factory to create the prepared statement. + * @param resultConsumer The consumer to process the result set. + * @param preparedParameters The prepared parameters for the query. + * @return The JdbcConnection instance. + * @throws SQLException if a database access error occurs. + */ + public JdbcConnection preparedQuery(String sql, PreparedStatementFactory preparedStatementFactory, JdbcResultSetConsumer resultConsumer, + PreparedParameter... preparedParameters) throws SQLException { + + Connection conn = connection(); + try (PreparedStatement preparedStatement = preparedStatementFactory.createPreparedStatement(conn, sql)) { + if (log.isDebugEnabled()) { + log.debug("Query sql '{}'", sql); + } + if (preparedParameters != null) { + for (int index = 0; index < preparedParameters.length; ++index) { + final PreparedParameter preparedParameter = preparedParameters[index]; + if (preparedParameter.getJdbcType() == null) { + preparedStatement.setObject(index + 1, preparedParameter.getValue()); + } else { + preparedStatement.setObject(index + 1, preparedParameter.getValue(), preparedParameter.getJdbcType().getVendorTypeNumber()); + } + } + } + try (ResultSet resultSet = preparedStatement.executeQuery()) { + if (resultConsumer != null) { + resultConsumer.accept(resultSet); + } + } + } + return this; + } + + + /** + * Executes a prepared query on the JDBC connection and maps the result set using the provided ResultSetMapper. + * + * @param sql The SQL query to execute. + * @param resultSetMapper The mapper to map the result set to an object. + * @param preparedParameters The prepared parameters for the query. + * @return The mapped object. + * @throws SQLException if a database access error occurs. + */ + public T preparedQuery(String sql, ResultSetMapper resultSetMapper, PreparedParameter... preparedParameters) + throws SQLException { + // Check if the connection is connected and valid + if (isConnected() && isValid()) { + connection(); + } + return preparedQuery(sql, (conn, statement) -> conn.prepareStatement(sql), resultSetMapper, preparedParameters); + } + + /** + * Executes a prepared query on the JDBC connection and maps the result set using the provided ResultSetMapper. + * + * @param sql The SQL query to execute. + * @param preparedStatementFactory The factory to create the prepared statement. + * @param resultSetMapper The mapper to map the result set to an object. + * @param preparedParameters The prepared parameters for the query. + * @return The mapped object. + * @throws SQLException if a database access error occurs. + */ + public T preparedQuery(String sql, PreparedStatementFactory preparedStatementFactory, ResultSetMapper resultSetMapper, + PreparedParameter... preparedParameters) throws SQLException { + + Connection conn = connection(); + try (PreparedStatement preparedStatement = preparedStatementFactory.createPreparedStatement(conn, sql)) { + if (log.isDebugEnabled()) { + log.debug("Query sql '{}'", sql); + } + if (preparedParameters != null) { + for (int index = 0; index < preparedParameters.length; ++index) { + final PreparedParameter preparedParameter = preparedParameters[index]; + if (preparedParameter.getJdbcType() == null) { + preparedStatement.setObject(index + 1, preparedParameter.getValue()); + } else { + preparedStatement.setObject(index + 1, preparedParameter.getValue(), preparedParameter.getJdbcType().getVendorTypeNumber()); + } + } + } + try (ResultSet resultSet = preparedStatement.executeQuery()) { + if (resultSetMapper != null) { + return resultSetMapper.map(resultSet); + } + } + } + return null; + } + + public Statement createStatement(int fetchSize, int defaultFetchSize) throws SQLException { + final Statement statement = connection().createStatement(); + statement.setFetchSize(fetchSize <= 0 ? defaultFetchSize : fetchSize); + return statement; + } + + + /** + * Functional interface for the initial operation on the JDBC connection. + */ + @FunctionalInterface + public interface InitialOperation { + + /** + * Applies the operation on the JDBC statement. + * + * @param statement The JDBC statement. + * @throws SQLException if a database access error occurs. + */ + void apply(Statement statement) throws SQLException; + } + + /** + * Functional interface for creating JDBC statements. + */ + @FunctionalInterface + public interface StatementFactory { + + /** + * Creates a JDBC statement. + * + * @param connection The JDBC connection. + * @return The JDBC statement. + * @throws SQLException if a database access error occurs. + */ + Statement createStatement(Connection connection) throws SQLException; + } + + /** + * Functional interface for creating JDBC prepared statements. + */ + @FunctionalInterface + public interface PreparedStatementFactory { + + /** + * Creates a JDBC prepared statement with the provided connection and SQL query. + * + * @param connection The JDBC connection. + * @param sql The SQL query. + * @return The JDBC prepared statement. + * @throws SQLException if a database access error occurs. + */ + PreparedStatement createPreparedStatement(Connection connection, String sql) throws SQLException; + } + + /** + * Functional interface for creating JDBC connections. + */ + @FunctionalInterface + @ThreadSafe + public interface ConnectionFactory { + + /** + * Creates a JDBC connection. + * + * @param config The JDBC configuration. + * @return The JDBC connection. + * @throws SQLException if a database access error occurs. + */ + Connection connect(JdbcConfig config) throws SQLException; + } + + /** + * Functional interface for mapping a ResultSet to an object of type T. + * + * @param The type of object to be mapped. + */ + @FunctionalInterface + public interface ResultSetMapper { + + /** + * Maps a ResultSet to an object of type T. + * + * @param rs The ResultSet to be mapped. + * @return The mapped object. + * @throws SQLException if a database access error occurs. + */ + T map(ResultSet rs) throws SQLException; + } + + /** + * Functional interface for consuming a ResultSet. + */ + @FunctionalInterface + public interface JdbcResultSetConsumer { + + /** + * Accepts a ResultSet and performs an operation on it. + * + * @param resultSet The ResultSet to be consumed. + * @throws SQLException if a database access error occurs. + */ + void accept(ResultSet resultSet) throws SQLException; + + } + + /** + * Creates a ConnectionFactory that uses a pattern-based URL with placeholder values. + * + * @param urlWithPlaceholder The URL pattern with placeholders. + * @param replaces The replacement values for the placeholders. + * @return The ConnectionFactory instance. + */ + @SuppressWarnings("unchecked") + public static ConnectionFactory createPatternConnectionFactory(String urlWithPlaceholder, String... replaces) { + return config -> { + String url; + if (replaces != null && replaces.length > 0) { + url = String.format(urlWithPlaceholder, (Object[]) replaces); + } else { + url = urlWithPlaceholder; + } + if (log.isDebugEnabled()) { + log.debug("URL: {}", url); + } + Connection connection = DriverManager.getConnection(url, config.asProperties()); + if (log.isDebugEnabled()) { + log.debug("User [{}] Connected to {}", config.getUser(), url); + } + return connection; + }; + } + +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/connection/JdbcConnectionProvider.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/connection/JdbcConnectionProvider.java new file mode 100644 index 0000000000..602208eb63 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/connection/JdbcConnectionProvider.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.connection; + +import org.apache.eventmesh.connector.jdbc.exception.JdbcConnectionException; + +import java.sql.Connection; +import java.sql.SQLException; + +/** + * Obtaining a database connection and checking its validity. + */ +public interface JdbcConnectionProvider extends AutoCloseable { + + /** + * Obtains a database connection. + * + * @return A database connection. + */ + JC getConnection(); + + JC newConnection(); + + /** + * Checks if a database connection is valid. + * + * @param connection The database connection to check. + * @param timeout The timeout in seconds. + * @return True if the connection is valid, false otherwise. + * @throws JdbcConnectionException If there is an error checking the connection. + * @throws SQLException If there is an error with the SQL query. + */ + boolean isValid(Connection connection, int timeout) throws JdbcConnectionException, SQLException; + +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/connection/PreparedParameter.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/connection/PreparedParameter.java new file mode 100644 index 0000000000..4461e1041b --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/connection/PreparedParameter.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.connection; + +import java.sql.JDBCType; + +import lombok.Data; + +/** + * Represents a parameter for a prepared statement. + */ +@Data +public class PreparedParameter { + + private Object value; + + private JDBCType jdbcType; + +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/ddl/AbstractDdlParser.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/ddl/AbstractDdlParser.java new file mode 100644 index 0000000000..fdac829dd3 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/ddl/AbstractDdlParser.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.ddl; + +import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; + +import lombok.Getter; + +public abstract class AbstractDdlParser implements DdlParser { + + private final boolean skipViews; + + private final boolean skipComments; + + @Getter + private final TableId tableId; + + public AbstractDdlParser(boolean skipViews, boolean skipComments) { + this.skipViews = skipViews; + this.skipComments = skipComments; + this.tableId = new TableId(); + } + + @Override + public void setCurrentDatabase(String databaseName) { + this.tableId.setCatalogName(databaseName); + } + + @Override + public void setCurrentSchema(String schema) { + this.tableId.setSchemaName(schema); + } + + public String getCurrentDatabase() { + return this.tableId.getCatalogName(); + } + + public boolean isSkipViews() { + return skipViews; + } + + public boolean isSkipComments() { + return skipComments; + } +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/ddl/DdlParser.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/ddl/DdlParser.java new file mode 100644 index 0000000000..7fcb9dfd78 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/ddl/DdlParser.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.ddl; + +/** + * Interface for parsing DDL SQL statements. + */ +public interface DdlParser { + + /** + * Parses the given DDL SQL statement. + * + * @param ddlSql the DDL SQL statement to be parsed + */ + default void parse(String ddlSql) { + parse(ddlSql, null); + } + + void parse(String ddlSql, DdlParserCallback callback); + + /** + * Sets the current database. + * + * @param databaseName the name of the database to be set as current + */ + void setCurrentDatabase(String databaseName); + + void setCurrentSchema(String schema); +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/ddl/DdlParserCallback.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/ddl/DdlParserCallback.java new file mode 100644 index 0000000000..b69fdcab67 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/ddl/DdlParserCallback.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.ddl; + +import org.apache.eventmesh.connector.jdbc.event.Event; + +/** + * Functional interface for a DDL parser listener. + */ +@FunctionalInterface +public interface DdlParserCallback { + + /** + * Handles the DDL event. + * + * @param event The DDL event to handle. + */ + void handle(Event event); + +} + diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/AbstractEvent.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/AbstractEvent.java new file mode 100644 index 0000000000..a5c7adf985 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/AbstractEvent.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.event; + +import org.apache.eventmesh.connector.jdbc.JdbcConnectData; +import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; + +public abstract class AbstractEvent implements Event { + + private final TableId tableId; + + private JdbcConnectData data; + + public AbstractEvent(TableId tableId) { + this.tableId = tableId; + this.data = new JdbcConnectData(); + } + + public AbstractEvent(TableId tableId, JdbcConnectData data) { + this.tableId = tableId; + this.data = data; + } + + /** + * Gets the table ID of the event. + * + * @return The table ID. + */ + @Override + public TableId getTableId() { + return tableId; + } + + /** + * @return + */ + @Override + public JdbcConnectData getJdbcConnectData() { + return data; + } +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/AlertDatabaseEvent.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/AlertDatabaseEvent.java new file mode 100644 index 0000000000..e9ed796c75 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/AlertDatabaseEvent.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.event; + + +import org.apache.eventmesh.connector.jdbc.JdbcConnectData; +import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; + +public class AlertDatabaseEvent extends GeneralSchemaChangeEvent { + + public AlertDatabaseEvent(TableId tableId) { + super(tableId); + } + + public AlertDatabaseEvent(TableId tableId, JdbcConnectData data) { + super(tableId, data); + } + + + @Override + public SchemaChangeEventType getSchemaChangeEventType() { + return SchemaChangeEventType.DATABASE_ALERT; + } +} + diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/AlertTableEvent.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/AlertTableEvent.java new file mode 100644 index 0000000000..9d7dbf9b30 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/AlertTableEvent.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.event; + + +import org.apache.eventmesh.connector.jdbc.JdbcConnectData; +import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; + +public class AlertTableEvent extends GeneralSchemaChangeEvent { + + public AlertTableEvent(TableId tableId) { + super(tableId); + } + + public AlertTableEvent(TableId tableId, JdbcConnectData data) { + super(tableId, data); + } + + + @Override + public SchemaChangeEventType getSchemaChangeEventType() { + return SchemaChangeEventType.TABLE_ALERT; + } +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/CreateDatabaseEvent.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/CreateDatabaseEvent.java new file mode 100644 index 0000000000..f733435b19 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/CreateDatabaseEvent.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.event; + + +import org.apache.eventmesh.connector.jdbc.JdbcConnectData; +import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; + +public class CreateDatabaseEvent extends GeneralSchemaChangeEvent { + + public CreateDatabaseEvent(TableId tableId) { + super(tableId); + } + + public CreateDatabaseEvent(TableId tableId, JdbcConnectData data) { + super(tableId, data); + } + + + @Override + public SchemaChangeEventType getSchemaChangeEventType() { + return SchemaChangeEventType.DATABASE_CREATE; + } +} + diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/CreateTableEvent.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/CreateTableEvent.java new file mode 100644 index 0000000000..d7dce51612 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/CreateTableEvent.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.event; + + +import org.apache.eventmesh.connector.jdbc.JdbcConnectData; +import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; + +public class CreateTableEvent extends GeneralSchemaChangeEvent { + + public CreateTableEvent(TableId tableId) { + super(tableId); + } + + public CreateTableEvent(TableId tableId, JdbcConnectData data) { + super(tableId, data); + } + + + @Override + public SchemaChangeEventType getSchemaChangeEventType() { + return SchemaChangeEventType.TABLE_CREATE; + } +} + diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/DataChangeEvent.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/DataChangeEvent.java new file mode 100644 index 0000000000..c1655c27f0 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/DataChangeEvent.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.event; + + +/** + * Represents a data change event, extending the common event interface. + */ +public interface DataChangeEvent extends Event { + + /** + * Gets the type of data change event. + * + * @return The data change event type. + */ + DataChangeEventType getDataChangeEventType(); + +} + diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/DataChangeEventType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/DataChangeEventType.java new file mode 100644 index 0000000000..be79cb3ebf --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/DataChangeEventType.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.event; + +public enum DataChangeEventType { + + INSERT("I"), UPDATE("U"), DELETE("D"); + private final String code; + + DataChangeEventType(String code) { + this.code = code; + } + + public String ofCode() { + return this.code; + } + +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/DeleteDataEvent.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/DeleteDataEvent.java new file mode 100644 index 0000000000..68f23ece8c --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/DeleteDataEvent.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.event; + + +import org.apache.eventmesh.connector.jdbc.JdbcConnectData; +import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; + +public class DeleteDataEvent extends GeneralDataChangeEvent { + + public DeleteDataEvent(TableId tableId) { + super(tableId); + } + + public DeleteDataEvent(TableId tableId, JdbcConnectData data) { + super(tableId, data); + } + + @Override + public DataChangeEventType getDataChangeEventType() { + return DataChangeEventType.INSERT; + } +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/DropDatabaseEvent.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/DropDatabaseEvent.java new file mode 100644 index 0000000000..1011216fab --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/DropDatabaseEvent.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.event; + + +import org.apache.eventmesh.connector.jdbc.JdbcConnectData; +import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; + +public class DropDatabaseEvent extends GeneralSchemaChangeEvent { + + public DropDatabaseEvent(TableId tableId) { + super(tableId); + } + + public DropDatabaseEvent(TableId tableId, JdbcConnectData data) { + super(tableId, data); + } + + + @Override + public SchemaChangeEventType getSchemaChangeEventType() { + return SchemaChangeEventType.DATABASE_DROP; + } +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/DropTableEvent.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/DropTableEvent.java new file mode 100644 index 0000000000..8629453f21 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/DropTableEvent.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.event; + + +import org.apache.eventmesh.connector.jdbc.JdbcConnectData; +import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; + +public class DropTableEvent extends GeneralSchemaChangeEvent { + + public DropTableEvent(TableId tableId) { + super(tableId); + } + + public DropTableEvent(TableId tableId, JdbcConnectData data) { + super(tableId, data); + } + + + @Override + public SchemaChangeEventType getSchemaChangeEventType() { + return SchemaChangeEventType.TABLE_DROP; + } +} + diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/Event.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/Event.java new file mode 100644 index 0000000000..7b254f960d --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/Event.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.event; + +import org.apache.eventmesh.connector.jdbc.JdbcConnectData; +import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; + +/** + * Top Event interface + */ +public interface Event { + + /** + * Gets the table ID of the event. + * + * @return The table ID. + */ + TableId getTableId(); + + JdbcConnectData getJdbcConnectData(); +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/EventConsumer.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/EventConsumer.java new file mode 100644 index 0000000000..9ad6d453c4 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/EventConsumer.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.event; + +/** + * Functional interface for consuming events. + */ +@FunctionalInterface +public interface EventConsumer { + + /** + * Accepts a snapshot event. + * + * @param event the snapshot event to be consumed + */ + void accept(Event event); +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/EventHandler.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/EventHandler.java new file mode 100644 index 0000000000..cf50dfefcd --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/EventHandler.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.eventmesh.connector.jdbc.event; + +/** + * Represents a handler for snapshot events. + */ +@FunctionalInterface +public interface EventHandler { + + /** + * Handles a snapshot event. + * + * @param event The SnapshotEvent to handle. + */ + void handle(Event event); +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/GeneralDataChangeEvent.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/GeneralDataChangeEvent.java new file mode 100644 index 0000000000..f7bcf24d37 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/GeneralDataChangeEvent.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.event; + +import org.apache.eventmesh.connector.jdbc.JdbcConnectData; +import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; + +public abstract class GeneralDataChangeEvent extends AbstractEvent implements DataChangeEvent { + + + public GeneralDataChangeEvent(TableId tableId) { + super(tableId); + } + + public GeneralDataChangeEvent(TableId tableId, JdbcConnectData data) { + super(tableId, data); + } +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/GeneralSchemaChangeEvent.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/GeneralSchemaChangeEvent.java new file mode 100644 index 0000000000..f07fee84b9 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/GeneralSchemaChangeEvent.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.event; + +import org.apache.eventmesh.connector.jdbc.JdbcConnectData; +import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; + +public abstract class GeneralSchemaChangeEvent extends AbstractEvent implements SchemaChangeEvent { + + public GeneralSchemaChangeEvent(TableId tableId) { + super(tableId); + } + + public GeneralSchemaChangeEvent(TableId tableId, JdbcConnectData data) { + super(tableId, data); + } +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/InsertDataEvent.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/InsertDataEvent.java new file mode 100644 index 0000000000..e339689f1b --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/InsertDataEvent.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.event; + + +import org.apache.eventmesh.connector.jdbc.JdbcConnectData; +import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; + +public class InsertDataEvent extends GeneralDataChangeEvent { + + public InsertDataEvent(TableId tableId) { + super(tableId); + } + + public InsertDataEvent(TableId tableId, JdbcConnectData data) { + super(tableId, data); + } + + @Override + public DataChangeEventType getDataChangeEventType() { + return DataChangeEventType.INSERT; + } +} + diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/SchemaChangeEvent.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/SchemaChangeEvent.java new file mode 100644 index 0000000000..ff1960837a --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/SchemaChangeEvent.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.event; + +/** + * Represents a schema change event, extending the common event interface. + */ +public interface SchemaChangeEvent extends Event { + + /** + * Gets the type of schema change event. + * + * @return The schema change event type. + */ + SchemaChangeEventType getSchemaChangeEventType(); +} + + diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/SchemaChangeEventType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/SchemaChangeEventType.java new file mode 100644 index 0000000000..fbf51ef2e8 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/SchemaChangeEventType.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.event; + +public enum SchemaChangeEventType { + + DATABASE_CREATE("D", "C"), + DATABASE_DROP("D", "D"), + DATABASE_ALERT("D", "A"), + TABLE_CREATE("T", "C"), + TABLE_DROP("T", "D"), + TABLE_ALERT("T", "A"), + ; + private final String type; + private final String operationType; + + SchemaChangeEventType(String type, String operationType) { + this.type = type; + this.operationType = operationType; + } + + public String ofType() { + return this.type; + } + + public String ofOperationType() { + return this.operationType; + } +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/UpdateDataEvent.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/UpdateDataEvent.java new file mode 100644 index 0000000000..dfa8990623 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/event/UpdateDataEvent.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.event; + + +import org.apache.eventmesh.connector.jdbc.JdbcConnectData; +import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; + +public class UpdateDataEvent extends GeneralDataChangeEvent { + + public UpdateDataEvent(TableId tableId) { + super(tableId); + } + + public UpdateDataEvent(TableId tableId, JdbcConnectData data) { + super(tableId, data); + } + + @Override + public DataChangeEventType getDataChangeEventType() { + return DataChangeEventType.INSERT; + } +} + diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/exception/CatalogException.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/exception/CatalogException.java new file mode 100644 index 0000000000..17fa873fbf --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/exception/CatalogException.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.exception; + +public class CatalogException extends RuntimeException { + + /** + * Constructs a new runtime exception with {@code null} as its detail message. The cause is not initialized, and may subsequently be initialized + * by a call to {@link #initCause}. + */ + public CatalogException() { + } + + /** + * Constructs a new runtime exception with the specified detail message. The cause is not initialized, and may subsequently be initialized by a + * call to {@link #initCause}. + * + * @param message the detail message. The detail message is saved for later retrieval by the {@link #getMessage()} method. + */ + public CatalogException(String message) { + super(message); + } + + /** + * Constructs a new runtime exception with the specified detail message and cause.

Note that the detail message associated with {@code cause} + * is + * not automatically incorporated in this runtime exception's detail message. + * + * @param message the detail message (which is saved for later retrieval by the {@link #getMessage()} method). + * @param cause the cause (which is saved for later retrieval by the {@link #getCause()} method). (A null value is permitted, and + * indicates that the cause is nonexistent or unknown.) + * @since 1.4 + */ + public CatalogException(String message, Throwable cause) { + super(message, cause); + } + + /** + * Constructs a new runtime exception with the specified cause and a detail message of (cause==null ? null : cause.toString()) (which + * typically contains the class and detail message of + * cause). This constructor is useful for runtime exceptions + * that are little more than wrappers for other throwables. + * + * @param cause the cause (which is saved for later retrieval by the {@link #getCause()} method). (A null value is permitted, and + * indicates that the cause is nonexistent or unknown.) + * @since 1.4 + */ + public CatalogException(Throwable cause) { + super(cause); + } + + /** + * Constructs a new runtime exception with the specified detail message, cause, suppression enabled or disabled, and writable stack trace enabled + * or disabled. + * + * @param message the detail message. + * @param cause the cause. (A {@code null} value is permitted, and indicates that the cause is nonexistent or unknown.) + * @param enableSuppression whether or not suppression is enabled or disabled + * @param writableStackTrace whether or not the stack trace should be writable + * @since 1.7 + */ + public CatalogException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/exception/DataTypeConvertException.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/exception/DataTypeConvertException.java new file mode 100644 index 0000000000..57b7d4f57b --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/exception/DataTypeConvertException.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.eventmesh.connector.jdbc.exception; + + +public class DataTypeConvertException extends RuntimeException { + + /** + * Constructs a new runtime exception with {@code null} as its detail message. The cause is not initialized, and may subsequently be initialized + * by a call to {@link #initCause}. + */ + public DataTypeConvertException() { + } + + /** + * Constructs a new runtime exception with the specified detail message. The cause is not initialized, and may subsequently be initialized by a + * call to {@link #initCause}. + * + * @param message the detail message. The detail message is saved for later retrieval by the {@link #getMessage()} method. + */ + public DataTypeConvertException(String message) { + super(message); + } + + /** + * Constructs a new runtime exception with the specified detail message and cause.

Note that the detail message associated with {@code cause} + * is + * not automatically incorporated in this runtime exception's detail message. + * + * @param message the detail message (which is saved for later retrieval by the {@link #getMessage()} method). + * @param cause the cause (which is saved for later retrieval by the {@link #getCause()} method). (A null value is permitted, and + * indicates that the cause is nonexistent or unknown.) + * @since 1.4 + */ + public DataTypeConvertException(String message, Throwable cause) { + super(message, cause); + } + + /** + * Constructs a new runtime exception with the specified cause and a detail message of (cause==null ? null : cause.toString()) (which + * typically contains the class and detail message of + * cause). This constructor is useful for runtime exceptions + * that are little more than wrappers for other throwables. + * + * @param cause the cause (which is saved for later retrieval by the {@link #getCause()} method). (A null value is permitted, and + * indicates that the cause is nonexistent or unknown.) + * @since 1.4 + */ + public DataTypeConvertException(Throwable cause) { + super(cause); + } + + /** + * Constructs a new runtime exception with the specified detail message, cause, suppression enabled or disabled, and writable stack trace enabled + * or disabled. + * + * @param message the detail message. + * @param cause the cause. (A {@code null} value is permitted, and indicates that the cause is nonexistent or unknown.) + * @param enableSuppression whether or not suppression is enabled or disabled + * @param writableStackTrace whether or not the stack trace should be writable + * @since 1.7 + */ + public DataTypeConvertException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/exception/DatabaseNotExistException.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/exception/DatabaseNotExistException.java new file mode 100644 index 0000000000..5c363d28f4 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/exception/DatabaseNotExistException.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.exception; + +public class DatabaseNotExistException extends RuntimeException { + + /** + * Constructs a new runtime exception with {@code null} as its detail message. The cause is not initialized, and may subsequently be initialized + * by a call to {@link #initCause}. + */ + public DatabaseNotExistException() { + } + + /** + * Constructs a new runtime exception with the specified detail message. The cause is not initialized, and may subsequently be initialized by a + * call to {@link #initCause}. + * + * @param message the detail message. The detail message is saved for later retrieval by the {@link #getMessage()} method. + */ + public DatabaseNotExistException(String message) { + super(message); + } + + /** + * Constructs a new runtime exception with the specified detail message and cause.

Note that the detail message associated with {@code cause} + * is + * not automatically incorporated in this runtime exception's detail message. + * + * @param message the detail message (which is saved for later retrieval by the {@link #getMessage()} method). + * @param cause the cause (which is saved for later retrieval by the {@link #getCause()} method). (A null value is permitted, and + * indicates that the cause is nonexistent or unknown.) + * @since 1.4 + */ + public DatabaseNotExistException(String message, Throwable cause) { + super(message, cause); + } + + /** + * Constructs a new runtime exception with the specified cause and a detail message of (cause==null ? null : cause.toString()) (which + * typically contains the class and detail message of + * cause). This constructor is useful for runtime exceptions + * that are little more than wrappers for other throwables. + * + * @param cause the cause (which is saved for later retrieval by the {@link #getCause()} method). (A null value is permitted, and + * indicates that the cause is nonexistent or unknown.) + * @since 1.4 + */ + public DatabaseNotExistException(Throwable cause) { + super(cause); + } + + /** + * Constructs a new runtime exception with the specified detail message, cause, suppression enabled or disabled, and writable stack trace enabled + * or disabled. + * + * @param message the detail message. + * @param cause the cause. (A {@code null} value is permitted, and indicates that the cause is nonexistent or unknown.) + * @param enableSuppression whether or not suppression is enabled or disabled + * @param writableStackTrace whether or not the stack trace should be writable + * @since 1.7 + */ + public DatabaseNotExistException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/exception/JdbcConnectionException.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/exception/JdbcConnectionException.java new file mode 100644 index 0000000000..dde57fc349 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/exception/JdbcConnectionException.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.exception; + +public class JdbcConnectionException extends RuntimeException { + +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/exception/TableNotExistException.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/exception/TableNotExistException.java new file mode 100644 index 0000000000..dc5a58dc03 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/exception/TableNotExistException.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.exception; + +public class TableNotExistException extends RuntimeException { + + /** + * Constructs a new runtime exception with {@code null} as its detail message. The cause is not initialized, and may subsequently be initialized + * by a call to {@link #initCause}. + */ + public TableNotExistException() { + } + + /** + * Constructs a new runtime exception with the specified detail message. The cause is not initialized, and may subsequently be initialized by a + * call to {@link #initCause}. + * + * @param message the detail message. The detail message is saved for later retrieval by the {@link #getMessage()} method. + */ + public TableNotExistException(String message) { + super(message); + } + + /** + * Constructs a new runtime exception with the specified detail message and cause.

Note that the detail message associated with {@code cause} + * is not automatically incorporated in this runtime exception's detail message. + * + * @param message the detail message (which is saved for later retrieval by the {@link #getMessage()} method). + * @param cause the cause (which is saved for later retrieval by the {@link #getCause()} method). (A null value is permitted, and + * indicates that the cause is nonexistent or unknown.) + * @since 1.4 + */ + public TableNotExistException(String message, Throwable cause) { + super(message, cause); + } + + /** + * Constructs a new runtime exception with the specified cause and a detail message of (cause==null ? null : cause.toString()) (which + * typically contains the class and detail message of + * cause). This constructor is useful for runtime exceptions + * that are little more than wrappers for other throwables. + * + * @param cause the cause (which is saved for later retrieval by the {@link #getCause()} method). (A null value is permitted, and + * indicates that the cause is nonexistent or unknown.) + * @since 1.4 + */ + public TableNotExistException(Throwable cause) { + super(cause); + } + + /** + * Constructs a new runtime exception with the specified detail message, cause, suppression enabled or disabled, and writable stack trace enabled + * or disabled. + * + * @param message the detail message. + * @param cause the cause. (A {@code null} value is permitted, and indicates that the cause is nonexistent or unknown.) + * @param enableSuppression whether or not suppression is enabled or disabled + * @param writableStackTrace whether or not the stack trace should be writable + * @since 1.7 + */ + public TableNotExistException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/server/JdbcConnectorServer.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/server/JdbcConnectorServer.java new file mode 100644 index 0000000000..546478ce93 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/server/JdbcConnectorServer.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.server; + +import org.apache.eventmesh.connector.jdbc.config.JdbcServerConfig; +import org.apache.eventmesh.connector.jdbc.source.JdbcSourceConnector; +import org.apache.eventmesh.openconnect.Application; +import org.apache.eventmesh.openconnect.util.ConfigUtil; + +/** + * JDBC connector server + */ +public class JdbcConnectorServer { + + public static void main(String[] args) throws Exception { + JdbcServerConfig serverConfig = ConfigUtil.parse(JdbcServerConfig.class, "server-config.yml"); + + if (serverConfig.isSourceEnable()) { + Application jdbcSourceApp = new Application(); + jdbcSourceApp.run(JdbcSourceConnector.class); + } + + if (serverConfig.isSinkEnable()) { + //TODO support + } + } + +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/AbstractEngine.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/AbstractEngine.java new file mode 100644 index 0000000000..2d1080e05b --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/AbstractEngine.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.source; + +import org.apache.eventmesh.common.ThreadWrapper; +import org.apache.eventmesh.connector.jdbc.DatabaseDialect; +import org.apache.eventmesh.connector.jdbc.source.config.JdbcSourceConfig; +import org.apache.eventmesh.connector.jdbc.source.config.SourceConnectorConfig; +import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; + +import org.apache.commons.collections4.CollectionUtils; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public abstract class AbstractEngine extends ThreadWrapper implements Engine { + + private final Set includeDatabaseTable = new HashSet<>(64); + + protected final JdbcSourceConfig jdbcSourceConfig; + + protected final SourceConnectorConfig sourceConnectorConfig; + + protected final DbDialect databaseDialect; + + public AbstractEngine(JdbcSourceConfig jdbcSourceConfig, DbDialect databaseDialect) { + this.jdbcSourceConfig = jdbcSourceConfig; + this.sourceConnectorConfig = this.jdbcSourceConfig.getSourceConnectorConfig(); + this.databaseDialect = databaseDialect; + + calculateNeedHandleTable(); + } + + @Override + public Set getHandledTables() { + return includeDatabaseTable; + } + + protected Set calculateNeedHandleTable() { + // Get the database and table include and exclude lists from the connector configuration + List databaseIncludeList = sourceConnectorConfig.getDatabaseIncludeList(); + + // If the database include list is empty, get a list of all databases and use that as the include list + if (CollectionUtils.isEmpty(databaseIncludeList)) { + List allDatabases = databaseDialect.listDatabases(); + databaseIncludeList = new ArrayList<>(allDatabases); + } + Set defaultExcludeDatabase = defaultExcludeDatabase(); + if (CollectionUtils.isNotEmpty(defaultExcludeDatabase)) { + databaseIncludeList.removeAll(defaultExcludeDatabase); + } + + List databaseExcludeList = sourceConnectorConfig.getDatabaseExcludeList(); + // Remove the default excluded databases from the include list + if (CollectionUtils.isNotEmpty(databaseExcludeList)) { + databaseIncludeList.removeAll(databaseExcludeList); + } + + List tableIncludeList = sourceConnectorConfig.getTableIncludeList(); + // Create a list of included tables based on the table include list + List includeTableList = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(tableIncludeList)) { + List tableIdList = buildTableId(tableIncludeList); + includeTableList.addAll(tableIdList); + } + + // If the table include list is empty, get a list of all tables for each database in the include list + if (CollectionUtils.isEmpty(tableIncludeList)) { + for (String database : databaseIncludeList) { + try { + List tableIds = databaseDialect.listTables(database); + includeTableList.addAll(tableIds); + } catch (SQLException e) { + log.warn("List database[{}] table error", database, e); + } + } + } + + List tableExcludeList = sourceConnectorConfig.getTableExcludeList(); + // Remove any tables in the exclude list from the included tables list + if (CollectionUtils.isNotEmpty(tableExcludeList)) { + includeTableList.removeAll(buildTableId(tableExcludeList)); + } + + includeDatabaseTable.addAll(includeTableList); + + return includeDatabaseTable; + } + + private List buildTableId(List tables) { + return Optional.ofNullable(tables).orElse(new ArrayList<>(0)).stream().map(table -> { + String[] split = table.split("\\."); + return new TableId(split[0], null, split[1]); + }).collect(Collectors.toList()); + } + + protected abstract Set defaultExcludeDatabase(); +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/AbstractEventMeshJdbcEventTask.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/AbstractEventMeshJdbcEventTask.java new file mode 100644 index 0000000000..3f7301fa7a --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/AbstractEventMeshJdbcEventTask.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.source; + +import org.apache.eventmesh.common.ThreadWrapper; +import org.apache.eventmesh.connector.jdbc.event.Event; + +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; + +public abstract class AbstractEventMeshJdbcEventTask extends ThreadWrapper implements EventMeshJdbcEventTask { + + protected BlockingQueue eventBlockingQueue = new LinkedBlockingQueue<>(10000); + + @Override + public void shutdown() { + super.shutdown(); + } + + @Override + public void close() throws Exception { + shutdown(); + } + + @Override + public void put(Event event) throws InterruptedException { + eventBlockingQueue.put(event); + } +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/AbstractEventMeshJdbcTask.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/AbstractEventMeshJdbcTask.java new file mode 100644 index 0000000000..f72aa2052e --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/AbstractEventMeshJdbcTask.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.source; + +import org.apache.eventmesh.common.ThreadWrapper; + +public abstract class AbstractEventMeshJdbcTask extends ThreadWrapper implements EventMeshJdbcTask { + + + @Override + public void shutdown() { + super.shutdown(); + } + + @Override + public void close() throws Exception { + shutdown(); + } + +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/AbstractJdbcTaskManager.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/AbstractJdbcTaskManager.java new file mode 100644 index 0000000000..a305bf53de --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/AbstractJdbcTaskManager.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.source; + + +import org.apache.eventmesh.connector.jdbc.source.config.JdbcSourceConfig; +import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +public abstract class AbstractJdbcTaskManager implements JdbcTaskManager { + + protected Map tableIdJdbcTaskMap = new ConcurrentHashMap<>(); + + protected Set includeDatabaseTable; + + protected JdbcSourceConfig jdbcSourceConfig; + + protected List taskList = new ArrayList<>(128); + + protected List listeners = new ArrayList<>(16); + + + @Override + public void start() { + taskList.forEach(EventMeshJdbcTask::start); + } + + @Override + public void shutdown() { + taskList.forEach(EventMeshJdbcTask::shutdown); + } + + @Override + public void close() throws Exception { + shutdown(); + } + + @Override + public void registerListener(TaskManagerListener listener) { + if (!Objects.isNull(listener)) { + listeners.add(listener); + } + } + + public abstract Task select(TableId tableId); +} \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/Engine.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/Engine.java new file mode 100644 index 0000000000..fa2cc73b4a --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/Engine.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.source; + +import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; + +import java.util.Set; + +/** + * Engine interface represents the core engine + */ +public interface Engine { + + /** + * Initializes the engine. + */ + void init(); + + /** + * Starts the engine. + */ + void start(); + + /** + * Retrieves the set of TableId objects representing the tables that the engine handles. + * + * @return The set of handled TableId objects. + */ + Set getHandledTables(); +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/EventDispatcher.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/EventDispatcher.java new file mode 100644 index 0000000000..17e59335eb --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/EventDispatcher.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.source; + +import org.apache.eventmesh.connector.jdbc.event.Event; +import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class EventDispatcher { + + private SourceJdbcTaskManager sourceJdbcTaskManager; + + public EventDispatcher(SourceJdbcTaskManager sourceJdbcTaskManager) { + this.sourceJdbcTaskManager = sourceJdbcTaskManager; + } + + /** + * Dispatch CDC events. + * + * @param event The CDC event to be dispatched. + */ + public void dispatch(Event event) { + TableId tableId = event.getTableId(); + SourceEventMeshJdbcEventTask task = sourceJdbcTaskManager.select(tableId); + try { + // Put the CDC event into the selected JDBC task. + task.put(event); + } catch (InterruptedException e) { + log.warn("Dispatch CdcEvent error", e); + } + } + +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/EventMeshJdbcEventTask.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/EventMeshJdbcEventTask.java new file mode 100644 index 0000000000..6b4c4d50dc --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/EventMeshJdbcEventTask.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.source; + +import org.apache.eventmesh.connector.jdbc.event.Event; +import org.apache.eventmesh.connector.jdbc.event.EventHandler; + +/** + * The EventMeshJdbcTask interface represents a task that interacts with the EventMesh through a JDBC connection. It extends the AutoCloseable + * interface, allowing the task to be managed efficiently. + */ +public interface EventMeshJdbcEventTask extends EventMeshJdbcTask { + + /** + * Puts an event into the task for processing. + * + * @param event The event to be processed. + * @throws InterruptedException If the operation is interrupted while waiting to put the event. + */ + void put(E event) throws InterruptedException; + + /** + * Registers a snapshot event handler to be executed when snapshot events occur. + * + * @param handler The SnapshotEventHandler to be registered. + */ + void registerEventHandler(EventHandler handler); + +} + diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/EventMeshJdbcTask.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/EventMeshJdbcTask.java new file mode 100644 index 0000000000..48830358b1 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/EventMeshJdbcTask.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.source; + +/** + * The EventMeshJdbcTask interface represents a task that interacts with the EventMesh through a JDBC connection. It extends the AutoCloseable + * interface, allowing the task to be managed efficiently. + */ +public interface EventMeshJdbcTask extends AutoCloseable { + + /** + * Starts the EventMesh JDBC task, initializing any necessary resources or connections. + */ + void start(); + + /** + * Shuts down the EventMesh JDBC task, releasing any acquired resources or connections. + */ + void shutdown(); + +} + diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/JdbcAllFactoryLoader.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/JdbcAllFactoryLoader.java new file mode 100644 index 0000000000..f3c09adb53 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/JdbcAllFactoryLoader.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.source; + +import static com.google.common.base.Preconditions.checkNotNull; + +import org.apache.eventmesh.connector.jdbc.source.dialect.DatabaseDialectFactory; +import org.apache.eventmesh.connector.jdbc.source.dialect.cdc.CdcEngineFactory; +import org.apache.eventmesh.connector.jdbc.source.dialect.snapshot.SnapshotEngineFactory; +import org.apache.eventmesh.spi.EventMeshExtensionFactory; + +import java.util.Objects; + +import lombok.experimental.UtilityClass; + +/** + * Get a CdcEngineFactory for a given database name + */ +@UtilityClass +public class JdbcAllFactoryLoader { + + /** + * Returns a CdcEngineFactory for the given database name. + *

Throws NullPointerException if databaseName is null.

+ *

Throws IllegalArgumentException if CdcEngineFactory is not supported for the given database name.

+ * + * @param databaseName Name of the database for which CdcEngineFactory is required. + * @return CdcEngineFactory for the given database name. + */ + public static CdcEngineFactory getCdcEngineFactory(String databaseName) { + checkNotNull(databaseName, "database name can not be null"); + CdcEngineFactory engineFactory = EventMeshExtensionFactory.getExtension(CdcEngineFactory.class, databaseName); + return checkNotNull(engineFactory, "CdcEngineFactory: " + databaseName + " is not supported"); + } + + /** + * Returns a DatabaseDialectFactory based on the specified database name. + * + * @param databaseName the name of the database + * @return the DatabaseDialectFactory for the specified database name + * @throws NullPointerException if the database name is null + * @throws IllegalArgumentException if the specified database name is not supported + */ + public static DatabaseDialectFactory getDatabaseDialectFactory(String databaseName) { + Objects.requireNonNull(databaseName, "database name can not be null"); + DatabaseDialectFactory databaseDialectFactory = EventMeshExtensionFactory.getExtension(DatabaseDialectFactory.class, databaseName); + Objects.requireNonNull(databaseDialectFactory, "DatabaseDialectFactory: " + databaseName + " is not supported"); + return databaseDialectFactory; + } + + public static SnapshotEngineFactory getSnapshotEngineFactory(String databaseName) { + Objects.requireNonNull(databaseName, "database name can not be null"); + SnapshotEngineFactory databaseDialectFactory = EventMeshExtensionFactory.getExtension(SnapshotEngineFactory.class, databaseName); + Objects.requireNonNull(databaseDialectFactory, "SnapshotEngineFactory: " + databaseName + " is not supported"); + return databaseDialectFactory; + } +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/JdbcSourceConnector.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/JdbcSourceConnector.java new file mode 100644 index 0000000000..d8147bf021 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/JdbcSourceConnector.java @@ -0,0 +1,207 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.source; + +import org.apache.eventmesh.connector.jdbc.DatabaseDialect; +import org.apache.eventmesh.connector.jdbc.event.Event; +import org.apache.eventmesh.connector.jdbc.source.config.JdbcSourceConfig; +import org.apache.eventmesh.connector.jdbc.source.dialect.DatabaseDialectFactory; +import org.apache.eventmesh.connector.jdbc.source.dialect.cdc.CdcEngine; +import org.apache.eventmesh.connector.jdbc.source.dialect.cdc.CdcEngineFactory; +import org.apache.eventmesh.connector.jdbc.source.dialect.snapshot.SnapshotEngine; +import org.apache.eventmesh.connector.jdbc.source.dialect.snapshot.SnapshotEngineFactory; +import org.apache.eventmesh.connector.jdbc.source.dialect.snapshot.SnapshotResult; +import org.apache.eventmesh.connector.jdbc.source.dialect.snapshot.SnapshotResult.SnapshotResultStatus; +import org.apache.eventmesh.openconnect.api.config.Config; +import org.apache.eventmesh.openconnect.api.config.SourceConfig; +import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; +import org.apache.eventmesh.openconnect.api.connector.SourceConnector; +import org.apache.eventmesh.openconnect.api.connector.SourceConnectorContext; +import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; + +import org.apache.commons.collections4.CollectionUtils; + +import java.util.List; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class JdbcSourceConnector extends SourceConnector { + + private DatabaseDialect databaseDialect; + + private CdcEngine cdcEngine; + + private JdbcSourceConfig sourceConfig; + + private EventDispatcher dispatcher; + + private SourceJdbcTaskManager sourceJdbcTaskManager; + + private SnapshotEngine snapshotEngine; + + private TaskManagerCoordinator taskManagerCoordinator; + + public JdbcSourceConnector() { + this(null); + } + + protected JdbcSourceConnector(SourceConfig sourceConfig) { + super(sourceConfig); + } + + /** + * Returns the class type of the configuration for this Connector. + * + * @return Class type of the configuration + */ + @Override + public Class configClass() { + return JdbcSourceConfig.class; + } + + /** + * Initializes the Connector with the provided configuration. + * + * @param config Configuration object + * @throws Exception if initialization fails + */ + @Override + public void init(Config config) throws Exception { + + if (!(config instanceof JdbcSourceConfig)) { + throw new IllegalArgumentException("Config not be JdbcSourceConfig"); + } + this.sourceConfig = (JdbcSourceConfig) config; + doInit(); + } + + /** + * Initializes the Connector with the provided context. + * + * @param connectorContext connectorContext + * @throws Exception if initialization fails + */ + @Override + public void init(ConnectorContext connectorContext) throws Exception { + SourceConnectorContext sourceConnectorContext = (SourceConnectorContext) connectorContext; + this.sourceConfig = (JdbcSourceConfig) sourceConnectorContext.getSourceConfig(); + doInit(); + } + + private void doInit() { + String databaseType = this.sourceConfig.getSourceConnectorConfig().getDatabaseType(); + + // Get the database dialect factory and create the database dialect. + final DatabaseDialectFactory databaseDialectFactory = JdbcAllFactoryLoader.getDatabaseDialectFactory(databaseType); + this.databaseDialect = databaseDialectFactory.createDatabaseDialect(sourceConfig); + this.databaseDialect.init(); + + // Get the snapshot engine factory and create the snapshot engine + final SnapshotEngineFactory snapshotEngineFactory = JdbcAllFactoryLoader.getSnapshotEngineFactory(databaseType); + this.snapshotEngine = snapshotEngineFactory.createSnapshotEngine(this.sourceConfig, this.databaseDialect); + this.snapshotEngine.registerSnapshotEventConsumer(this::eventConsumer); + this.snapshotEngine.init(); + + // Get the CDC engine factory and create the CDC engine. + final CdcEngineFactory cdcEngineFactory = JdbcAllFactoryLoader.getCdcEngineFactory(databaseType); + // Check if the CDC engine factory supports the JDBC protocol. + if (!cdcEngineFactory.acceptJdbcProtocol(this.databaseDialect.jdbcProtocol())) { + throw new IllegalArgumentException("CdcEngineFactory not supports " + databaseType); + } + // Set the CDC engine and register the CDC event consumer. + this.cdcEngine = cdcEngineFactory.createCdcEngine(this.sourceConfig, this.databaseDialect); + if (CollectionUtils.isEmpty(this.cdcEngine.getHandledTables())) { + throw new RuntimeException("No database tables need to be processed"); + } + this.cdcEngine.registerCdcEventConsumer(this::eventConsumer); + this.cdcEngine.init(); + + // Create the task manager and dispatcher. + this.sourceJdbcTaskManager = new SourceJdbcTaskManager(cdcEngine, this.sourceConfig); + this.sourceJdbcTaskManager.init(); + + this.dispatcher = new EventDispatcher(this.sourceJdbcTaskManager); + + this.taskManagerCoordinator = new TaskManagerCoordinator(); + this.taskManagerCoordinator.registerTaskManager(SourceJdbcTaskManager.class.getName(), sourceJdbcTaskManager); + this.taskManagerCoordinator.init(); + } + + private void eventConsumer(Event event) { + this.dispatcher.dispatch(event); + } + + /** + * Starts the Connector. + * + * @throws Exception if the start operation fails + */ + @Override + @SuppressWarnings("unchecked") + public void start() throws Exception { + this.databaseDialect.start(); + this.taskManagerCoordinator.start(); + this.snapshotEngine.start(); + SnapshotResult result = this.snapshotEngine.execute(); + this.snapshotEngine.close(); + //success and skip status can run cdc engine + if (result.getStatus() != SnapshotResultStatus.ABORTED) { + this.cdcEngine.setContext(result.getContext()); + this.cdcEngine.start(); + } + } + + /** + * Commits the specified ConnectRecord object. + * + * @param record ConnectRecord object to commit + */ + @Override + public void commit(ConnectRecord record) { + + } + + /** + * Returns the name of the Connector. + * + * @return String name of the Connector + */ + @Override + public String name() { + return "JDBC Source Connector"; + } + + /** + * Stops the Connector. + * + * @throws Exception if stopping fails + */ + @Override + public void stop() throws Exception { + + } + + @Override + public List poll() { + + List connectRecords = this.taskManagerCoordinator.poll(); + + return connectRecords; + } +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/JdbcTaskManager.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/JdbcTaskManager.java new file mode 100644 index 0000000000..7bef16d9b8 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/JdbcTaskManager.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.source; + +/** + * The JdbcTaskManager interface represents a manager for JDBC tasks. It extends the AutoCloseable interface, allowing the manager to be managed + * efficiently. + */ +public interface JdbcTaskManager extends AutoCloseable { + + /** + * Initializes the JDBC task manager, setting up any required configurations or resources. + */ + void init(); + + /** + * Starts the JDBC task manager, allowing it to begin managing tasks. + */ + void start(); + + /** + * Shuts down the JDBC task manager, releasing any acquired resources or stopping task management. + */ + void shutdown(); + + /** + * Registers a listener to receive events and notifications from the JDBC task manager. + * + * @param listener The listener to be registered. + */ + void registerListener(TaskManagerListener listener); + +} + diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/SourceEventMeshJdbcEventTask.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/SourceEventMeshJdbcEventTask.java new file mode 100644 index 0000000000..7c2b0d7173 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/SourceEventMeshJdbcEventTask.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.source; + +import org.apache.eventmesh.connector.jdbc.event.Event; +import org.apache.eventmesh.connector.jdbc.event.EventHandler; + +import java.util.Objects; +import java.util.concurrent.TimeUnit; + +public class SourceEventMeshJdbcEventTask extends AbstractEventMeshJdbcEventTask { + + + private final String taskName; + + private EventHandler eventHandler; + + public SourceEventMeshJdbcEventTask(String taskName) { + this.taskName = taskName; + + } + + @Override + public String getThreadName() { + return taskName; + } + + /** + * When an object implementing interface Runnable is used to create a thread, starting the thread causes the object's + * run method to be called in that separately executing + * thread. + *

+ * The general contract of the method run is that it may take any action whatsoever. + * + * @see Thread#run() + */ + @Override + public void run() { + while (isRunning) { + try { + Event snapshotEvent = eventBlockingQueue.poll(5, TimeUnit.SECONDS); + if (Objects.isNull(snapshotEvent)) { + continue; + } + eventHandler.handle(snapshotEvent); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + } + + + /** + * Registers a snapshot event handler to be executed when snapshot events occur. + * + * @param handler The SnapshotEventHandler to be registered. + */ + @Override + public void registerEventHandler(EventHandler handler) { + this.eventHandler = handler; + } +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/SourceJdbcTaskManager.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/SourceJdbcTaskManager.java new file mode 100644 index 0000000000..39243e3f98 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/SourceJdbcTaskManager.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.source; + +import org.apache.eventmesh.connector.jdbc.JdbcConnectData; +import org.apache.eventmesh.connector.jdbc.event.Event; +import org.apache.eventmesh.connector.jdbc.source.config.JdbcSourceConfig; +import org.apache.eventmesh.connector.jdbc.source.dialect.cdc.CdcEngine; +import org.apache.eventmesh.connector.jdbc.source.dialect.cdc.RandomTaskSelectStrategy; +import org.apache.eventmesh.connector.jdbc.source.dialect.cdc.TaskSelectStrategy; +import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; +import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; +import org.apache.eventmesh.openconnect.offsetmgmt.api.data.RecordOffset; +import org.apache.eventmesh.openconnect.offsetmgmt.api.data.RecordPartition; + +import org.apache.commons.collections4.CollectionUtils; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class SourceJdbcTaskManager extends AbstractJdbcTaskManager { + + private final Set includeDatabaseTable; + + private final JdbcSourceConfig jdbcSourceConfig; + + private TaskSelectStrategy cdcTaskSelectStrategy; + + public SourceJdbcTaskManager(CdcEngine cdcEngine, JdbcSourceConfig jdbcSourceConfig) { + this.jdbcSourceConfig = jdbcSourceConfig; + this.includeDatabaseTable = + CollectionUtils.isEmpty(cdcEngine.getHandledTables()) ? new HashSet<>() : new HashSet<>(cdcEngine.getHandledTables()); + } + + + @SuppressWarnings("unchecked") + public void init() { + //init Jdbc Task + int maxTaskNum = this.jdbcSourceConfig.getSourceConnectorConfig().getMaxTask(); + int taskNum = Math.min(maxTaskNum, this.includeDatabaseTable.size()); + log.info("Source jdbc task num {}", taskNum); + for (int index = 0; index < taskNum; ++index) { + SourceEventMeshJdbcEventTask eventTask = new SourceEventMeshJdbcEventTask("source-jdbc-task-" + (index + 1)); + eventTask.registerEventHandler(this::doHandleEvent); + taskList.add(eventTask); + } + cdcTaskSelectStrategy = new RandomTaskSelectStrategy(taskList); + + } + + private void doHandleEvent(Event event) { + + JdbcConnectData jdbcConnectData = event.getJdbcConnectData(); + RecordPartition partition = new RecordPartition(); + RecordOffset offset = new RecordOffset(); + ConnectRecord record = new ConnectRecord(partition, offset, System.currentTimeMillis(), jdbcConnectData); + if (null == record) { + return; + } + List records = Arrays.asList(record); + for (TaskManagerListener listener : listeners) { + listener.listen(records); + } + } + + + @Override + public SourceEventMeshJdbcEventTask select(TableId tableId) { + return tableIdJdbcTaskMap.computeIfAbsent(tableId, key -> cdcTaskSelectStrategy.select(tableId)); + } + + public int getTaskCount() { + return tableIdJdbcTaskMap.size(); + } + +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/SourceMateData.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/SourceMateData.java new file mode 100644 index 0000000000..732b125246 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/SourceMateData.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.source; + +import lombok.Data; + +/** + * Represents metadata related to a data source. + */ +@Data +public class SourceMateData { + + /** + * The connector used for the connector source. e.g: mysql, oracle etc. + */ + private String connector; + + /** + * The name of the connector source. + */ + private String name; + + /** + * The timestamp when the metadata was captured. + */ + private long timestamp; + + /** + * Flag indicating whether this metadata belongs to a snapshot. + */ + private boolean snapshot; + + /** + * The catalog name associated with the connector source. + */ + private String catalogName; + + /** + * The schema name associated with the connector source. + */ + private String schemaName; + + /** + * The table name associated with the connector source. + */ + private String tableName; + +} + diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/TaskManagerCoordinator.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/TaskManagerCoordinator.java new file mode 100644 index 0000000000..b22dafa8e8 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/TaskManagerCoordinator.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.eventmesh.connector.jdbc.source; + + +import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; + +import org.apache.commons.collections4.CollectionUtils; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; + +import lombok.extern.slf4j.Slf4j; + + +/** + * The TaskManagerCoordinator is responsible for coordinating multiple JDBC task managers and managing the processing of ConnectRecords. It provides + * methods for registering task managers, initializing them, and starting their processing. + */ +@Slf4j +public class TaskManagerCoordinator { + + private static final int BATCH_MAX = 10; + private static final int DEFAULT_QUEUE_SIZE = 1 << 13; + + private BlockingQueue recordBlockingQueue = new LinkedBlockingQueue<>(DEFAULT_QUEUE_SIZE); + private Map taskManagerCache = new HashMap<>(8); + + /** + * Constructs a new TaskManagerCoordinator. + */ + public TaskManagerCoordinator() { + } + + /** + * Registers a JDBC task manager with the given name. + * + * @param name The name of the task manager. + * @param taskManager The JDBC task manager to register. + */ + public void registerTaskManager(String name, JdbcTaskManager taskManager) { + taskManagerCache.put(name, taskManager); + } + + /** + * Initializes all registered JDBC task managers. + */ + public void init() { + taskManagerCache.values().forEach(JdbcTaskManager::init); + + // Register a listener on each task manager to process incoming records and add them to the blocking queue. + taskManagerCache.values().forEach(taskManager -> taskManager.registerListener(records -> { + if (CollectionUtils.isEmpty(records)) { + return; + } + records.forEach(record -> { + try { + recordBlockingQueue.put(record); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + }); + })); + } + + /** + * Starts the processing of all registered JDBC task managers. + */ + public void start() { + taskManagerCache.values().forEach(JdbcTaskManager::start); + } + + /** + * Polls for a batch of ConnectRecords from the blocking queue. + * + * @return A list of ConnectRecords, up to the maximum batch size defined by BATCH_MAX. + */ + public List poll() { + List records = new ArrayList<>(BATCH_MAX); + for (int index = 0; index < BATCH_MAX; ++index) { + try { + ConnectRecord record = recordBlockingQueue.poll(3, TimeUnit.SECONDS); + if (Objects.isNull(record)) { + break; + } + records.add(record); + } catch (InterruptedException e) { + break; + } + } + return records; + } +} + diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/TaskManagerListener.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/TaskManagerListener.java new file mode 100644 index 0000000000..abfbfcc5d6 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/TaskManagerListener.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.source; + +import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; + +import java.util.List; + +/** + * The TaskManagerListener is a functional interface used to listen for events from the TaskManager. It defines a single method, "listen", that takes + * a list of ConnectRecord objects as its parameter. + */ +@FunctionalInterface +public interface TaskManagerListener { + + /** + * Listens for events from the TaskManager and processes the given list of ConnectRecords. + * + * @param records The list of ConnectRecord objects to be processed. + */ + void listen(List records); + +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/config/JdbcSourceConfig.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/config/JdbcSourceConfig.java new file mode 100644 index 0000000000..b330c331bf --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/config/JdbcSourceConfig.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.source.config; + +import org.apache.eventmesh.openconnect.api.config.SourceConfig; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +@Data +@EqualsAndHashCode(callSuper = true) +public class JdbcSourceConfig extends SourceConfig { + + private SourceConnectorConfig sourceConnectorConfig; +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/config/MysqlConfig.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/config/MysqlConfig.java new file mode 100644 index 0000000000..032921350f --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/config/MysqlConfig.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.source.config; + +import lombok.Data; + +@Data +public class MysqlConfig { + + private int serverId; + + private boolean keepAlive = true; + + private long keepAliveInterval; + + private SnapshotLockingMode snapshotLockingMode = SnapshotLockingMode.MINIMAL; + + private boolean useGlobalLock = true; + + public enum SnapshotLockingMode { + + EXTENDED("extended"), + + MINIMAL("minimal"), + + NONE("none"); + + private final String value; + + SnapshotLockingMode(String value) { + this.value = value; + } + + public boolean usesLocking() { + return !value.equals(NONE.value); + } + } + +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/config/SourceConnectorConfig.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/config/SourceConnectorConfig.java new file mode 100644 index 0000000000..99210da8e8 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/config/SourceConnectorConfig.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.source.config; + +import org.apache.eventmesh.connector.jdbc.config.JdbcConfig; + +import java.util.List; + +import lombok.Data; + +/** + * Represents the configuration for a database connector. + */ +@Data +public class SourceConnectorConfig { + + private static final int DEFAULT_SNAPSHOT_FETCH_SIZE = 100; + + /** + * Max task number,The maximum cannot exceed the number of tables scanned. If it exceeds, it will be set to the number of tables. + */ + private int maxTask; + + private int batchMaxRows; + + private boolean skipSnapshot = false; + + // A list of database names to include in the connector. + private List databaseIncludeList; + + // A list of database names to exclude from the connector. + private List databaseExcludeList; + + // A list of table names to include in the connector. + private List tableIncludeList; + + // A list of table names to exclude from the connector. + private List tableExcludeList; + + // database type e.g. mysql + private String databaseType; + + private int snapshotMaxThreads; + + //The snapshot mode also require handling the database schema (database and table schema) + private boolean snapshotSchema = true; + + //The snapshot mode require handling table data + private boolean snapshotData = true; + + private int snapshotFetchSize = DEFAULT_SNAPSHOT_FETCH_SIZE; + + private JdbcConfig jdbcConfig; + + private boolean skipViews = false; + + private boolean skipComments = false; + + // The configuration for the MySQL database. + private MysqlConfig mysqlConfig; + + private String name = "mysql-connector"; +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/AbstractGeneralDatabaseDialect.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/AbstractGeneralDatabaseDialect.java new file mode 100644 index 0000000000..7041a06833 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/AbstractGeneralDatabaseDialect.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.source.dialect; + +import org.apache.eventmesh.connector.jdbc.DatabaseDialect; +import org.apache.eventmesh.connector.jdbc.connection.JdbcConnection; +import org.apache.eventmesh.connector.jdbc.exception.JdbcConnectionException; +import org.apache.eventmesh.connector.jdbc.source.config.SourceConnectorConfig; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; + + +public abstract class AbstractGeneralDatabaseDialect implements DatabaseDialect { + + private static final int DEFAULT_BATCH_MAX_ROWS = 20; + + private SourceConnectorConfig config; + + private int batchMaxRows = DEFAULT_BATCH_MAX_ROWS; + + public AbstractGeneralDatabaseDialect(SourceConnectorConfig config) { + this.config = config; + } + + @Override + public boolean isValid(Connection connection, int timeout) throws JdbcConnectionException, SQLException { + return connection == null ? false : connection.isValid(timeout); + } + + @Override + public PreparedStatement createPreparedStatement(Connection connection, String sql) throws SQLException { + PreparedStatement preparedStatement = connection.prepareStatement(sql); + if (batchMaxRows > 0) { + preparedStatement.setFetchSize(batchMaxRows); + } + return preparedStatement; + } +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/DatabaseDialectFactory.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/DatabaseDialectFactory.java new file mode 100644 index 0000000000..8f73f9f53e --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/DatabaseDialectFactory.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.source.dialect; + +import org.apache.eventmesh.connector.jdbc.DatabaseDialect; +import org.apache.eventmesh.openconnect.api.config.SourceConfig; +import org.apache.eventmesh.spi.EventMeshExtensionType; +import org.apache.eventmesh.spi.EventMeshSPI; + +/** + * Interface for creating a database dialect based on the provided source configuration. + */ +@EventMeshSPI(eventMeshExtensionType = EventMeshExtensionType.JDBC_DATABASE_DIALECT) +public interface DatabaseDialectFactory { + + /** + * Creates a database dialect based on the provided source configuration. + * + * @param config the source configuration to create a database dialect for + * @return the created database dialect + */ + DatabaseDialect createDatabaseDialect(SourceConfig config); + +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/DatabaseType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/DatabaseType.java new file mode 100644 index 0000000000..fa5fc90604 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/DatabaseType.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.source.dialect; + +import org.apache.commons.lang3.StringUtils; + +public enum DatabaseType { + MYSQL("mysql"); + + private String name; + + DatabaseType(String name) { + this.name = name; + } + + public static DatabaseType ofValue(String name) { + DatabaseType[] databaseTypes = values(); + for (DatabaseType databaseType : databaseTypes) { + if (StringUtils.equalsIgnoreCase(databaseType.name, name)) { + return databaseType; + } + } + return null; + } +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/AbstractCdcEngine.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/AbstractCdcEngine.java new file mode 100644 index 0000000000..c164cb7ee9 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/AbstractCdcEngine.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.source.dialect.cdc; + + +import org.apache.eventmesh.common.ThreadWrapper; +import org.apache.eventmesh.connector.jdbc.DatabaseDialect; +import org.apache.eventmesh.connector.jdbc.JdbcContext; +import org.apache.eventmesh.connector.jdbc.ddl.DdlParser; +import org.apache.eventmesh.connector.jdbc.source.config.JdbcSourceConfig; +import org.apache.eventmesh.connector.jdbc.source.config.SourceConnectorConfig; +import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; +import org.apache.eventmesh.openconnect.api.config.SourceConfig; + +import org.apache.commons.collections4.CollectionUtils; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public abstract class AbstractCdcEngine extends + ThreadWrapper implements CdcEngine { + + protected final JdbcSourceConfig jdbcSourceConfig; + + protected final DbDialect databaseDialect; + + protected final SourceConnectorConfig sourceConnectorConfig; + + private final Set includeDatabaseTable = new HashSet<>(64); + + public AbstractCdcEngine(SourceConfig config, DbDialect databaseDialect) { + if (!(config instanceof JdbcSourceConfig)) { + throw new IllegalArgumentException("config "); + } + this.jdbcSourceConfig = (JdbcSourceConfig) config; + this.databaseDialect = databaseDialect; + this.sourceConnectorConfig = this.jdbcSourceConfig.getSourceConnectorConfig(); + + calculateNeedHandleTable(); + } + + @Override + public Set getHandledTables() { + return includeDatabaseTable; + } + + + protected Set calculateNeedHandleTable() { + // Get the database and table include and exclude lists from the connector configuration + List databaseIncludeList = sourceConnectorConfig.getDatabaseIncludeList(); + + // If the database include list is empty, get a list of all databases and use that as the include list + if (CollectionUtils.isEmpty(databaseIncludeList)) { + List allDatabases = databaseDialect.listDatabases(); + databaseIncludeList = new ArrayList<>(allDatabases); + } + Set defaultExcludeDatabase = defaultExcludeDatabase(); + if (CollectionUtils.isNotEmpty(defaultExcludeDatabase)) { + databaseIncludeList.removeAll(defaultExcludeDatabase); + } + + List databaseExcludeList = sourceConnectorConfig.getDatabaseExcludeList(); + // Remove the default excluded databases from the include list + if (CollectionUtils.isNotEmpty(databaseExcludeList)) { + databaseIncludeList.removeAll(databaseExcludeList); + } + + List tableIncludeList = sourceConnectorConfig.getTableIncludeList(); + // Create a list of included tables based on the table include list + List includeTableList = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(tableIncludeList)) { + List tableIdList = buildTableId(tableIncludeList); + includeTableList.addAll(tableIdList); + } + + // If the table include list is empty, get a list of all tables for each database in the include list + if (CollectionUtils.isEmpty(tableIncludeList)) { + for (String database : databaseIncludeList) { + try { + List tableIds = databaseDialect.listTables(database); + includeTableList.addAll(tableIds); + } catch (SQLException e) { + log.warn("List database[{}] table error", database, e); + } + } + } + + List tableExcludeList = sourceConnectorConfig.getTableExcludeList(); + // Remove any tables in the exclude list from the included tables list + if (CollectionUtils.isNotEmpty(tableExcludeList)) { + includeTableList.removeAll(buildTableId(tableExcludeList)); + } + + includeDatabaseTable.addAll(includeTableList); + + return includeDatabaseTable; + } + + + private List buildTableId(List tables) { + return Optional.ofNullable(tables).orElse(new ArrayList<>(0)).stream().map(table -> { + String[] split = table.split("\\."); + return new TableId(split[0], null, split[1]); + }).collect(Collectors.toList()); + } + + protected abstract Set defaultExcludeDatabase(); + + protected abstract Parse getDdlParser(); +} + diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/AbstractCdcEngineFactory.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/AbstractCdcEngineFactory.java new file mode 100644 index 0000000000..d00d4c56fa --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/AbstractCdcEngineFactory.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.source.dialect.cdc; + +public abstract class AbstractCdcEngineFactory implements CdcEngineFactory { + +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/CdcEngine.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/CdcEngine.java new file mode 100644 index 0000000000..566ffccf4c --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/CdcEngine.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.source.dialect.cdc; + +import org.apache.eventmesh.connector.jdbc.JdbcContext; +import org.apache.eventmesh.connector.jdbc.event.EventConsumer; +import org.apache.eventmesh.connector.jdbc.source.Engine; + +/** + * CdcEngine is a service that captures data change events. + */ +public interface CdcEngine extends Engine, AutoCloseable { + + /** + * Stops the CDC Engine. + */ + @Override + void close() throws Exception; + + /** + * Returns the name of the CDC Engine. + * + * @return String representing the name of the CDC Engine. + */ + String getCdcEngineName(); + + /** + * Registers the CDC event consumer. + * + * @param consumer The CDC event consumer to register. + */ + void registerCdcEventConsumer(EventConsumer consumer); + + void setContext(Context context); +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/CdcEngineFactory.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/CdcEngineFactory.java new file mode 100644 index 0000000000..531bd3f2bc --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/CdcEngineFactory.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.source.dialect.cdc; + + +import org.apache.eventmesh.connector.jdbc.DatabaseDialect; +import org.apache.eventmesh.openconnect.api.config.SourceConfig; +import org.apache.eventmesh.spi.EventMeshExtensionType; +import org.apache.eventmesh.spi.EventMeshSPI; + +/** + * This interface defines the methods required to create a Change Data Capture (CDC) engine + */ +@EventMeshSPI(eventMeshExtensionType = EventMeshExtensionType.JDBC_CDC_ENGINE) +public interface CdcEngineFactory { + + /** + * Determines whether the provided JDBC URL is compatible with the CDC engine + * + * @param url jdbc url, e.g. mysql: jdbc:mysql://localhost:3306/ + * @return true if the JDBC URL is compatible with the CDC engine, false otherwise + */ + boolean acceptJdbcProtocol(String url); + + /** + * Creates a CDC engine based on the provided source configuration and database dialect. + * + * @param config the source configuration for the CDC engine + * @param databaseDialect the database dialect for the CDC engine + * @return the created CDC engine + */ + CdcEngine createCdcEngine(SourceConfig config, DatabaseDialect databaseDialect); + +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/RandomTaskSelectStrategy.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/RandomTaskSelectStrategy.java new file mode 100644 index 0000000000..4b05d41b31 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/RandomTaskSelectStrategy.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.source.dialect.cdc; + +import org.apache.eventmesh.connector.jdbc.source.EventMeshJdbcTask; +import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; + +import java.util.List; +import java.util.Random; + +public class RandomTaskSelectStrategy implements TaskSelectStrategy { + + private List cdcTasks; + + public RandomTaskSelectStrategy(List cdcTasks) { + this.cdcTasks = cdcTasks; + } + + /** + * Selects a JdbcTask for the specified TableId. + * + * @param tableId the TableId for which to select a JdbcTask + * @return the selected JdbcTask + */ + @Override + public Task select(TableId tableId) { + Random random = new Random(System.currentTimeMillis()); + int randomNum = random.nextInt(cdcTasks.size()); + return cdcTasks.get(randomNum); + } +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/TaskSelectStrategy.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/TaskSelectStrategy.java new file mode 100644 index 0000000000..62e04c0507 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/TaskSelectStrategy.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.source.dialect.cdc; + +import org.apache.eventmesh.connector.jdbc.source.EventMeshJdbcTask; +import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; + +/** + * Represents a strategy for selecting a CdcTask for a given TableId. + */ +public interface TaskSelectStrategy { + + /** + * Selects a CdcTask for the specified TableId. + * + * @param tableId the TableId for which to select a CdcTask + * @return the selected CdcTask + */ + Task select(TableId tableId); + +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/AbstractSnapshotEngine.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/AbstractSnapshotEngine.java new file mode 100644 index 0000000000..49c1f9c707 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/AbstractSnapshotEngine.java @@ -0,0 +1,343 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.source.dialect.snapshot; + +import org.apache.eventmesh.common.ThreadPoolFactory; +import org.apache.eventmesh.connector.jdbc.DatabaseDialect; +import org.apache.eventmesh.connector.jdbc.JdbcContext; +import org.apache.eventmesh.connector.jdbc.OffsetContext; +import org.apache.eventmesh.connector.jdbc.Partition; +import org.apache.eventmesh.connector.jdbc.connection.JdbcConnection; +import org.apache.eventmesh.connector.jdbc.event.Event; +import org.apache.eventmesh.connector.jdbc.source.AbstractEngine; +import org.apache.eventmesh.connector.jdbc.source.config.JdbcSourceConfig; +import org.apache.eventmesh.connector.jdbc.source.dialect.snapshot.SnapshotResult.SnapshotResultStatus; +import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; +import org.apache.eventmesh.connector.jdbc.table.type.EventMeshRow; + +import org.apache.commons.collections4.CollectionUtils; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.HashSet; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.Queue; +import java.util.Set; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Callable; +import java.util.concurrent.CompletionService; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ExecutorCompletionService; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; + +import lombok.extern.slf4j.Slf4j; + +/** + * Abstract base class for snapshot engines. + * + * @param The Database Dialect + * @param The JDBC context type + * @param The partition type + * @param The offset context type + */ +@Slf4j +public abstract class AbstractSnapshotEngine, Jc extends JdbcContext, Part extends Partition, + Offset extends OffsetContext, Jconn extends JdbcConnection> extends AbstractEngine implements SnapshotEngine { + + private final Jconn jdbcConnection; + + private Jc context; + + private Part partition; + + private Offset offsetContext; + + protected BlockingQueue eventQueue = new LinkedBlockingQueue<>(10000); + + public AbstractSnapshotEngine(JdbcSourceConfig jdbcSourceConfig, DbDialect databaseDialect, Jc jdbcContext, Part partition, + Offset context) { + super(jdbcSourceConfig, databaseDialect); + this.context = jdbcContext; + this.partition = partition; + this.offsetContext = context; + this.jdbcConnection = databaseDialect.getConnection(); + } + + + @Override + public SnapshotResult execute() { + if (jdbcSourceConfig.getSourceConnectorConfig().isSkipSnapshot()) { + return new SnapshotResult<>(SnapshotResultStatus.SKIPPED, null); + } + SnapshotContext snapshotContext = new SnapshotContext<>(partition, offsetContext); + return doExecute(context, snapshotContext); + } + + + /** + * Template method that executes the snapshot logic. + * + * @param context the JDBC context + * @param snapshotContext the snapshot context + * @return the snapshot result + */ + protected SnapshotResult doExecute(Jc context, SnapshotContext snapshotContext) { + + Connection masterConnection = null; + Queue connectionPool = null; + try { + masterConnection = createMasterConnection(); + log.info("Snapshot 1: Preparations for Snapshot Work"); + preSnapshot(context, snapshotContext); + + log.info("Snapshot 2: Retrieve tables requiring snapshot handling"); + determineTable2Process(context, snapshotContext); + + log.info("Snapshot 3: Put locks on the tables that need to be processed"); + if (sourceConnectorConfig.isSnapshotSchema()) { + lockTables4SchemaSnapshot(context, snapshotContext); + } + + log.info("Snapshot 4: Determining snapshot offset"); + determineSnapshotOffset(context, snapshotContext); + + log.info("Snapshot 5: Obtain the schema of the captured tables"); + readStructureOfTables(context, snapshotContext); + + //Release locks + releaseSnapshotLocks(context, snapshotContext); + + //Whether to determine whether to process the table data? + if (sourceConnectorConfig.isSnapshotData()) { + connectionPool = createConnectionPool(snapshotContext); + createDataEvents(context, snapshotContext, connectionPool); + } + log.info("Snapshot 6: Release the locks"); + releaseSnapshotLocks(context, snapshotContext); + } catch (Exception e) { + throw new RuntimeException(e); + } finally { + //close connection pool's connection + try { + if (CollectionUtils.isNotEmpty(connectionPool)) { + for (JdbcConnection conn : connectionPool) { + conn.close(); + } + } + //Roll back master connection transaction + rollbackMasterConnTransaction(masterConnection); + } catch (Exception e) { + log.warn("Handle snapshot finally error", e); + } + } + return new SnapshotResult<>(SnapshotResultStatus.COMPLETED, context); + } + + private void rollbackMasterConnTransaction(Connection connection) { + if (connection != null) { + try { + connection.rollback(); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + } + + private Connection createMasterConnection() throws SQLException { + JdbcConnection connection = databaseDialect.getConnection(); + connection.setAutoCommit(false); + return connection.connection(); + } + + /** + * Creates data events. + * + * @param context The context. + * @param snapshotContext The snapshot context. + * @param connectionPool The connection pool. + * @throws SQLException If an error occurs. + */ + private void createDataEvents(Jc context, SnapshotContext snapshotContext, Queue connectionPool) + throws Exception { + + int handleDataThreadNum = connectionPool.size(); + //Create thread pool to process table data + ThreadPoolExecutor tableDataPoolExecutor = ThreadPoolFactory.createThreadPoolExecutor(handleDataThreadNum, handleDataThreadNum, + "snapshot-table-data-thread"); + CompletionService completionService = new ExecutorCompletionService<>(tableDataPoolExecutor); + + try { + for (TableId tableId : snapshotContext.determineTables) { + String sql = getSnapshotTableSelectSql(context, snapshotContext, tableId).get(); + Callable callable = createSnapshotDataEvent4TableCallable(context, snapshotContext, connectionPool, sql, tableId); + completionService.submit(callable); + } + int tableSize = snapshotContext.determineTables.size(); + for (int index = 0; index < tableSize; ++index) { + completionService.take().get(); + } + } finally { + tableDataPoolExecutor.shutdownNow(); + } + } + + private Callable createSnapshotDataEvent4TableCallable(Jc context, SnapshotContext snapshotContext, + Queue connectionPool, String sql, TableId tableId) { + return () -> { + JdbcConnection connection = connectionPool.poll(); + try (Statement statement = connection.createStatement(jdbcSourceConfig.getSourceConnectorConfig().getSnapshotFetchSize(), 100)) { + ResultSet resultSet = statement.executeQuery(sql); + while (resultSet.next()) { + int columnCount = resultSet.getMetaData().getColumnCount(); + EventMeshRow row = new EventMeshRow(columnCount); + row.setTableId(tableId); + Object[] values = new Object[columnCount]; + for (int index = 1; index <= columnCount; ++index) { + values[index - 1] = resultSet.getObject(index); + } + row.setFieldValues(values); + //TableDataSnapshotEvent event = new TableDataSnapshotEvent(SnapshotEventType.TABLE_DATA, tableId, row); + // eventQueue.put(event); + } + } finally { + connectionPool.add(connection); + } + return null; + }; + } + + protected OptionalLong getRowCount4Table(TableId tableId) { + return OptionalLong.empty(); + } + + private Queue createConnectionPool(final SnapshotContext snapshotContext) throws SQLException { + Queue connectionPool = new ConcurrentLinkedQueue<>(); + int snapshotMaxThreads = Math.max(1, + Math.min(this.jdbcSourceConfig.getSourceConnectorConfig().getSnapshotMaxThreads(), snapshotContext.determineTables.size())); + + for (int i = 0; i < snapshotMaxThreads; i++) { + JdbcConnection conn = databaseDialect.newConnection().setAutoCommit(false); + //Get transaction isolation from master connection and then set to connection of pool + conn.connection().setTransactionIsolation(jdbcConnection.connection().getTransactionIsolation()); + connectionPool.add(conn); + } + + log.info("Created connection pool with {} number", snapshotMaxThreads); + return connectionPool; + } + + /** + * Pre-snapshot preparations. + * + * @param jdbcContext the JDBC context + * @param snapshotContext the snapshot context + */ + protected abstract void preSnapshot(Jc jdbcContext, SnapshotContext snapshotContext); + + /** + * Determine tables that need snapshotting. + * + * @param jdbcContext the JDBC context + * @param snapshotContext the snapshot context + */ + protected abstract void determineTable2Process(Jc jdbcContext, SnapshotContext snapshotContext); + + /** + * Lock tables for consistent snapshot schema. + * + * @param jdbcContext the JDBC context + * @param snapshotContext the snapshot context + * @throws SQLException if a database error occurs + */ + protected abstract void lockTables4SchemaSnapshot(Jc jdbcContext, SnapshotContext snapshotContext) throws SQLException; + + /** + * Determine snapshot offset. + * + * @param jdbcContext the JDBC context + * @param snapshotContext the snapshot context + * @throws SQLException if a database error occurs + */ + protected abstract void determineSnapshotOffset(Jc jdbcContext, SnapshotContext snapshotContext) throws SQLException; + + /** + * Read and store table schemas. + * + * @param jdbcContext the JDBC context + * @param snapshotContext the snapshot context + * @throws SQLException if a database error occurs + * @throws InterruptedException if interrupted + */ + protected abstract void readStructureOfTables(Jc jdbcContext, SnapshotContext snapshotContext) + throws SQLException, InterruptedException; + + /** + * Release locks after snapshot. + * + * @param jdbcContext the JDBC context + * @param snapshotContext the snapshot context + */ + protected abstract void releaseSnapshotLocks(Jc jdbcContext, SnapshotContext snapshotContext) throws Exception; + + protected abstract Optional getSnapshotTableSelectSql(Jc jdbcContext, SnapshotContext snapshotContext, TableId tableId); + + public static class SnapshotContext

implements AutoCloseable { + + protected P partition; + + protected O offset; + + protected Set determineTables = new HashSet<>(); + + public SnapshotContext(P partition, O offset) { + this.partition = partition; + this.offset = offset; + } + + @Override + public void close() throws Exception { + + } + + public void add(TableId tableId) { + SnapshotContext.this.determineTables.add(tableId); + } + + public void addAll(Set tableIds) { + if (tableIds != null) { + SnapshotContext.this.determineTables.addAll(tableIds); + } + } + + public P getPartition() { + return partition; + } + + public O getOffset() { + return offset; + } + + public Set getDetermineTables() { + return determineTables; + } + } +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/SnapshotEngine.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/SnapshotEngine.java new file mode 100644 index 0000000000..25cc19676c --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/SnapshotEngine.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.source.dialect.snapshot; + +import org.apache.eventmesh.connector.jdbc.JdbcContext; +import org.apache.eventmesh.connector.jdbc.event.EventConsumer; +import org.apache.eventmesh.connector.jdbc.source.Engine; + +/** + * The SnapshotEngine interface extends the Engine interface and represents an engine capable of performing snapshots. + * + * @param The type of JdbcContext used for the snapshot. + */ +public interface SnapshotEngine extends Engine, AutoCloseable { + + /** + * Executes the snapshot operation and returns the result containing the snapshot offset. + * + * @return The SnapshotResult containing the snapshot offset. + */ + SnapshotResult execute(); + + /** + * Registers a SnapshotEventConsumer to receive snapshot events from the engine. + * + * @param consumer The SnapshotEventConsumer to register. + */ + void registerSnapshotEventConsumer(EventConsumer consumer); +} + diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/SnapshotEngineFactory.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/SnapshotEngineFactory.java new file mode 100644 index 0000000000..33d756ba51 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/SnapshotEngineFactory.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.source.dialect.snapshot; + +import org.apache.eventmesh.connector.jdbc.DatabaseDialect; +import org.apache.eventmesh.connector.jdbc.JdbcContext; +import org.apache.eventmesh.connector.jdbc.source.config.JdbcSourceConfig; +import org.apache.eventmesh.spi.EventMeshExtensionType; +import org.apache.eventmesh.spi.EventMeshSPI; + +/** + * The SnapshotEngineFactory interface represents a factory for creating snapshot engines. It provides a method to create a snapshot engine based on + * the given configuration and database dialect. + */ +@EventMeshSPI(eventMeshExtensionType = EventMeshExtensionType.JDBC_SNAPSHOT_ENGINE) +public interface SnapshotEngineFactory { + + /** + * Creates a snapshot engine with the specified JDBC source configuration and database dialect. + * + * @param jdbcSourceConfig The JDBC source configuration. + * @param databaseDialect The database dialect. + * @return A snapshot engine that can perform snapshot operations. + */ + SnapshotEngine createSnapshotEngine(final JdbcSourceConfig jdbcSourceConfig, + final DatabaseDialect databaseDialect); +} + diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/SnapshotResult.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/SnapshotResult.java new file mode 100644 index 0000000000..d36c85468c --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/SnapshotResult.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.source.dialect.snapshot; + +import org.apache.eventmesh.connector.jdbc.JdbcContext; + +import lombok.Getter; + +@Getter +public class SnapshotResult { + + private final SnapshotResultStatus status; + + private final Jc context; + + public SnapshotResult(SnapshotResultStatus status, Jc jc) { + this.status = status; + this.context = jc; + } + + public enum SnapshotResultStatus { + COMPLETED, + ABORTED, + SKIPPED + } +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/SnapshotType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/SnapshotType.java new file mode 100644 index 0000000000..426835bb93 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/SnapshotType.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.source.dialect.snapshot; + +public enum SnapshotType { + + /** + * Every time the service starts, it reads the data of the tables that need to be processed. + */ + INITIALIZATION, + + /** + * Continue processing from the last handled position. + */ + INCREASE //TODO Need to support next version +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/Catalog.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/Catalog.java new file mode 100644 index 0000000000..80f384429c --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/Catalog.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.table.catalog; + +import org.apache.eventmesh.connector.jdbc.exception.CatalogException; +import org.apache.eventmesh.connector.jdbc.exception.DatabaseNotExistException; +import org.apache.eventmesh.connector.jdbc.exception.TableNotExistException; + +import java.sql.SQLException; +import java.util.List; + +/** + * Interacting with a catalog of databases and tables. + */ +public interface Catalog extends AutoCloseable { + + /** + * Opens the catalog. + * + * @throws CatalogException if there is an error opening the catalog. + */ + void open() throws CatalogException; + + /** + * Gets the name of the default database. + * + * @return the name of the default database. + */ + String getDefaultDatabase(); + + /** + * Checks if a database with the given name exists. + * + * @param databaseName the name of the database to check. + * @return true if the database exists, false otherwise. + * @throws CatalogException if there is an error checking for the database. + */ + boolean databaseExists(String databaseName) throws CatalogException; + + /** + * Gets a list of all databases in the catalog. + * + * @return a list of all databases in the catalog. + * @throws CatalogException if there is an error getting the list of databases. + */ + List listDatabases() throws CatalogException; + + /** + * Gets a list of all tables in the given database. + * + * @param databaseName the name of the database to get the tables for. + * @return a list of all tables in the given database. + * @throws CatalogException if there is an error getting the list of tables. + * @throws DatabaseNotExistException if the database does not exist. + * @throws SQLException if there is an error accessing the database. + */ + List listTables(String databaseName) throws CatalogException, DatabaseNotExistException, SQLException; + + /** + * Checks if a table with the given ID exists. + * + * @param tableId the ID of the table to check. + * @return true if the table exists, false otherwise. + * @throws CatalogException if there is an error checking for the table. + * @throws SQLException if there is an error accessing the database. + */ + boolean tableExists(TableId tableId) throws CatalogException, SQLException; + + /** + * Gets the table with the given ID. + * + * @param tableId the ID of the table to get. + * @return the table with the given ID. + * @throws CatalogException if there is an error getting the table. + * @throws TableNotExistException if the table does not exist. + * @throws SQLException if there is an error accessing the database. + */ + CatalogTable getTable(TableId tableId) throws CatalogException, TableNotExistException, SQLException; + + //TODO: support create table, drop table and update table +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/CatalogSchema.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/CatalogSchema.java new file mode 100644 index 0000000000..2caae2a841 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/CatalogSchema.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.table.catalog; + +import java.io.Serializable; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * Represents catalog schema information, including its name and character set. + */ +@Data +@NoArgsConstructor +@AllArgsConstructor +public class CatalogSchema implements Serializable { + + /** + * The name of the catalog schema. + */ + private String name; + + /** + * The character set used by the catalog schema. + */ + private String characterSet; + + public CatalogSchema(String name) { + this.name = name; + } +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/CatalogTable.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/CatalogTable.java new file mode 100644 index 0000000000..4aa2b89351 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/CatalogTable.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.table.catalog; + +import java.io.Serializable; + +import lombok.Data; + +/** + * Represents a catalog table. + */ +@Data +public class CatalogTable implements Serializable { + + private static final long serialVersionUID = -9159821671858779282L; + + // The ID of the table. + private TableId tableId; + + // The schema of the table. + private TableSchema tableSchema; + + // A comment describing the table. + private String comment; + + private Options options; + +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/CatalogTableSet.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/CatalogTableSet.java new file mode 100644 index 0000000000..309ab6c0d2 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/CatalogTableSet.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.table.catalog; + +import org.apache.commons.lang3.StringUtils; + +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +public final class CatalogTableSet { + + private final TableIdSet tableIdSet; + + private final TableSchemaMap tableSchemaMap; + + public CatalogTableSet() { + + this.tableIdSet = new TableIdSet(); + this.tableSchemaMap = new TableSchemaMap(); + + } + + public static TableEditor ofCatalogTableEditor(TableId tableId) { + return new DefaultTableEditorImpl(tableId); + } + + public void removeDatabase(String catalogName) { + removeDatabase(catalogName, null); + } + + public void removeDatabase(String catalogName, String schemaName) { + tableSchemaMap.removeDatabase(catalogName, schemaName); + tableIdSet.removeDatabase(catalogName, schemaName); + } + + private static class TableIdSet { + + private final Set values; + + public TableIdSet() { + values = new HashSet<>(32); + } + + public void removeDatabase(String catalogName, String schemaName) { + values.removeIf(entry -> { + return StringUtils.equals(entry.getCatalogName(), catalogName) && StringUtils.equals(entry.getSchemaName(), schemaName); + }); + } + } + + private static class TableSchemaMap { + + private final ConcurrentMap values; + + public TableSchemaMap() { + this.values = new ConcurrentHashMap<>(32); + } + + public void removeDatabase(String catalogName, String schemaName) { + values.entrySet().removeIf(entry -> { + TableId key = entry.getKey(); + return StringUtils.equals(key.getCatalogName(), catalogName) && StringUtils.equals(key.getSchemaName(), schemaName); + }); + } + } +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/Column.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/Column.java new file mode 100644 index 0000000000..d31675af12 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/Column.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.table.catalog; + +import org.apache.eventmesh.connector.jdbc.table.type.EventMeshDataType; + +import java.io.Serializable; +import java.sql.JDBCType; +import java.sql.Types; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * Column of {@link TableSchema}. + */ +@Data +@AllArgsConstructor +@NoArgsConstructor +public abstract class Column implements Serializable { + + /** + * Name of the column + */ + protected String name; + + /** + * Data type of the column + */ + protected EventMeshDataType dataType; + + /** + * {@link Types JDBC type} + */ + protected JDBCType jdbcType; + + /** + * Length of the column + */ + protected Integer columnLength; + + /** + * Decimal point of the column + */ + protected Integer decimal; + + /** + * Indicates if the column can be null or not + */ + protected boolean notNull; + + /** + * Comment for the column + */ + protected String comment; + + /** + * Default value for the column + */ + protected Object defaultValue; + + protected String defaultValueExpression; + + //protected int index; + + /** + * Creates a new instance of the ColumnEditor. + * + * @return A new instance of the ColumnEditor. + */ + public static ColumnEditor ofEditor() { + return new DefaultColumnEditorImpl(); + } + + public static ColumnEditor ofEditor(String name) { + return new DefaultColumnEditorImpl(name); + } + + /** + * creates a clone of the Column + * + * @return clone of column + */ + public abstract Column clone(); + +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/ColumnEditor.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/ColumnEditor.java new file mode 100644 index 0000000000..df206c7cb9 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/ColumnEditor.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.table.catalog; + +import org.apache.eventmesh.connector.jdbc.table.type.EventMeshDataType; + +import java.sql.JDBCType; +import java.sql.Types; + +/** + * Interface for editing column properties. + */ +public interface ColumnEditor { + + /** + * Sets the name of the column. + * + * @param name The name of the column. + * @return The ColumnEditor instance. + */ + ColumnEditor withName(String name); + + /** + * Sets the type of the column. + * + * @param typeName The type of the column. + * @return The ColumnEditor instance. + */ + ColumnEditor withType(String typeName); + + /** + * Sets the {@link Types JDBC type} of this column. + * + * @param jdbcType The JDBC type of the column. + * @return The ColumnEditor instance. + */ + ColumnEditor withJdbcType(JDBCType jdbcType); + + /** + * Sets the event mesh type of the column. + * + * @param eventMeshType The event mesh type of the column. + * @return The ColumnEditor instance. + */ + ColumnEditor withEventMeshType(EventMeshDataType eventMeshType); + + /** + * Sets the character set name of the column. + * + * @param charsetName The character set name of the column. + * @return The ColumnEditor instance. + */ + ColumnEditor charsetName(String charsetName); + + /** + * Sets the character set name of the table associated with the column. + * + * @param charsetName The character set name of the table. + * @return The ColumnEditor instance. + */ + ColumnEditor charsetNameOfTable(String charsetName); + + /** + * Sets the length of the column. + * + * @param length The length of the column. + * @return The ColumnEditor instance. + */ + ColumnEditor length(int length); + + /** + * Sets the scale of the column. + * + * @param scale The scale of the column. + * @return The ColumnEditor instance. + */ + ColumnEditor scale(Integer scale); + + /** + * Sets whether the column is optional. + * + * @param optional Whether the column is optional. + * @return The ColumnEditor instance. + */ + ColumnEditor optional(boolean optional); + + /** + * Sets the comment of the column. + * + * @param comment The comment of the column. + * @return The ColumnEditor instance. + */ + ColumnEditor comment(String comment); + + /** + * Sets whether the column is auto-incremented. + * + * @param autoIncremented Whether the column is auto-incremented. + * @return The ColumnEditor instance. + */ + ColumnEditor autoIncremented(boolean autoIncremented); + + /** + * Sets whether the column is generated. + * + * @param generated Whether the column is generated. + * @return The ColumnEditor instance. + */ + ColumnEditor generated(boolean generated); + + /** + * Sets the default value expression of the column. + * + * @param defaultValueExpression The default value expression of the column. + * @return The ColumnEditor instance. + */ + ColumnEditor defaultValueExpression(String defaultValueExpression); + + /** + * Specifies whether the column should be marked as "NOT NULL." + * + * @param notNull True if the column should be marked as "NOT NULL," false otherwise. + * @return The updated ColumnEditor. + */ + ColumnEditor notNull(boolean notNull); + + + /** + * Builds the Column object with the edited properties. + * + * @return The built Column object. + */ + Column build(); +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/DefaultColumn.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/DefaultColumn.java new file mode 100644 index 0000000000..8ba1d43b9a --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/DefaultColumn.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.table.catalog; + +import org.apache.eventmesh.connector.jdbc.table.type.EventMeshDataType; + +import java.sql.JDBCType; + +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; + +@EqualsAndHashCode(callSuper = true) +@NoArgsConstructor +public class DefaultColumn extends Column { + + public DefaultColumn(String name, EventMeshDataType dataType, JDBCType jdbcType, Integer columnLength, Integer decimal, boolean notNull, + String comment, Object defaultValue, String defaultValueExpression) { + super(name, dataType, jdbcType, columnLength, decimal, notNull, comment, defaultValue, defaultValueExpression); + } + + public static DefaultColumn of( + String name, EventMeshDataType dataType, JDBCType jdbcType, Integer columnLength, Integer decimal, boolean notNull, + String comment, Object defaultValue, String defaultValueExpression) { + return new DefaultColumn(name, dataType, jdbcType, columnLength, decimal, notNull, comment, defaultValue, defaultValueExpression); + } + + /** + * creates a clone of the Column + * + * @return clone of column + */ + @Override + public Column clone() { + return DefaultColumn.of(name, dataType, jdbcType, columnLength, decimal, notNull, comment, defaultValue, defaultValueExpression); + } +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/DefaultColumnEditorImpl.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/DefaultColumnEditorImpl.java new file mode 100644 index 0000000000..052aa1725e --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/DefaultColumnEditorImpl.java @@ -0,0 +1,258 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.table.catalog; + +import org.apache.eventmesh.connector.jdbc.table.type.EventMeshDataType; + +import java.sql.JDBCType; + +public class DefaultColumnEditorImpl implements ColumnEditor { + + /** + * Name of the column + */ + private String name; + + /** + * Data type of the column + */ + private EventMeshDataType dataType; + + /** + * Length of the column + */ + private Integer columnLength; + + /** + * Decimal point of the column + */ + private Integer decimal; + + /** + * Indicates if the column can be null or not + */ + private boolean notNull; + + /** + * Comment for the column + */ + private String comment; + + /** + * Default value for the column + */ + private Object defaultValue; + + private String typeName; + + private JDBCType jdbcType; + + private String defaultValueExpression; + + private String characterSet; + + private boolean optional; + + private boolean autoIncremented; + + private boolean generated; + + public DefaultColumnEditorImpl(String name) { + this.name = name; + } + + public DefaultColumnEditorImpl() { + } + + /** + * Sets the name of the column. + * + * @param name The name of the column. + * @return The ColumnEditor instance. + */ + @Override + public ColumnEditor withName(String name) { + this.name = name; + return this; + } + + /** + * Sets the type of the column. + * + * @param typeName The type of the column. + * @return The ColumnEditor instance. + */ + @Override + public ColumnEditor withType(String typeName) { + this.typeName = typeName; + return this; + } + + /** + * Sets the JDBC type of the column. + * + * @param jdbcType The JDBC type of the column. + * @return The ColumnEditor instance. + */ + @Override + public ColumnEditor withJdbcType(JDBCType jdbcType) { + this.jdbcType = jdbcType; + return this; + } + + /** + * Sets the event mesh type of the column. + * + * @param eventMeshType The event mesh type of the column. + * @return The ColumnEditor instance. + */ + @Override + public ColumnEditor withEventMeshType(EventMeshDataType eventMeshType) { + this.dataType = eventMeshType; + return this; + } + + /** + * Sets the character set name of the column. + * + * @param charsetName The character set name of the column. + * @return The ColumnEditor instance. + */ + @Override + public ColumnEditor charsetName(String charsetName) { + this.characterSet = charsetName; + return this; + } + + /** + * Sets the character set name of the table associated with the column. + * + * @param charsetName The character set name of the table. + * @return The ColumnEditor instance. + */ + @Override + public ColumnEditor charsetNameOfTable(String charsetName) { + return this; + } + + /** + * Sets the length of the column. + * + * @param length The length of the column. + * @return The ColumnEditor instance. + */ + @Override + public ColumnEditor length(int length) { + this.columnLength = length; + return this; + } + + /** + * Sets the scale of the column. + * + * @param scale The scale of the column. + * @return The ColumnEditor instance. + */ + @Override + public ColumnEditor scale(Integer scale) { + this.decimal = scale; + return this; + } + + /** + * Sets whether the column is optional. + * + * @param optional Whether the column is optional. + * @return The ColumnEditor instance. + */ + @Override + public ColumnEditor optional(boolean optional) { + this.optional = optional; + return this; + } + + /** + * Sets the comment of the column. + * + * @param comment The comment of the column. + * @return The ColumnEditor instance. + */ + @Override + public ColumnEditor comment(String comment) { + this.comment = comment; + return this; + } + + /** + * Sets whether the column is auto-incremented. + * + * @param autoIncremented Whether the column is auto-incremented. + * @return The ColumnEditor instance. + */ + @Override + public ColumnEditor autoIncremented(boolean autoIncremented) { + this.autoIncremented = autoIncremented; + return this; + } + + /** + * Sets whether the column is generated. + * + * @param generated Whether the column is generated. + * @return The ColumnEditor instance. + */ + @Override + public ColumnEditor generated(boolean generated) { + this.generated = generated; + return this; + } + + /** + * Sets the default value expression of the column. + * + * @param defaultValueExpression The default value expression of the column. + * @return The ColumnEditor instance. + */ + @Override + public ColumnEditor defaultValueExpression(String defaultValueExpression) { + this.defaultValueExpression = defaultValueExpression; + return this; + } + + /** + * Specifies whether the column should be marked as "NOT NULL." + * + * @param notNull True if the column should be marked as "NOT NULL," false otherwise. + * @return The updated ColumnEditor. + */ + @Override + public ColumnEditor notNull(boolean notNull) { + this.notNull = notNull; + return this; + } + + /** + * Builds the Column object with the edited properties. + * + * @return The built Column object. + */ + @Override + public Column build() { + return DefaultColumn.of(name, dataType, jdbcType, columnLength, decimal, notNull, comment, defaultValue, defaultValueExpression); + } +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/DefaultTableEditorImpl.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/DefaultTableEditorImpl.java new file mode 100644 index 0000000000..5d8747f6c7 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/DefaultTableEditorImpl.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.table.catalog; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public class DefaultTableEditorImpl implements TableEditor { + + private TableId tableId; + + private PrimaryKey primaryKey; + + private List uniqueKeys = new ArrayList<>(8); + + private Map columns = new HashMap<>(16); + + private String comment; + + private String sql; + + public DefaultTableEditorImpl(TableId tableId) { + this.tableId = tableId; + } + + @Override + public TableId ofTableId() { + return tableId; + } + + @Override + public List columns() { + return columns.values().stream().collect(Collectors.toList()); + } + + @Override + public Column columnWithName(String name) { + return columns.get(name); + } + + @Override + public List primaryKeyColumnNames() { + return primaryKey == null ? null : primaryKey.getColumnNames(); + } + + /** + * Returns a list of column that make up the primary key of the table. + * + * @return The list of primary key column names. + */ + @Override + public List primaryKeyColumns() { + return Arrays.asList(primaryKey); + } + + @Override + public TableEditor addColumns(Column... columns) { + if (columns != null && columns.length > 0) { + for (Column column : columns) { + this.columns.put(column.getName(), column); + } + } + return this; + } + + @Override + public TableEditor removeColumn(String columnName) { + this.columns.remove(columnName); + return this; + } + + @Override + public TableEditor setPrimaryKeyNames(List pkColumnNames, String comment) { + this.primaryKey = new PrimaryKey(pkColumnNames, comment); + return this; + } + + /** + * Specifies the unique key columns for the table. + * + * @param ukColumnNames The names of the columns that form the unique key. + * @param comment The comment for the unique key constraint. + * @return The updated TableEditor. + */ + @Override + public TableEditor setUniqueKeyColumnsNames(String ukName, List ukColumnNames, String comment) { + this.uniqueKeys.add(new UniqueKey(ukName, ukColumnNames, comment)); + return this; + } + + @Override + public TableEditor withComment(String comment) { + this.comment = comment; + return this; + } + + @Override + public TableSchema create() { + TableSchema.newTableSchemaBuilder() + .withName(this.tableId.getTableName()) + .withColumns(new ArrayList<>(columns.values())) + .withPrimaryKey(primaryKey) + .withUniqueKeys(uniqueKeys) + .withComment(comment) + .build(); + return new TableSchema(); + } + + @Override + public String ofSql() { + return sql; + } + + @Override + public TableEditor withSql(String sql) { + this.sql = sql; + return this; + } +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/Options.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/Options.java new file mode 100644 index 0000000000..de2429695e --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/Options.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.table.catalog; + +import java.util.HashMap; + +public class Options extends HashMap { + +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/PrimaryKey.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/PrimaryKey.java new file mode 100644 index 0000000000..521733943b --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/PrimaryKey.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.table.catalog; + +import java.util.List; + +public class PrimaryKey extends UniqueKey { + + + public PrimaryKey(String name, List columnNames, String comment) { + super(name, columnNames, comment); + } + + public PrimaryKey(String name, List columnNames) { + super(name, columnNames); + } + + public PrimaryKey(List columnNames) { + super(columnNames); + } + + public PrimaryKey(List columnNames, String comment) { + super(null, columnNames, comment); + } + + /** + * Creates a new PrimaryKey instance with the given name and column names. + * + * @param columnNames The list of column names that make up the primary key. + * @return A new PrimaryKey instance. + */ + public static PrimaryKey of(List columnNames) { + return new PrimaryKey(columnNames); + } + + /** + * Creates a copy of this PrimaryKey instance. + * + * @return A new PrimaryKey instance with the same name and column names. + */ + public PrimaryKey copy() { + return new PrimaryKey(getName(), getColumnNames(), getComment()); + } +} \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/Table.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/Table.java new file mode 100644 index 0000000000..4e9077191d --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/Table.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.table.catalog; + +import java.util.List; + +public class Table { + + private String name; + + private PrimaryKey primaryKey; + + private List uniqueKeys; + + private String comment; + +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/TableEditor.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/TableEditor.java new file mode 100644 index 0000000000..f2f1fb04cc --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/TableEditor.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.table.catalog; + + +import java.util.List; +import java.util.stream.Collectors; + +/** + * Edit and configure a database table schema. + */ +public interface TableEditor { + + String ofSql(); + + /** + * Returns the TableId associated with this table. + * + * @return The TableId of the table. + */ + TableId ofTableId(); + + /** + * Returns a list of columns in the table. + * + * @return The list of columns in the table. + */ + List columns(); + + /** + * Returns a list of column names in the table. + * + * @return The list of column names in the table. + */ + default List columnNames() { + return columns().stream().map(Column::getName).collect(Collectors.toList()); + } + + /** + * Returns the Column object with the given name. + * + * @param name The name of the column to retrieve. + * @return The Column object with the given name, or null if not found. + */ + Column columnWithName(String name); + + /** + * Returns a list of column names that make up the primary key of the table. + * + * @return The list of primary key column names. + */ + List primaryKeyColumnNames(); + + /** + * Returns a list of column that make up the primary key of the table. + * + * @return The list of primary key column names. + */ + List primaryKeyColumns(); + + /** + * Returns true if the table has a primary key, false otherwise. + * + * @return True if the table has a primary key, false otherwise. + */ + default boolean hasPrimaryKey() { + return !primaryKeyColumnNames().isEmpty(); + } + + /** + * Adds the specified columns to the table schema. + * + * @param columns The columns to be added to the table. + * @return The updated TableEditor with the added columns. + */ + TableEditor addColumns(Column... columns); + + /** + * Removes the column with the given column name from the table schema. + * + * @param columnName The name of the column to be removed. + * @return The updated TableEditor with the specified column removed. + */ + TableEditor removeColumn(String columnName); + + /** + * Sets the names of the columns that form the primary key of the table. + * + * @param pkColumnNames The list of column names that form the primary key. + * @param comment The comment for the primary key. + * @return The updated TableEditor with the new primary key information. + */ + TableEditor setPrimaryKeyNames(List pkColumnNames, String comment); + + + /** + * Sets the unique key information for the table. + * + * @param ukName The name of the unique key. + * @param ukColumnNames The list of column names that form the unique key. + * @param comment The comment for the unique key. + * @return The updated TableEditor with the new unique key information. + */ + TableEditor setUniqueKeyColumnsNames(String ukName, List ukColumnNames, String comment); + + + /** + * Sets the comment for the table schema. + * + * @param comment The comment to be set for the table. + * @return The updated TableEditor with the new comment set. + */ + TableEditor withComment(String comment); + + + TableEditor withSql(String sql); + + /** + * Creates and returns the final TableSchema based on the configured table properties. + * + * @return The TableSchema representing the final table configuration. + */ + TableSchema create(); +} + diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/TableId.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/TableId.java new file mode 100644 index 0000000000..acecc833eb --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/TableId.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.table.catalog; + +import java.io.Serializable; +import java.util.Objects; + +import lombok.Getter; +import lombok.Setter; + +/** + * Represents a table identifier with catalog name, schema name and table name. + */ +@Getter +@Setter +public class TableId implements Serializable { + + /** + * The default mapper that converts a TableId to its string representation. + */ + public static final TableIdToStringMapper DEFAULT_TABLEIDTOSTRINGMAPPER = new DefaultTableIdToStringMapper(); + + private String catalogName; + + private String schemaName; + + private String tableName; + + private String id; + + public TableId() { + } + + /** + * Constructs a TableId instance without a TableIdToStringMapper. + * + * @param catalogName the catalog name of the table + * @param schemaName the schema name of the table + * @param tableName the name of the table + */ + public TableId(String catalogName, String schemaName, String tableName) { + this(catalogName, schemaName, tableName, null); + } + + /** + * Constructs a TableId instance without a TableIdToStringMapper. + * + * @param catalogName the catalog name of the table + */ + public TableId(String catalogName) { + this(catalogName, null, null, null); + } + + /** + * Constructs a TableId instance with a TableIdToStringMapper. If the mapper is null, the default mapper will be used. + * + * @param catalogName the catalog name of the table + * @param schemaName the schema name of the table + * @param tableName the name of the table + * @param mapper the mapper that converts a TableId to its string representation + */ + public TableId(String catalogName, String schemaName, String tableName, TableIdToStringMapper mapper) { + this.catalogName = catalogName; + this.schemaName = schemaName; + this.tableName = tableName; + this.id = mapper == null ? DEFAULT_TABLEIDTOSTRINGMAPPER.toString(this) : mapper.toString(this); + + } + + @Override + public String toString() { + return id == null ? DEFAULT_TABLEIDTOSTRINGMAPPER.toString(this) : id; + } + + /** + * Returns the string representation of the TableId, which is the same as calling toString(). + * + * @return the string representation of the TableId + */ + public String tablePath() { + return id; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof TableId)) { + return false; + } + TableId tableId = (TableId) o; + return Objects.equals(getCatalogName(), tableId.getCatalogName()) && Objects.equals(getSchemaName(), tableId.getSchemaName()) + && Objects.equals(getTableName(), tableId.getTableName()) && Objects.equals(getId(), tableId.getId()); + } + + @Override + public int hashCode() { + return Objects.hash(getCatalogName(), getSchemaName(), getTableName(), getId()); + } + + /** + * A functional interface that converts a TableId to its string representation. + */ + @FunctionalInterface + public interface TableIdToStringMapper { + + String toString(TableId tableId); + } + + /** + * Returns the string representation of a TableId. If catalog or schema is null or empty, they will be excluded from the string. + * + * @param catalog the catalog name of the table + * @param schema the schema name of the table + * @param table the name of the table + * @return the string representation of the TableId + */ + private static String tableId(String catalog, String schema, String table) { + if (catalog == null || catalog.length() == 0) { + if (schema == null || schema.length() == 0) { + return table; + } + return schema + "." + table; + } + if (schema == null || schema.length() == 0) { + return catalog + "." + table; + } + return catalog + "." + schema + "." + table; + } + + /** + * The default mapper that converts a TableId to its string representation. + */ + private static class DefaultTableIdToStringMapper implements TableIdToStringMapper { + + public String toString(TableId tableId) { + return tableId(tableId.getCatalogName(), tableId.getSchemaName(), tableId.getTableName()); + } + } +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/TableSchema.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/TableSchema.java new file mode 100644 index 0000000000..3d2c0446f7 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/TableSchema.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.table.catalog; + +import java.io.Serializable; +import java.util.List; +import java.util.Map; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@Builder +public class TableSchema implements Serializable { + + private String name; + + /** + * A map of column names to their respective column objects. + */ + private Map columnMap; + + /** + * A list of columns in the table. + */ + private List columns; + + /** + * The primary key of the table. + */ + private PrimaryKey primaryKey; + + private List uniqueKeys; + + private String comment; + + public TableSchema(String name) { + this.name = name; + } + + public static TableSchemaBuilder newTableSchemaBuilder() { + return new TableSchemaBuilder(); + } + + public static class TableSchemaBuilder { + + private String name; + private Map columnMap; + private List columns; + private PrimaryKey primaryKey; + private List uniqueKeys; + private String comment; + + public TableSchemaBuilder() { + + } + + public TableSchemaBuilder withName(String name) { + this.name = name; + return this; + } + + public TableSchemaBuilder withColumns(Map columnMap) { + this.columnMap = columnMap; + return this; + } + + public TableSchemaBuilder withColumns(List columns) { + this.columns = columns; + return this; + } + + public TableSchemaBuilder withPrimaryKey(PrimaryKey primaryKey) { + this.primaryKey = primaryKey; + return this; + } + + public TableSchemaBuilder withUniqueKeys(List uniqueKeys) { + this.uniqueKeys = uniqueKeys; + return this; + } + + public TableSchemaBuilder withComment(String comment) { + this.comment = comment; + return this; + } + + public TableSchema build() { + return new TableSchema(name, columnMap, columns, primaryKey, uniqueKeys, comment); + } + + } +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/UniqueKey.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/UniqueKey.java new file mode 100644 index 0000000000..c589ae3819 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/catalog/UniqueKey.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.table.catalog; + +import java.io.Serializable; +import java.util.List; + +import lombok.Getter; +import lombok.Setter; + + +@Setter +@Getter +public class UniqueKey implements Serializable { + + private String name; + + private final List columnNames; + + private String comment; + + public UniqueKey(String name, List columnNames, String comment) { + this.name = name; + this.columnNames = columnNames; + this.comment = comment; + } + + public UniqueKey(String name, List columnNames) { + this.name = name; + this.columnNames = columnNames; + } + + public UniqueKey(List columnNames) { + this.columnNames = columnNames; + } + + public UniqueKey copy() { + return new UniqueKey(name, columnNames, comment); + } +} + diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/CalendarType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/CalendarType.java new file mode 100644 index 0000000000..0c5aa0a743 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/CalendarType.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.table.type; + +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.temporal.Temporal; +import java.util.Objects; + + +public class CalendarType implements EventMeshDataType { + + // Constants for LocalDate, LocalTime, and LocalDateTime types + public static final CalendarType LOCAL_DATE_TYPE = new CalendarType<>(LocalDate.class, SQLType.DATE); + + public static final CalendarType LOCAL_TIME_TYPE = new CalendarType<>(LocalTime.class, SQLType.TIME); + + public static final CalendarType LOCAL_DATE_TIME_TYPE = new CalendarType<>(LocalDateTime.class, SQLType.TIMESTAMP); + + private final Class typeClass; + private final SQLType sqlType; + + private CalendarType(Class typeClass, SQLType sqlType) { + this.typeClass = typeClass; + this.sqlType = sqlType; + } + + /** + * Returns the type class of the data. + * + * @return the type class of the data. + */ + @Override + public Class getTypeClass() { + return typeClass; + } + + /** + * Returns the SQL type of the data. + * + * @return the SQL type of the data. + */ + @Override + public SQLType getSQLType() { + return sqlType; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof CalendarType)) { + return false; + } + CalendarType that = (CalendarType) o; + return Objects.equals(getTypeClass(), that.getTypeClass()) && sqlType == that.sqlType; + } + + @Override + public int hashCode() { + return Objects.hash(getTypeClass(), sqlType); + } + + @Override + public String toString() { + return typeClass.getName(); + } +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/DecimalType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/DecimalType.java new file mode 100644 index 0000000000..e08f508124 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/DecimalType.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.table.type; + +import java.math.BigDecimal; +import java.util.Objects; + +import lombok.Getter; + + +public class DecimalType extends PrimitiveType { + + @Getter + private final int precision; + + @Getter + private final int scale; + + public DecimalType(int precision, int scale) { + super(BigDecimal.class, SQLType.DECIMAL); + this.precision = precision; + this.scale = scale; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof DecimalType)) { + return false; + } + if (!super.equals(o)) { + return false; + } + DecimalType that = (DecimalType) o; + return precision == that.precision && scale == that.scale; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), precision, scale); + } + + @Override + public String toString() { + return getTypeClass().getName(); + } +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/EventMeshDataType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/EventMeshDataType.java new file mode 100644 index 0000000000..d2a2df1be4 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/EventMeshDataType.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.table.type; + +/** + * Defines Event Mesh data type with methods to get the type class and SQL type of the data. + */ +public interface EventMeshDataType { + + /** + * Gets the type class of the data. + * + * @return the type class of the data. + */ + Class getTypeClass(); + + /** + * Gets the SQL type of the data. + * + * @return the SQL type of the data. + */ + SQLType getSQLType(); +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/EventMeshRow.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/EventMeshRow.java new file mode 100644 index 0000000000..81c24a3a08 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/EventMeshRow.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.table.type; + +import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; + +import java.io.Serializable; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +import lombok.Getter; +import lombok.Setter; + +/** + * Represents a row in an event mesh. + */ +@Getter +@Setter +public class EventMeshRow implements Serializable { + + /** + * The mode for handling the row (e.g. INSERT, UPDATE, DELETE) {@link RowHandleMode} + */ + private RowHandleMode mode = RowHandleMode.INSERT; + + /** + * The ID of the table that the row belongs to + */ + private TableId tableId; + + /** + * The values of the fields in the row + */ + private final Object[] fieldValues; + + /** + * Any additional metadata associated with the row + */ + private Map ext = new HashMap<>(); + + public EventMeshRow(int fieldNum) { + this.fieldValues = new Object[fieldNum]; + } + + public EventMeshRow(int fieldNum, TableId tableId) { + this.tableId = tableId; + this.fieldValues = new Object[fieldNum]; + } + + public EventMeshRow(RowHandleMode mode, int fieldNum, TableId tableId) { + this.mode = mode; + this.tableId = tableId; + this.fieldValues = new Object[fieldNum]; + } + + /** + * Sets the values of the fields in the row. + * + * @param fieldValues the new field values + * @throws NullPointerException if fieldValues is null + * @throws IllegalArgumentException if fieldValues has a different length than the existing field values + */ + public void setFieldValues(Object[] fieldValues) { + Objects.requireNonNull(fieldValues, "Parameter fields can not be null"); + if (this.fieldValues.length != fieldValues.length) { + throw new IllegalArgumentException(); + } + System.arraycopy(fieldValues, 0, this.fieldValues, 0, this.fieldValues.length); + } + +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/EvetMeshRowType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/EvetMeshRowType.java new file mode 100644 index 0000000000..738d57b2d0 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/EvetMeshRowType.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.eventmesh.connector.jdbc.table.type; + +import java.util.List; + +public class EvetMeshRowType implements EventMeshDataType { + + private List>> fields; + + + public EvetMeshRowType(List>> fields) { + this.fields = fields; + } + + /** + * Returns the type class of the data. + * + * @return the type class of the data. + */ + @Override + public Class getTypeClass() { + return EventMeshRow.class; + } + + /** + * Returns the SQL type of the data. + * + * @return the SQL type of the data. + */ + @Override + public SQLType getSQLType() { + return SQLType.ROW; + } +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/MapType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/MapType.java new file mode 100644 index 0000000000..c5c684f927 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/MapType.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.table.type; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Objects; + + +public class MapType implements EventMeshDataType> { + + private static final List SUPPORTED_KEY_TYPES = + Arrays.asList( + SQLType.NULL, + SQLType.BOOLEAN, + SQLType.TINYINT, + SQLType.SMALLINT, + SQLType.INTEGER, + SQLType.BIGINT, + SQLType.DATE, + SQLType.TIME, + SQLType.TIMESTAMP, + SQLType.FLOAT, + SQLType.DOUBLE, + SQLType.STRING, + SQLType.DECIMAL); + + private final EventMeshDataType keyType; + + private final EventMeshDataType valueType; + + public MapType(EventMeshDataType keyType, EventMeshDataType valueType) { + Objects.requireNonNull(keyType, "The key type is required."); + Objects.requireNonNull(valueType, "The value type is required."); + + if (!SUPPORTED_KEY_TYPES.contains(keyType.getSQLType())) { + throw new IllegalArgumentException(String.format("Not support type: %s", keyType.getSQLType())); + } + + this.keyType = keyType; + this.valueType = valueType; + } + + /** + * Returns the type class of the data. + * + * @return the type class of the data. + */ + @SuppressWarnings("unchecked") + @Override + public Class> getTypeClass() { + return (Class>) (Class) Map.class; + } + + /** + * Returns the SQL type of the data. + * + * @return the SQL type of the data. + */ + @Override + public SQLType getSQLType() { + return SQLType.MAP; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof MapType)) { + return false; + } + MapType mapType = (MapType) o; + return Objects.equals(keyType, mapType.keyType) && Objects.equals(valueType, mapType.valueType); + } + + @Override + public int hashCode() { + return Objects.hash(keyType, valueType); + } + + public EventMeshDataType keyType() { + return this.keyType; + } + + public EventMeshDataType valueType() { + return this.valueType; + } +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/Pair.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/Pair.java new file mode 100644 index 0000000000..e5f0dd72ae --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/Pair.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.table.type; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@Builder +public class Pair { + + private Left left; + + private Right right; + +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/PrimitiveArrayType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/PrimitiveArrayType.java new file mode 100644 index 0000000000..ecfed9dbaa --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/PrimitiveArrayType.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.table.type; + +import java.util.Objects; + + +public class PrimitiveArrayType implements EventMeshDataType { + + @SuppressWarnings("unchecked") + public static final PrimitiveArrayType STRING_ARRAY_TYPE = new PrimitiveArrayType(String[].class, PrimitiveType.STRING_TYPE); + + @SuppressWarnings("unchecked") + public static final PrimitiveArrayType BOOLEAN_ARRAY_TYPE = new PrimitiveArrayType(Boolean[].class, + PrimitiveType.BOOLEAN_TYPE); + + @SuppressWarnings("unchecked") + public static final PrimitiveArrayType BYTE_ARRAY_TYPE = new PrimitiveArrayType(Byte[].class, PrimitiveType.BYTE_TYPE); + + @SuppressWarnings("unchecked") + public static final PrimitiveArrayType SHORT_ARRAY_TYPE = new PrimitiveArrayType(Short[].class, PrimitiveType.SHORT_TYPE); + + @SuppressWarnings("unchecked") + public static final PrimitiveArrayType INT_ARRAY_TYPE = new PrimitiveArrayType(Integer[].class, PrimitiveType.INT_TYPE); + + @SuppressWarnings("unchecked") + public static final PrimitiveArrayType LONG_ARRAY_TYPE = new PrimitiveArrayType(Long[].class, PrimitiveType.LONG_TYPE); + + @SuppressWarnings("unchecked") + public static final PrimitiveArrayType FLOAT_ARRAY_TYPE = new PrimitiveArrayType(Float[].class, PrimitiveType.FLOAT_TYPE); + + @SuppressWarnings("unchecked") + public static final PrimitiveArrayType DOUBLE_ARRAY_TYPE = new PrimitiveArrayType(Double[].class, PrimitiveType.DOUBLE_TYPE); + + private final Class typeClass; + + private final PrimitiveType sqlType; + + private PrimitiveArrayType(Class arrayClass, PrimitiveType elementType) { + this.typeClass = arrayClass; + this.sqlType = elementType; + } + + + /** + * Returns the type class of the data. + * + * @return the type class of the data. + */ + @Override + public Class getTypeClass() { + return typeClass; + } + + /** + * Returns the SQL type of the data. + * + * @return the SQL type of the data. + */ + @Override + public SQLType getSQLType() { + return SQLType.ARRAY; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof PrimitiveArrayType)) { + return false; + } + PrimitiveArrayType that = (PrimitiveArrayType) o; + return Objects.equals(getTypeClass(), that.getTypeClass()) && Objects.equals(sqlType, that.sqlType); + } + + @Override + public int hashCode() { + return Objects.hash(getTypeClass(), sqlType); + } + + @Override + public String toString() { + return typeClass.getName(); + } +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/PrimitiveByteArrayType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/PrimitiveByteArrayType.java new file mode 100644 index 0000000000..74c2ed9127 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/PrimitiveByteArrayType.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.table.type; + + +public class PrimitiveByteArrayType implements EventMeshDataType { + + public static final PrimitiveByteArrayType BYTES_TYPE = new PrimitiveByteArrayType(); + + private PrimitiveByteArrayType() { + + } + + /** + * Returns the type class of the data. + * + * @return the type class of the data. + */ + @Override + public Class getTypeClass() { + return byte[].class; + } + + /** + * Returns the SQL type of the data. + * + * @return the SQL type of the data. + */ + @Override + public SQLType getSQLType() { + return SQLType.BINARY; + } + + @Override + public int hashCode() { + return byte[].class.hashCode(); + } + + @Override + public boolean equals(Object obj) { + if (obj == this) { + return true; + } + return obj instanceof PrimitiveByteArrayType; + } + + @Override + public String toString() { + return byte[].class.getName(); + } +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/PrimitiveType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/PrimitiveType.java new file mode 100644 index 0000000000..d6e8bdd514 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/PrimitiveType.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.table.type; + +import java.util.Objects; + +public class PrimitiveType implements EventMeshDataType { + + public static final PrimitiveType STRING_TYPE = new PrimitiveType<>(String.class, SQLType.STRING); + public static final PrimitiveType BOOLEAN_TYPE = new PrimitiveType<>(Boolean.class, SQLType.BOOLEAN); + public static final PrimitiveType BYTE_TYPE = new PrimitiveType<>(Byte.class, SQLType.TINYINT); + public static final PrimitiveType SHORT_TYPE = new PrimitiveType<>(Short.class, SQLType.SMALLINT); + public static final PrimitiveType INT_TYPE = new PrimitiveType<>(Integer.class, SQLType.INTEGER); + public static final PrimitiveType LONG_TYPE = new PrimitiveType<>(Long.class, SQLType.BIGINT); + public static final PrimitiveType FLOAT_TYPE = new PrimitiveType<>(Float.class, SQLType.FLOAT); + public static final PrimitiveType DOUBLE_TYPE = new PrimitiveType<>(Double.class, SQLType.DOUBLE); + public static final PrimitiveType VOID_TYPE = new PrimitiveType<>(Void.class, SQLType.NULL); + + private final Class typeClass; + + private final SQLType sqlType; + + public PrimitiveType(Class typeClass, SQLType sqlType) { + this.typeClass = typeClass; + this.sqlType = sqlType; + } + + /** + * Returns the type class of the data. + * + * @return the type class of the data. + */ + @Override + public Class getTypeClass() { + return typeClass; + } + + /** + * Returns the SQL type of the data. + * + * @return the SQL type of the data. + */ + @Override + public SQLType getSQLType() { + return sqlType; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof PrimitiveType)) { + return false; + } + PrimitiveType that = (PrimitiveType) o; + return Objects.equals(getTypeClass(), that.getTypeClass()) && sqlType == that.sqlType; + } + + @Override + public int hashCode() { + return Objects.hash(getTypeClass(), sqlType); + } + + @Override + public String toString() { + return typeClass.getName(); + } +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/RowHandleMode.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/RowHandleMode.java new file mode 100644 index 0000000000..ed428f132e --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/RowHandleMode.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.table.type; + +/** + * An enum representing the different modes in which a row can be handled. + */ +public enum RowHandleMode { + INSERT("+I", (byte) 1), + UPDATE_BEFORE("-UB", (byte) 2), + UPDATE_AFTER("+UA", (byte) 3), + DELETE("-D", (byte) 1), + ; + + private final String shortCut; + + private final byte value; + + /** + * Constructor for RowHandleMode. + * + * @param shortCut a string representing the shorthand for the row handle mode. + * @param value a byte representing the value of the row handle mode. + */ + RowHandleMode(String shortCut, byte value) { + this.shortCut = shortCut; + this.value = value; + } + + /** + * Returns the shorthand for the row handle mode. + * + * @return a string representing the shorthand for the row handle mode. + */ + public String toShortCut() { + return shortCut; + } + + /** + * Returns the value of the row handle mode. + * + * @return a byte representing the value of the row handle mode. + */ + public byte toValue() { + return value; + } + + /** + * Returns the row handle mode corresponding to the given byte value. + * + * @param value a byte representing the value of the row handle mode. + * @return the row handle mode corresponding to the given byte value. + * @throws UnsupportedOperationException if the byte value is not supported. + */ + public static RowHandleMode fromByteValue(byte value) { + switch (value) { + case 0: + return INSERT; + case 1: + return UPDATE_BEFORE; + case 2: + return UPDATE_AFTER; + case 3: + return DELETE; + default: + throw new UnsupportedOperationException( + "Unsupported byte value '" + value + "' for row handle mode."); + } + } +} + diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/SQLType.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/SQLType.java new file mode 100644 index 0000000000..203ceb84bb --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/table/type/SQLType.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.table.type; + +/** + * see {@link java.sql.SQLType} + */ +public enum SQLType { + + /** + * Identifies the generic SQL type {@code TINYINT}. + */ + TINYINT, + /** + * Identifies the generic SQL type {@code SMALLINT}. + */ + SMALLINT, + /** + * Identifies the generic SQL type {@code INTEGER}. + */ + INTEGER, + /** + * Identifies the generic SQL type {@code BIGINT}. + */ + BIGINT, + /** + * Identifies the generic SQL type {@code FLOAT}. + */ + FLOAT, + + /** + * Identifies the generic SQL type {@code DOUBLE}. + */ + DOUBLE, + + /** + * Identifies the generic SQL type {@code DECIMAL}. + */ + DECIMAL, + + /** + * Identifies the generic SQL type {@code DATE}. + */ + DATE, + /** + * Identifies the generic SQL type {@code TIME}. + */ + TIME, + /** + * Identifies the generic SQL type {@code TIMESTAMP}. + */ + TIMESTAMP, + /** + * Identifies the generic SQL type {@code BINARY}. + */ + BINARY, + + /** + * Identifies the generic SQL value {@code NULL}. + */ + NULL, + + /** + * Identifies the generic SQL type {@code ARRAY}. + */ + ARRAY, + + /** + * Identifies the generic SQL type {@code BOOLEAN}. + */ + BOOLEAN, + + /** + * EventMesh generic SQL type + */ + ROW, + + MAP, + + STRING +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/utils/Antlr4Utils.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/utils/Antlr4Utils.java new file mode 100644 index 0000000000..c003357fd7 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/utils/Antlr4Utils.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.utils; + +import org.antlr.v4.runtime.ParserRuleContext; +import org.antlr.v4.runtime.misc.Interval; + +import lombok.experimental.UtilityClass; + +@UtilityClass +public class Antlr4Utils { + + public static String getText(ParserRuleContext ctx) { + return getText(ctx, ctx.start.getStartIndex(), ctx.stop.getStopIndex()); + } + + public static String getText(ParserRuleContext ctx, int start, int stop) { + Interval interval = new Interval(start, stop); + return ctx.start.getInputStream().getText(interval); + } +} \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/utils/JdbcStringUtils.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/utils/JdbcStringUtils.java new file mode 100644 index 0000000000..d74017737e --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/utils/JdbcStringUtils.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.utils; + +import lombok.experimental.UtilityClass; + +@UtilityClass +public class JdbcStringUtils { + + public static boolean isWrapped(String possiblyWrapped) { + if (possiblyWrapped.length() < 2) { + return false; + } + if (possiblyWrapped.startsWith("`") && possiblyWrapped.endsWith("`")) { + return true; + } + if (possiblyWrapped.startsWith("'") && possiblyWrapped.endsWith("'")) { + return true; + } + if (possiblyWrapped.startsWith("\"") && possiblyWrapped.endsWith("\"")) { + return true; + } + return false; + } + + public static boolean isWrapped(char c) { + return c == '\'' || c == '"' || c == '`'; + } + + public static String withoutWrapper(String possiblyWrapped) { + return isWrapped(possiblyWrapped) ? possiblyWrapped.substring(1, possiblyWrapped.length() - 1) : possiblyWrapped; + } + + +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/utils/MysqlUtils.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/utils/MysqlUtils.java new file mode 100644 index 0000000000..38f9a10f1a --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/utils/MysqlUtils.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.utils; + + +import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; + +public class MysqlUtils { + + private MysqlUtils() { + + } + + public static String wrapper(TableId tableId) { + return wrapper(tableId.toString()); + } + + public static String wrapper(String original) { + return "`" + original + "`"; + } + +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.connector.jdbc.source.dialect.DatabaseDialectFactory b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.connector.jdbc.source.dialect.DatabaseDialectFactory new file mode 100644 index 0000000000..8524b42ac8 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.connector.jdbc.source.dialect.DatabaseDialectFactory @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +mysql=org.apache.eventmesh.connector.jdbc.source.dialect.mysql.MysqlDatabaseDialectFactory diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.connector.jdbc.source.dialect.cdc.CdcEngineFactory b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.connector.jdbc.source.dialect.cdc.CdcEngineFactory new file mode 100644 index 0000000000..06eb1b99e7 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.connector.jdbc.source.dialect.cdc.CdcEngineFactory @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +mysql=org.apache.eventmesh.connector.jdbc.source.dialect.cdc.mysql.MysqlCdcEngineFactory diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.connector.jdbc.source.dialect.snapshot.SnapshotEngineFactory b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.connector.jdbc.source.dialect.snapshot.SnapshotEngineFactory new file mode 100644 index 0000000000..39075175e2 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.connector.jdbc.source.dialect.snapshot.SnapshotEngineFactory @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +mysql=org.apache.eventmesh.connector.jdbc.source.dialect.snapshot.mysql.MysqlSnapshotEngineFactory diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/resources/log4j2.xml b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/resources/log4j2.xml new file mode 100644 index 0000000000..cd699adad8 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/resources/log4j2.xml @@ -0,0 +1,31 @@ + + + + + + + + + + + + + + + + diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/resources/server-config.yml b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/resources/server-config.yml new file mode 100644 index 0000000000..5f66dd0f68 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/resources/server-config.yml @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +sourceEnable: true +sinkEnable: true diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/resources/source-config.yml b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/resources/source-config.yml new file mode 100644 index 0000000000..cc841fca7e --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/resources/source-config.yml @@ -0,0 +1,60 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +pubSubConfig: + meshAddress: 127.0.0.1:10001 + subject: TopicTest + idc: FT + env: PRD + group: rocketmqSource + appId: 5032 + userName: jdbcSourceUser + passWord: jdbcPassWord +sourceConnectorConfig: + maxTask: 10 #max task number + batchMaxRows: 100 + skipSnapshot: false + snapshotMaxThreads: 10 + snapshotSchema: true + snapshotData: true + snapshotFetchSize: 100 + databaseType: mysql + databaseIncludeList: + - mxsm + databaseExcludeList: + tableIncludeList: + tableExcludeList: + jdbcConfig: + hostname: localhost + port: 3306 + user: root + password: Mxsm22## + initialStatements: + connectTimeout: 10 + mysqlConfig: + serverId: 123 + keepAlive: true + keepAliveInterval: 6000 +offsetStorageConfig: + offsetStorageType: nacos + offsetStorageAddr: 127.0.0.1:8848 + extensions: { + #same with topic + dataId: TopicTest, + #same with group + group: rocketmqSource + } \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/test/java/org/apache/eventmesh/connector/jdbc/utils/JdbcStringUtilsTest.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/test/java/org/apache/eventmesh/connector/jdbc/utils/JdbcStringUtilsTest.java new file mode 100644 index 0000000000..e69f017538 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/test/java/org/apache/eventmesh/connector/jdbc/utils/JdbcStringUtilsTest.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.jdbc.utils; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; + +public class JdbcStringUtilsTest { + + @Test + public void testIsWrapped() { + assertTrue(JdbcStringUtils.isWrapped("`Hello`")); + assertTrue(JdbcStringUtils.isWrapped("'World'")); + assertTrue(JdbcStringUtils.isWrapped("\"Java\"")); + assertFalse(JdbcStringUtils.isWrapped("NotWrapped")); + assertFalse(JdbcStringUtils.isWrapped("`NotClosed")); + assertFalse(JdbcStringUtils.isWrapped("NotOpened`")); + } + + @Test + public void testWithoutWrapper() { + assertEquals("Hello", JdbcStringUtils.withoutWrapper("`Hello`")); + assertEquals("World", JdbcStringUtils.withoutWrapper("'World'")); + assertEquals("Java", JdbcStringUtils.withoutWrapper("\"Java\"")); + assertEquals("NotWrapped", JdbcStringUtils.withoutWrapper("NotWrapped")); + assertEquals("`NotClosed", JdbcStringUtils.withoutWrapper("`NotClosed")); + assertEquals("NotOpened`", JdbcStringUtils.withoutWrapper("NotOpened`")); + } + + @Test + public void testIsWrappedWithChar() { + assertTrue(JdbcStringUtils.isWrapped('`')); + assertTrue(JdbcStringUtils.isWrapped('\'')); + assertTrue(JdbcStringUtils.isWrapped('\"')); + assertFalse(JdbcStringUtils.isWrapped('A')); + } +} diff --git a/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/grpc/producer/EventMeshGrpcProducer.java b/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/grpc/producer/EventMeshGrpcProducer.java index 0d159190f6..29e867d537 100644 --- a/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/grpc/producer/EventMeshGrpcProducer.java +++ b/eventmesh-sdks/eventmesh-sdk-java/src/main/java/org/apache/eventmesh/client/grpc/producer/EventMeshGrpcProducer.java @@ -92,6 +92,7 @@ public Response publish(List messageList) { } } + @SuppressWarnings("unchecked") public T requestReply(final T message, final long timeout) { if (message instanceof CloudEvent) { diff --git a/eventmesh-spi/src/main/java/org/apache/eventmesh/spi/EventMeshExtensionFactory.java b/eventmesh-spi/src/main/java/org/apache/eventmesh/spi/EventMeshExtensionFactory.java index 5a65e43a58..62307109b5 100644 --- a/eventmesh-spi/src/main/java/org/apache/eventmesh/spi/EventMeshExtensionFactory.java +++ b/eventmesh-spi/src/main/java/org/apache/eventmesh/spi/EventMeshExtensionFactory.java @@ -29,6 +29,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; @@ -43,7 +44,7 @@ public class EventMeshExtensionFactory { private static final List EXTENSION_CLASS_LOADERS = new ArrayList<>(); - private static final ConcurrentHashMap EXTENSION_INSTANCE_CACHE = new ConcurrentHashMap<>(16); + private static final ConcurrentHashMap EXTENSION_INSTANCE_CACHE = new ConcurrentHashMap<>(16); static { EXTENSION_CLASS_LOADERS.add(MetaInfExtensionClassLoader.getInstance()); @@ -56,52 +57,72 @@ private EventMeshExtensionFactory() { } /** - * @param extensionType extension plugin class type - * @param extensionName extension instance name - * @param the type of the plugin + * Get an instance of an extension plugin. + * + * @param extensionClass extension plugin class type + * @param extensionName extension instance name + * @param the type of the plugin * @return plugin instance */ - public static T getExtension(Class extensionType, String extensionName) { - if (extensionType == null) { - throw new ExtensionException("extensionType is null"); + public static T getExtension(Class extensionClass, String extensionName) { + if (extensionClass == null) { + throw new ExtensionException("extensionClass is null"); } if (StringUtils.isEmpty(extensionName)) { throw new ExtensionException("extensionName is null"); } - if (!extensionType.isInterface() || !extensionType.isAnnotationPresent(EventMeshSPI.class)) { - throw new ExtensionException(String.format("extensionType:%s is invalided", extensionType)); + if (!extensionClass.isInterface() || !extensionClass.isAnnotationPresent(EventMeshSPI.class)) { + throw new ExtensionException(String.format("extensionClass:%s is invalided", extensionClass)); } - EventMeshSPI eventMeshSPIAnnotation = extensionType.getAnnotation(EventMeshSPI.class); + EventMeshSPI eventMeshSPIAnnotation = extensionClass.getAnnotation(EventMeshSPI.class); if (eventMeshSPIAnnotation.isSingleton()) { - return getSingletonExtension(extensionType, extensionName); + return getSingletonExtension(extensionClass, eventMeshSPIAnnotation, extensionName); } - return getPrototypeExtension(extensionType, extensionName); + return getPrototypeExtension(extensionClass, extensionName); } + /** + * Get a singleton instance of an extension plugin. + * + * @param extensionClass the type of the extension plugin + * @param spi the type of the spi + * @param extensionInstanceName the name of the extension instance + * @param the type of the extension plugin + * @return a singleton instance of the extension plugin + */ @SuppressWarnings("unchecked") - private static T getSingletonExtension(Class extensionType, String extensionInstanceName) { - return (T) EXTENSION_INSTANCE_CACHE.computeIfAbsent(extensionInstanceName, name -> { - Class extensionInstanceClass = getExtensionInstanceClass(extensionType, extensionInstanceName); + private static T getSingletonExtension(Class extensionClass, EventMeshSPI spi, String extensionInstanceName) { + return (T) EXTENSION_INSTANCE_CACHE.computeIfAbsent(new Extension(spi, extensionInstanceName), name -> { + Class extensionInstanceClass = getExtensionInstanceClass(extensionClass, extensionInstanceName); if (extensionInstanceClass == null) { + log.warn("Get extension instance class {} is null", extensionClass.getName()); return null; } try { T extensionInstance = extensionInstanceClass.getDeclaredConstructor().newInstance(); ConfigService.getInstance().populateConfigForObject(extensionInstance); - log.info("initialize extension instance success, extensionType: {}, extensionInstanceName: {}", - extensionType, extensionInstanceName); + log.info("initialize extension instance success, extensionClass: {}, extensionInstanceName: {}", + extensionClass, extensionInstanceName); return extensionInstance; } catch (NoSuchMethodException | InstantiationException | IllegalAccessException | InvocationTargetException e) { throw new ExtensionException("Extension initialize error", e); } catch (NoSuchFieldException | IOException e) { - log.error("initialize extension instance config failed, extensionType: {}, extensionInstanceName: {}", - extensionType, extensionInstanceName, e); + log.error("initialize extension instance config failed, extensionClass: {}, extensionInstanceName: {}", + extensionClass, extensionInstanceName, e); throw new ExtensionException("Extension initialize error", e); } }); } + /** + * Get a new instance of an extension plugin. + * + * @param extensionType the type of the extension plugin + * @param extensionInstanceName the name of the extension instance + * @param the type of the extension plugin + * @return a new instance of the extension plugin + */ private static T getPrototypeExtension(Class extensionType, String extensionInstanceName) { Class extensionInstanceClass = getExtensionInstanceClass(extensionType, extensionInstanceName); if (extensionInstanceClass == null) { @@ -123,6 +144,14 @@ private static T getPrototypeExtension(Class extensionType, String extens } } + /** + * Get the class of an extension instance. + * + * @param extensionType the type of the extension instance + * @param extensionInstanceName the name of the extension instance + * @param the type of the extension instance + * @return the class of the extension instance + */ @SuppressWarnings("unchecked") private static Class getExtensionInstanceClass(Class extensionType, String extensionInstanceName) { for (ExtensionClassLoader extensionClassLoader : EXTENSION_CLASS_LOADERS) { @@ -134,4 +163,33 @@ private static Class getExtensionInstanceClass(Class extensionType, St } return null; } + + private static class Extension { + + private EventMeshSPI spi; + + private String extensionInstanceName; + + public Extension(EventMeshSPI spi, String extensionInstanceName) { + this.spi = spi; + this.extensionInstanceName = extensionInstanceName; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof Extension)) { + return false; + } + Extension extension = (Extension) o; + return Objects.equals(spi, extension.spi) && Objects.equals(extensionInstanceName, extension.extensionInstanceName); + } + + @Override + public int hashCode() { + return Objects.hash(spi, extensionInstanceName); + } + } } diff --git a/eventmesh-spi/src/main/java/org/apache/eventmesh/spi/EventMeshExtensionType.java b/eventmesh-spi/src/main/java/org/apache/eventmesh/spi/EventMeshExtensionType.java index 1de315b97a..49daf4e31a 100644 --- a/eventmesh-spi/src/main/java/org/apache/eventmesh/spi/EventMeshExtensionType.java +++ b/eventmesh-spi/src/main/java/org/apache/eventmesh/spi/EventMeshExtensionType.java @@ -29,7 +29,9 @@ public enum EventMeshExtensionType { PROTOCOL("protocol"), METRICS("metrics"), TRACE("trace"), - + JDBC_CDC_ENGINE("jdbc_cdc_engine"), + JDBC_SNAPSHOT_ENGINE("jdbc_snapshot_engine"), + JDBC_DATABASE_DIALECT("jdbc_database_dialect"), OFFSETMGMT("offsetMgmt"), ; diff --git a/eventmesh-spi/src/test/java/org/apache/eventmesh/spi/EventMeshExtensionFactoryTest.java b/eventmesh-spi/src/test/java/org/apache/eventmesh/spi/EventMeshExtensionFactoryTest.java index 55b3156c78..e374fc7969 100644 --- a/eventmesh-spi/src/test/java/org/apache/eventmesh/spi/EventMeshExtensionFactoryTest.java +++ b/eventmesh-spi/src/test/java/org/apache/eventmesh/spi/EventMeshExtensionFactoryTest.java @@ -17,6 +17,7 @@ package org.apache.eventmesh.spi; +import org.apache.eventmesh.spi.example.TestAnotherSingletonExtension; import org.apache.eventmesh.spi.example.TestPrototypeExtension; import org.apache.eventmesh.spi.example.TestSingletonExtension; @@ -30,6 +31,14 @@ public void testGetSingletonExtension() { TestSingletonExtension extensionA = EventMeshExtensionFactory.getExtension(TestSingletonExtension.class, "singletonExtension"); TestSingletonExtension extensionB = EventMeshExtensionFactory.getExtension(TestSingletonExtension.class, "singletonExtension"); Assert.assertSame(extensionA, extensionB); + + TestAnotherSingletonExtension singletonExtension = EventMeshExtensionFactory.getExtension(TestAnotherSingletonExtension.class, + "singletonExtension"); + Assert.assertNotNull(singletonExtension); + TestSingletonExtension singletonExtension1 = EventMeshExtensionFactory.getExtension(TestSingletonExtension.class, "singletonExtension"); + Assert.assertNotNull(singletonExtension1); + + } @Test diff --git a/eventmesh-spi/src/test/java/org/apache/eventmesh/spi/example/AnotherSingletonExtension.java b/eventmesh-spi/src/test/java/org/apache/eventmesh/spi/example/AnotherSingletonExtension.java new file mode 100644 index 0000000000..4663204e17 --- /dev/null +++ b/eventmesh-spi/src/test/java/org/apache/eventmesh/spi/example/AnotherSingletonExtension.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.spi.example; + + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class AnotherSingletonExtension implements TestAnotherSingletonExtension { + + @Override + public void hello() { + log.info("I am SingletonExtension"); + } +} diff --git a/eventmesh-spi/src/test/java/org/apache/eventmesh/spi/example/TestAnotherSingletonExtension.java b/eventmesh-spi/src/test/java/org/apache/eventmesh/spi/example/TestAnotherSingletonExtension.java new file mode 100644 index 0000000000..92d7b1fd33 --- /dev/null +++ b/eventmesh-spi/src/test/java/org/apache/eventmesh/spi/example/TestAnotherSingletonExtension.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.spi.example; + +import org.apache.eventmesh.spi.EventMeshExtensionType; +import org.apache.eventmesh.spi.EventMeshSPI; + +/** + * TestAnotherSingletonExtension + */ +@EventMeshSPI(eventMeshExtensionType = EventMeshExtensionType.SECURITY) +public interface TestAnotherSingletonExtension { + + void hello(); +} diff --git a/eventmesh-spi/src/test/resources/META-INF/eventmesh/org.apache.eventmesh.spi.example.TestAnotherSingletonExtension b/eventmesh-spi/src/test/resources/META-INF/eventmesh/org.apache.eventmesh.spi.example.TestAnotherSingletonExtension new file mode 100644 index 0000000000..c8db961c38 --- /dev/null +++ b/eventmesh-spi/src/test/resources/META-INF/eventmesh/org.apache.eventmesh.spi.example.TestAnotherSingletonExtension @@ -0,0 +1,17 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +singletonExtension=org.apache.eventmesh.spi.example.AnotherSingletonExtension diff --git a/settings.gradle b/settings.gradle index 5b7c0ea8c2..03973a89ed 100644 --- a/settings.gradle +++ b/settings.gradle @@ -34,6 +34,7 @@ include 'eventmesh-connectors:eventmesh-connector-mongodb' include 'eventmesh-connectors:eventmesh-connector-openfunction' include 'eventmesh-connectors:eventmesh-connector-pulsar' include 'eventmesh-connectors:eventmesh-connector-kafka' +include 'eventmesh-connectors:eventmesh-connector-jdbc' include 'eventmesh-storage-plugin:eventmesh-storage-api' include 'eventmesh-storage-plugin:eventmesh-storage-standalone' include 'eventmesh-storage-plugin:eventmesh-storage-kafka'