diff --git a/sink-connector-lightweight/docker/config.yml b/sink-connector-lightweight/docker/config.yml index 8a4778445..e6a5e4746 100644 --- a/sink-connector-lightweight/docker/config.yml +++ b/sink-connector-lightweight/docker/config.yml @@ -1,25 +1,75 @@ +#### Some of the properties are part of Debezium MYSQL Connector +#### https://debezium.io/documentation/reference/stable/connectors/mysql.html#mysql-connector-properties + +# Unique name for the connector. Attempting to register again with the same name will fail. name: "company-1" + +# IP address or hostname of the MySQL database server. database.hostname: "mysql-master" + +# Integer port number of the MySQL database server listening for client connections. database.port: "3306" + +# Name of the MySQL database user to be used when connecting to the database. database.user: "root" + +# Password of the MySQL database user to be used when connecting to the database. database.password: "root" + +# The name of the MySQL database from which events are to be captured when not using snapshot mode. database.server.name: "ER54" + +# database.include.list An optional list of regular expressions that match database names to be monitored; +# any database name not included in the whitelist will be excluded from monitoring. By default all databases will be monitored. database.include.list: test -#table.include.list=sbtest1 + +# table.include.list An optional list of regular expressions that match fully-qualified table identifiers for tables to be monitored; +table.include.list: "" + +# Clickhouse Server URL, Specify only the hostname. clickhouse.server.url: "clickhouse" + +# Clickhouse Server User clickhouse.server.user: "root" + +#Clickhouse Server Password clickhouse.server.password: "root" + +# Clickhouse Server Port clickhouse.server.port: "8123" + +# Clickhouse Server Database clickhouse.server.database: "test" + +# database.allowPublicKeyRetrieval: "true" https://rmoff.net/2019/10/23/debezium-mysql-v8-public-key-retrieval-is-not-allowed/ database.allowPublicKeyRetrieval: "true" + +# snapshot.mode: Debezium can use different modes when it runs a snapshot. The snapshot mode is determined by the snapshot.mode configuration property. +# The default value of the property is initial. You can customize the way that the connector creates snapshots by changing the value of the snapshot.mode property snapshot.mode: "initial" + +# offset.flush.interval.ms: The number of milliseconds to wait before flushing recent offsets to Kafka. This ensures that offsets are committed within the specified time interval. offset.flush.interval.ms: 5000 + +# connector.class: The Java class for the connector. This must be set to io.debezium.connector.mysql.MySqlConnector. connector.class: "io.debezium.connector.mysql.MySqlConnector" + +# offset.storage: The Java class that implements the offset storage strategy. This must be set to io.debezium.storage.jdbc.offset.JdbcOffsetBackingStore. offset.storage: "io.debezium.storage.jdbc.offset.JdbcOffsetBackingStore" + +# offset.storage.jdbc.offset.table.name: The name of the database table where connector offsets are to be stored. offset.storage.jdbc.offset.table.name: "altinity_sink_connector.replica_source_info" + +# offset.storage.jdbc.url: The JDBC URL for the database where connector offsets are to be stored. offset.storage.jdbc.url: "jdbc:clickhouse://clickhouse:8123/altinity_sink_connector" + +# offset.storage.jdbc.user: The name of the database user to be used when connecting to the database where connector offsets are to be stored. offset.storage.jdbc.user: "root" + +# offset.storage.jdbc.password: The password of the database user to be used when connecting to the database where connector offsets are to be stored. offset.storage.jdbc.password: "root" + +# offset.storage.jdbc.offset.table.ddl: The DDL statement used to create the database table where connector offsets are to be stored.(Advanced) offset.storage.jdbc.offset.table.ddl: "CREATE TABLE if not exists %s ( `id` String, @@ -32,22 +82,64 @@ offset.storage.jdbc.offset.table.ddl: "CREATE TABLE if not exists %s ENGINE = ReplacingMergeTree(_version) ORDER BY id SETTINGS index_granularity = 8198" + +# offset.storage.jdbc.offset.table.delete: The DML statement used to delete the database table where connector offsets are to be stored.(Advanced) offset.storage.jdbc.offset.table.delete: "delete from %s where 1=1" + +# schema.history.internal: The Java class that implements the schema history strategy. This must be set to io.debezium.storage.jdbc.history.JdbcSchemaHistory. schema.history.internal: "io.debezium.storage.jdbc.history.JdbcSchemaHistory" + +# schema.history.internal.jdbc.url: The JDBC URL for the database where connector schema history is to be stored. schema.history.internal.jdbc.url: "jdbc:clickhouse://clickhouse:8123/altinity_sink_connector" + +# schema.history.internal.jdbc.user: The name of the database user to be used when connecting to the database where connector schema history is to be stored. schema.history.internal.jdbc.user: "root" + +# schema.history.internal.jdbc.password: The password of the database user to be used when connecting to the database where connector schema history is to be stored. schema.history.internal.jdbc.password: "root" + +# schema.history.internal.jdbc.schema.history.table.ddl: The DDL statement used to create the database table where connector schema history is to be stored.(Advanced) schema.history.internal.jdbc.schema.history.table.ddl: "CREATE TABLE if not exists %s (`id` VARCHAR(36) NOT NULL, `history_data` VARCHAR(65000), `history_data_seq` INTEGER, `record_insert_ts` TIMESTAMP NOT NULL, `record_insert_seq` INTEGER NOT NULL) ENGINE=ReplacingMergeTree(record_insert_seq) order by id" +# schema.history.internal.jdbc.schema.history.table.name: The name of the database table where connector schema history is to be stored. schema.history.internal.jdbc.schema.history.table.name: "altinity_sink_connector.replicate_schema_history" + +# enable.snapshot.ddl: If set to true, the connector will parse the DDL statements from the initial load enable.snapshot.ddl: "true" + +# persist.raw.bytes: If set to true, the connector will persist raw bytes as received in a String column. persist.raw.bytes: "false" + +# auto.create.tables: If set to true, the connector will create tables in the target based on the schema received in the incoming message. auto.create.tables: "true" + +# database.connectionTimeZone: The timezone of the MySQL database server used to correctly shift the commit transaction timestamp. #database.connectionTimeZone: "US/Samoa" + +# clickhouse.datetime.timezone: This timezone will override the default timezone of ClickHouse server. Timezone columns will be set to this timezone. #clickhouse.datetime.timezone: "UTC" + +# skip_replica_start: If set to true, the connector will skip replication on startup. sink-connector-client start_replica will start replication. #skip_replica_start: "false" + +# binary.handling.mode: The mode for handling binary values. Possible values are bytes, base64, and decode. The default is bytes. #binary.handling.mode: "base64" + +# ignore_delete: If set to true, the connector will ignore delete events. The default is false. +#ignore_delete: "true" + +#disable.ddl: If set to true, the connector will ignore DDL events. The default is false. +#disable.ddl: "false" + +#disable.drop.truncate: If set to true, the connector will ignore drop and truncate events. The default is false. +#disable.drop.truncate: "false" + +# restart.event.loop: This will restart the CDC event loop if there are no messages received after timeout specified in restart.event.loop.timeout.period.secs #restart.event.loop: "true" + +# restart.event.loop.timeout.period.secs: Defines the restart timeout period. #restart.event.loop.timeout.period.secs: "3000" + +# Max number of records for the flush buffer. #buffer.max.records: "10000" diff --git a/sink-connector-lightweight/docker/config_postgres.yml b/sink-connector-lightweight/docker/config_postgres.yml index 392564c2f..c903d58bd 100644 --- a/sink-connector-lightweight/docker/config_postgres.yml +++ b/sink-connector-lightweight/docker/config_postgres.yml @@ -1,26 +1,75 @@ +#### Some of the properties are part of Debezium PostgreSQL Connector +#### https://debezium.io/documentation/reference/stable/connectors/postgresql.html +# name: Unique name for the connector. Attempting to register again with the same name will fail. name: "debezium-embedded-postgres" + +# database.hostname: IP address or hostname of the PostgreSQL database server. database.hostname: "postgres" + +# database.port: Integer port number of the PostgreSQL database server listening for client connections. database.port: "5432" + +# database.user: Name of the PostgreSQL database user to be used when connecting to the database. database.user: "root" + +# database.password: Password of the PostgreSQL database user to be used when connecting to the database. database.password: "root" + +# database.server.name: The name of the PostgreSQL database from which events are to be captured when not using snapshot mode. database.server.name: "ER54" + +# schema.include.list: An optional list of regular expressions that match schema names to be monitored; schema.include.list: public + +# plugin.name: The name of the PostgreSQL logical decoding plug-in installed on the PostgreSQL server. Supported values are decoderbufs, and pgoutput. plugin.name: "pgoutput" + +# table.include.list: An optional list of regular expressions that match fully-qualified table identifiers for tables to be monitored; table.include.list: "public.tm,public.tm2" + +# clickhouse.server.url: Specify only the hostname of the Clickhouse Server. clickhouse.server.url: "clickhouse" + +# clickhouse.server.user: Clickhouse Server User clickhouse.server.user: "root" + +# clickhouse.server.password: Clickhouse Server Password clickhouse.server.password: "root" + +# clickhouse.server.port: Clickhouse Server Port clickhouse.server.port: "8123" + +# clickhouse.server.database: Clickhouse Server Database clickhouse.server.database: "test" + +# database.allowPublicKeyRetrieval: "true" https://rmoff.net/2019/10/23/debezium-mysql-v8-public-key-retrieval-is-not-allowed/ database.allowPublicKeyRetrieval: "true" + +# snapshot.mode: Debezium can use different modes when it runs a snapshot. The snapshot mode is determined by the snapshot.mode configuration property. snapshot.mode: "initial" + +# offset.flush.interval.ms: The number of milliseconds to wait before flushing recent offsets to Kafka. This ensures that offsets are committed within the specified time interval. offset.flush.interval.ms: 5000 + +# connector.class: The Java class for the connector. This must be set to io.debezium.connector.postgresql.PostgresConnector. connector.class: "io.debezium.connector.postgresql.PostgresConnector" + +# offset.storage: The Java class that implements the offset storage strategy. This must be set to io.debezium.storage.jdbc.offset.JdbcOffsetBackingStore. offset.storage: "io.debezium.storage.jdbc.offset.JdbcOffsetBackingStore" + +# offset.storage.jdbc.offset.table.name: The name of the database table where connector offsets are to be stored. offset.storage.jdbc.offset.table.name: "altinity_sink_connector.replica_source_info" + +# offset.storage.jdbc.url: The JDBC URL for the database where connector offsets are to be stored. offset.storage.jdbc.url: "jdbc:clickhouse://clickhouse:8123/altinity_sink_connector" + +# offset.storage.jdbc.user: The name of the database user to be used when connecting to the database where connector offsets are to be stored. offset.storage.jdbc.user: "root" + +# offset.storage.jdbc.password: The password of the database user to be used when connecting to the database where connector offsets are to be stored. offset.storage.jdbc.password: "root" + +# offset.storage.jdbc.offset.table.ddl: The DDL statement used to create the database table where connector offsets are to be stored. offset.storage.jdbc.offset.table.ddl: "CREATE TABLE if not exists %s ( `id` String, @@ -41,7 +90,32 @@ schema.history.internal.jdbc.password: "root" schema.history.internal.jdbc.schema.history.table.ddl: "CREATE TABLE if not exists %s (`id` VARCHAR(36) NOT NULL, `history_data` VARCHAR(65000), `history_data_seq` INTEGER, `record_insert_ts` TIMESTAMP NOT NULL, `record_insert_seq` INTEGER NOT NULL) ENGINE=ReplacingMergeTree(record_insert_seq) order by id" +# schema.history.internal.schema.history.table.name: The name of the database table where connector schema history is to be stored. schema.history.internal.jdbc.schema.history.table.name: "altinity_sink_connector.replicate_schema_history" + +# enable.snapshot.ddl: If set to true, the connector wil parse the DDL statements as part of initial load. enable.snapshot.ddl: "true" + +# auto.create.tables: If set to true, the connector will create the database tables for the destination tables if they do not already exist. auto.create.tables: "true" -database.dbname: "public" \ No newline at end of file + +# database.dbname: The name of the PostgreSQL database from which events are to be captured when not using snapshot mode. +database.dbname: "public" + +# clickhouse.datetime.timezone: This timezone will override the default timezone of ClickHouse server. Timezone columns will be set to this timezone. +#clickhouse.datetime.timezone: "UTC" + +# skip_replica_start: If set to true, the connector will skip replication on startup. sink-connector-client start_replica will start replication. +#skip_replica_start: "false" + +# binary.handling.mode: The mode for handling binary values. Possible values are bytes, base64, and decode. The default is bytes. +#binary.handling.mode: "base64" + +# ignore_delete: If set to true, the connector will ignore delete events. The default is false. +#ignore_delete: "true" + +#disable.ddl: If set to true, the connector will ignore DDL events. The default is false. +#disable.ddl: "false" + +#disable.drop.truncate: If set to true, the connector will ignore drop and truncate events. The default is false. +#disable.drop.truncate: "false" \ No newline at end of file