Merge branch 'develop' into add_comments_config_file

.github/workflows/testflows-sink-connector-kafka.yml

@@ -48,17 +48,9 @@ jobs:
         working-directory: sink-connector/tests/integration
         run: echo "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@$(hostname -I | cut -d ' ' -f 1)"
 
-      - name: Install docker-compose
+      - name: Install all dependencies
         working-directory: sink-connector/tests/integration
-        run: pip3 install docker-compose==1.29.2
-
-      - name: Install testflows
-        working-directory: sink-connector/tests/integration
-        run: pip3 install testflows
-
-      - name: Install awscli
-        working-directory: sink-connector/tests/integration
-        run: pip3 install awscli
+        run: pip3 install -r requirements.txt
 
       - name: Get current date
         id: date

.github/workflows/testflows-sink-connector-lightweight.yml

@@ -50,7 +50,7 @@ jobs:
 
       - name: Install all dependencies
         working-directory: sink-connector-lightweight/tests/integration
-        run: pip install -r requirements.txt
+        run: pip3 install -r requirements.txt
 
       - name: Get current date
         id: date

README.md

@@ -4,6 +4,7 @@
 <a href="https://join.slack.com/t/altinitydbworkspace/shared_invite/zt-w6mpotc1-fTz9oYp0VM719DNye9UvrQ">
   <img src="https://img.shields.io/static/v1?logo=slack&logoColor=959DA5&label=Slack&labelColor=333a41&message=join%20conversation&color=3AC358" alt="AltinityDB Slack" />
 </a>
+<img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/altinityinfra/clickhouse-sink-connector">
 
 # Altinity Sink Connector for ClickHouse
 
@@ -25,7 +26,8 @@ for analysis.
 * Multiple deployment models
   * Lightweight: single process that transfers from source to target (prod)
   * Kafka: separate source and target processes using Kafka as transport (experimental)
-* Distribution as Docker container
+* Distribution as [Docker](https://hub.docker.com/layers/altinityinfra/clickhouse-sink-connector/408-97b1d3d83ef93c1b76a2b1c4d9c544dc67fbbec3-lt/images/sha256-d134bc05e50df7f63025e776ab6e3216c6622cd159eb0f2d459ea2ce8975f396?context=explore)
+ container
 
 ## Getting Started
 

doc/quickstart.md

@@ -31,7 +31,7 @@ sudo apt install clickhouse-client
 Use Docker Compose to start containers. 
 ```
 cd sink-connector-lightweight/docker
-export SINK_LIGHTWEIGHT_VERSION=latest
+export CLICKHOUSE_SINK_CONNECTOR_LT_IMAGE=altinityinfra/clickhouse-sink-connector:408-97b1d3d83ef93c1b76a2b1c4d9c544dc67fbbec3-lt
 docker compose -f docker-compose-mysql.yml up --renew-anon-volumes
 ```
 

sink-connector-lightweight/docker/clickhouse-service.yml

@@ -0,0 +1,21 @@
+version: "3.4"
+
+services:
+  clickhouse:
+    container_name: clickhouse
+    image: clickhouse/clickhouse-server:latest
+    restart: "no"
+    ports:
+      - "8123:8123"
+      - "9000:9000"
+    environment:
+      - CLICKHOUSE_USER=root
+      - CLICKHOUSE_PASSWORD=root
+      - CLICKHOUSE_DB=test
+      - CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=0
+    ulimits:
+      nofile:
+        soft: "262144"
+        hard: "262144"
+    volumes:
+      - ../clickhouse/users.xml:/etc/clickhouse-server/users.xml

sink-connector-lightweight/docker/clickhouse-sink-connector-lt-service.yml

@@ -0,0 +1,15 @@
+version: "3.4"
+
+services:
+  clickhouse-sink-connector-lt:
+    image: ${CLICKHOUSE_SINK_CONNECTOR_LT_IMAGE}
+    entrypoint: ["sh", "-c", "java -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005 -Xms4g -Xmx4g -jar /app.jar /config.yml com.altinity.clickhouse.debezium.embedded.ClickHouseDebeziumEmbeddedApplication"]
+    restart: "no"
+    ports:
+      - "8083:8083"
+      - "5005:5005"
+      - "7000:7000"
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    volumes:
+      - ./config.yml:/config.yml

sink-connector-lightweight/docker/config.yml

@@ -133,4 +133,13 @@ auto.create.tables: "true"
 #disable.ddl: "false"
 
 #disable.drop.truncate: If set to true, the connector will ignore drop and truncate events. The default is false.
-#disable.drop.truncate: "false"
+#disable.drop.truncate: "false"
+
+# restart.event.loop: This will restart the CDC event loop if there are no messages received after timeout specified in restart.event.loop.timeout.period.secs
+#restart.event.loop: "true"
+
+# restart.event.loop.timeout.period.secs: Defines the restart timeout period.
+#restart.event.loop.timeout.period.secs: "3000"
+
+# Max number of records for the flush buffer.
+#buffer.max.records: "10000"

sink-connector-lightweight/docker/config_postgres.yml

@@ -73,7 +73,14 @@ offset.storage.jdbc.password: "root"
 offset.storage.jdbc.offset.table.ddl: "CREATE TABLE if not exists %s
 (
     `id` String,
-	@@ -35,13 +37,14 @@ ORDER BY id
+    `offset_key` String,
+    `offset_val` String,
+    `record_insert_ts` DateTime,
+    `record_insert_seq` UInt64,
+    `_version` UInt64 MATERIALIZED toUnixTimestamp64Nano(now64(9))
+)
+ENGINE = ReplacingMergeTree(_version)
+ORDER BY id
 SETTINGS index_granularity = 8198"
 offset.storage.jdbc.offset.table.delete: "delete from %s where 1=1"
 schema.history.internal: "io.debezium.storage.jdbc.history.JdbcSchemaHistory"

sink-connector-lightweight/docker/docker-compose-mariadb.yml

@@ -8,102 +8,42 @@ version: "3.4"
 
 services:
   mysql-master:
-    container_name: mysql-master
-    command: --transaction-isolation=READ-COMMITTED --log-bin=binlog --binlog-format=ROW
-    image: mariadb
-    restart: "no"
-    ports:
-      - "3306:3306"
-    environment:
-      - MYSQL_ROOT_PASSWORD=root
-      - MYSQL_DATABASE=test
-      - MYSQL_REPLICATION_MODE=master
-      - MYSQL_REPLICATION_USER=repl_user
-      - ALLOW_EMPTY_PASSWORD=yes
-      - MYSQL_BINLOG_FORMAT=row
-    volumes:
-      - ./mysqld.cnf:/opt/bitnami/mysql/conf/my_custom.cnf
-      - ../sql/init_mariadb.sql:/docker-entrypoint-initdb.d/init_mysql.sql
-    healthcheck:
-      test: [ 'CMD', '/opt/bitnami/scripts/mysql/healthcheck.sh' ]
-      interval: 15s
-      timeout: 5s
-      retries: 6
+    extends:
+      file: mysql-master-service.yml
+      service: mysql-master
 
   clickhouse:
-    container_name: clickhouse
-    image: clickhouse/clickhouse-server:latest
-    restart: "no"
-    ports:
-      - "8123:8123"
-      - "9000:9000"
-    environment:
-      - CLICKHOUSE_USER=root
-      - CLICKHOUSE_PASSWORD=root
-      - CLICKHOUSE_DB=test
-      - CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=0
-    ulimits:
-      nofile:
-        soft: "262144"
-        hard: "262144"
-    volumes:
-      -  ../clickhouse/users.xml:/etc/clickhouse-server/users.xml
+    extends:
+      file: clickhouse-service.yml
+      service: clickhouse
     depends_on:
       zookeeper:
         condition: service_healthy
 
   zookeeper:
-    image: zookeeper:3.6.2
-    expose:
-      - "2181"
-    environment:
-      ZOO_TICK_TIME: 500
-      ZOO_MY_ID: 1
-    healthcheck:
-      test: echo stat | nc localhost 2181
-      interval: 3s
-      timeout: 2s
-      retries: 5
-      start_period: 2s
-    security_opt:
-      - label:disable
+    extends:
+      file: zookeeper-service.yml
+      service: zookeeper
 
-  debezium-embedded:
-    image: registry.gitlab.com/altinity-public/container-images/clickhouse_debezium_embedded:${SINK_LIGHTWEIGHT_VERSION}
-    entrypoint: ["sh", "-c", "java -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005 -Xms4g -Xmx4g -jar /app.jar /config.yml com.altinity.clickhouse.debezium.embedded.ClickHouseDebeziumEmbeddedApplication"]
-    restart: "no"
-    ports:
-      - "8083:8083"
-      - "5005:5005"
-      - "7000:7000"
-    extra_hosts:
-      - "host.docker.internal:host-gateway"
-    volumes:
-      - ./config.yml:/config.yml
+  clickhouse-sink-connector-lt:
+    extends:
+      file: clickhouse-sink-connector-lt-service.yml
+      service: clickhouse-sink-connector-lt
 
   ### MONITORING ####
   prometheus:
-    container_name: prometheus
-    image: bitnami/prometheus:2.36.0
-    restart: "no"
-    ports:
-      - "9090:9090"
-    volumes:
-      - ./config/prometheus.yml:/opt/bitnami/prometheus/conf/prometheus.yml
+    extends:
+      file: prometheus-service.yml
+      service: prometheus
 
   grafana:
-    build:
-      context: ./config/grafana
-      args:
-        GRAFANA_VERSION: latest
-    restart: "no"
-    ports:
-      - "3000:3000"
-    environment:
-      - DS_PROMETHEUS=prometheus
-      - GF_USERS_DEFAULT_THEME=light
-      - GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS=vertamedia-clickhouse-datasource,grafana-clickhouse-datasource
-      - GF_INSTALL_PLUGINS=vertamedia-clickhouse-datasource,grafana-clickhouse-datasource
+    extends:
+      file: grafana-service.yml
+      service: grafana
+    volumes:
+      - ./config/grafana/config/dashboard.yml:/etc/grafana/provisioning/dashboards/dashboard.yml
+      - ./config/grafana/config/datasource.yml:/etc/grafana/provisioning/datasources/datasource.yml
+      - ./config/grafana/config/altinity_sink_connector.json:/var/lib/grafana/dashboards/altinity_sink_connector.json
     depends_on:
       - prometheus
     ## END OF MONITORING ###

sink-connector-lightweight/docker/docker-compose-mysql-standalone.yml

@@ -1,39 +1,23 @@
 services:
-  debezium-embedded:
-    image: registry.gitlab.com/altinity-public/container-images/clickhouse_debezium_embedded:${SINK_LIGHTWEIGHT_VERSION}
-    entrypoint: ["sh", "-c", "java -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005 -Xms4g -Xmx4g -jar /app.jar /config.yml com.altinity.clickhouse.debezium.embedded.ClickHouseDebeziumEmbeddedApplication"]
-    restart: "no"
-    ports:
-      - "8083:8083"
-      - "5005:5005"
-    extra_hosts:
-      - "host.docker.internal:host-gateway"
-    volumes:
-      - ./config.yml:/config.yml
+  clickhouse-sink-connector-lt:
+    extends:
+      file: clickhouse-sink-connector-lt-service.yml
+      service: clickhouse-sink-connector-lt
 
   ### MONITORING ####
   prometheus:
-    container_name: prometheus
-    image: bitnami/prometheus:2.36.0
-    restart: "no"
-    ports:
-      - "9090:9090"
-    volumes:
-      - ./config/prometheus.yml:/opt/bitnami/prometheus/conf/prometheus.yml
+    extends:
+      file: prometheus-service.yml
+      service: prometheus
 
   grafana:
-    build:
-      context: ./config/grafana
-      args:
-        GRAFANA_VERSION: latest
-    restart: "no"
-    ports:
-      - "3000:3000"
-    environment:
-      - DS_PROMETHEUS=prometheus
-      - GF_USERS_DEFAULT_THEME=light
-      - GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS=vertamedia-clickhouse-datasource,grafana-clickhouse-datasource
-      - GF_INSTALL_PLUGINS=vertamedia-clickhouse-datasource,grafana-clickhouse-datasource
+    extends:
+      file: grafana-service.yml
+      service: grafana
+    volumes:
+      - ./config/grafana/config/dashboard.yml:/etc/grafana/provisioning/dashboards/dashboard.yml
+      - ./config/grafana/config/datasource.yml:/etc/grafana/provisioning/datasources/datasource.yml
+      - ./config/grafana/config/altinity_sink_connector.json:/var/lib/grafana/dashboards/altinity_sink_connector.json
     depends_on:
       - prometheus
     ## END OF MONITORING ###