From 9092ac03a9d3cf2045aca09cb61ab73cf5a92aaf Mon Sep 17 00:00:00 2001 From: Chunzhu Li Date: Tue, 9 Nov 2021 16:57:10 +0800 Subject: [PATCH 1/4] update kafka-max-message-size --- tidb-binlog/deploy-tidb-binlog.md | 2 ++ tidb-binlog/tidb-binlog-configuration-file.md | 1 + 2 files changed, 3 insertions(+) diff --git a/tidb-binlog/deploy-tidb-binlog.md b/tidb-binlog/deploy-tidb-binlog.md index 1de73edb0019..02d975ff70e1 100644 --- a/tidb-binlog/deploy-tidb-binlog.md +++ b/tidb-binlog/deploy-tidb-binlog.md @@ -357,6 +357,8 @@ Drainer="192.168.0.13" # kafka-addrs = "127.0.0.1:9092" # kafka-version = "0.8.2.0" # kafka-max-messages = 1024 + # 配置单条 kafka 消息的最大 size（单位为 Byte），默认为 1G，最大配置 2G + # kafka-max-message-size = 1073741824 # 保存 binlog 数据的 Kafka 集群的 topic 名称，默认值为 _obinlog # 如果运行多个 Drainer 同步数据到同一个 Kafka 集群，每个 Drainer 的 topic-name 需要设置不同的名称 diff --git a/tidb-binlog/tidb-binlog-configuration-file.md b/tidb-binlog/tidb-binlog-configuration-file.md index 8ffc64f3585e..13361ea5b246 100644 --- a/tidb-binlog/tidb-binlog-configuration-file.md +++ b/tidb-binlog/tidb-binlog-configuration-file.md @@ -329,6 +329,7 @@ tbl-name = "~^a.*" * `kafka-addrs` * `kafka-version` * `kafka-max-messages` +* `kafka-max-message-size` * `topic-name` ### syncer.to.checkpoint From 72dc56fccaaf3c62b7c83ccf231d50fc02d4ff18 Mon Sep 17 00:00:00 2001 From: TomShawn <41534398+TomShawn@users.noreply.github.com> Date: Wed, 10 Nov 2021 10:15:32 +0800 Subject: [PATCH 2/4] Update tidb-binlog/deploy-tidb-binlog.md --- tidb-binlog/deploy-tidb-binlog.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tidb-binlog/deploy-tidb-binlog.md b/tidb-binlog/deploy-tidb-binlog.md index 02d975ff70e1..804bd2444c31 100644 --- a/tidb-binlog/deploy-tidb-binlog.md +++ b/tidb-binlog/deploy-tidb-binlog.md @@ -357,7 +357,7 @@ Drainer="192.168.0.13" # kafka-addrs = "127.0.0.1:9092" # kafka-version = "0.8.2.0" # kafka-max-messages = 1024 - # 配置单条 kafka 消息的最大 size（单位为 Byte），默认为 1G，最大配置 2G + # 配置单条 Kafka 消息的最大 size（单位为 Byte），默认为 1 GB，最大可配置为 2 GB # kafka-max-message-size = 1073741824 # 保存 binlog 数据的 Kafka 集群的 topic 名称，默认值为 _obinlog From 8708b86c9fe7cf6cb76d61cb2e74ab36ef597883 Mon Sep 17 00:00:00 2001 From: Chunzhu Li Date: Wed, 10 Nov 2021 15:03:17 +0800 Subject: [PATCH 3/4] address comments --- tidb-binlog/deploy-tidb-binlog.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tidb-binlog/deploy-tidb-binlog.md b/tidb-binlog/deploy-tidb-binlog.md index 804bd2444c31..a3692c7b5ac4 100644 --- a/tidb-binlog/deploy-tidb-binlog.md +++ b/tidb-binlog/deploy-tidb-binlog.md @@ -356,8 +356,9 @@ Drainer="192.168.0.13" # zookeeper-addrs = "127.0.0.1:2181" # kafka-addrs = "127.0.0.1:9092" # kafka-version = "0.8.2.0" + # 配置单条 broker request 中的最大 message 数（即 binlog 数），不配置或配置小于等于 0 的值时会使用默认值 1024 # kafka-max-messages = 1024 - # 配置单条 Kafka 消息的最大 size（单位为 Byte），默认为 1 GB，最大可配置为 2 GB + # 配置单条 broker request 的最大 size（单位为 Byte），默认为 1 GiB，最大可配置为 2 GiB # kafka-max-message-size = 1073741824 # 保存 binlog 数据的 Kafka 集群的 topic 名称，默认值为 _obinlog From 67f848630ba657eb48889c2be2c80ebac30bda21 Mon Sep 17 00:00:00 2001 From: Chunzhu Li Date: Thu, 25 Nov 2021 21:18:48 -0600 Subject: [PATCH 4/4] Update tidb-binlog/deploy-tidb-binlog.md Co-authored-by: Liuxiaozhen12 <82579298+Liuxiaozhen12@users.noreply.github.com> --- tidb-binlog/deploy-tidb-binlog.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tidb-binlog/deploy-tidb-binlog.md b/tidb-binlog/deploy-tidb-binlog.md index a3692c7b5ac4..989dc81704fc 100644 --- a/tidb-binlog/deploy-tidb-binlog.md +++ b/tidb-binlog/deploy-tidb-binlog.md @@ -356,7 +356,7 @@ Drainer="192.168.0.13" # zookeeper-addrs = "127.0.0.1:2181" # kafka-addrs = "127.0.0.1:9092" # kafka-version = "0.8.2.0" - # 配置单条 broker request 中的最大 message 数（即 binlog 数），不配置或配置小于等于 0 的值时会使用默认值 1024 + # 配置单条 broker request 中的最大 message 数（即 binlog 数），不配置或配置小于等于 0 时会使用默认值 1024 # kafka-max-messages = 1024 # 配置单条 broker request 的最大 size（单位为 Byte），默认为 1 GiB，最大可配置为 2 GiB # kafka-max-message-size = 1073741824