diff --git a/sink-connector-lightweight/tests/integration/requirements/requirements.md b/sink-connector-lightweight/tests/integration/requirements/requirements.md index 854608831..57699139a 100644 --- a/sink-connector-lightweight/tests/integration/requirements/requirements.md +++ b/sink-connector-lightweight/tests/integration/requirements/requirements.md @@ -63,6 +63,7 @@ * 12.1 [Inserts](#inserts) * 12.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Queries.Inserts](#rqsrs-030clickhousemysqltoclickhousereplicationqueriesinserts) * 12.1.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Queries.Inserts.PartitionLimits](#rqsrs-030clickhousemysqltoclickhousereplicationqueriesinsertspartitionlimits) + * 12.1.1.2 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Queries.Inserts.PartitionTypes](#rqsrs-030clickhousemysqltoclickhousereplicationqueriesinsertspartitiontypes) * 12.2 [Updates](#updates) * 12.2.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Queries.Updates](#rqsrs-030clickhousemysqltoclickhousereplicationqueriesupdates) * 12.3 [Deletes](#deletes) @@ -176,8 +177,21 @@ * 28 [Replication Interruption](#replication-interruption) * 28.1 [Retry Replication When ClickHouse Instance Is Not Active](#retry-replication-when-clickhouse-instance-is-not-active) * 28.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Interruption.ClickHouse.Instance.Stopped](#rqsrs-030clickhousemysqltoclickhousereplicationinterruptionclickhouseinstancestopped) -* 29 [Prometheus ](#prometheus-) - * 29.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Prometheus](#rqsrs-030clickhousemysqltoclickhousereplicationprometheus) +* 29 [System Actions](#system-actions) + * 29.1 [Handling Network Interruptions](#handling-network-interruptions) + * 29.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.SystemActions.Network](#rqsrs-030clickhousemysqltoclickhousereplicationsystemactionsnetwork) + * 29.2 [Handling Process Interruptions](#handling-process-interruptions) + * 29.2.1 [Behaviour When Different Processes Were Killed](#behaviour-when-different-processes-were-killed) + * 29.2.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.SystemActions.Process.Die](#rqsrs-030clickhousemysqltoclickhousereplicationsystemactionsprocessdie) + * 29.2.2 [Behaviour When Different Processes Were Restarted](#behaviour-when-different-processes-were-restarted) + * 29.2.2.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.SystemActions.Process.Restarted](#rqsrs-030clickhousemysqltoclickhousereplicationsystemactionsprocessrestarted) + * 29.3 [Behaviour When There Are Issues With Disk](#behaviour-when-there-are-issues-with-disk) + * 29.3.1 [Disk Is out of Space](#disk-is-out-of-space) + * 29.3.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.SystemActions.Disk.OutOfSpace](#rqsrs-030clickhousemysqltoclickhousereplicationsystemactionsdiskoutofspace) + * 29.3.2 [Disk Is Corrupted](#disk-is-corrupted) + * 29.3.2.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.SystemActions.Disk.Corrupted](#rqsrs-030clickhousemysqltoclickhousereplicationsystemactionsdiskcorrupted) +* 30 [Prometheus](#prometheus) + * 30.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Prometheus](#rqsrs-030clickhousemysqltoclickhousereplicationprometheus) ## Introduction @@ -197,187 +211,232 @@ Software Requirements Specification ## Test Schema ```yaml -Services: - - Source Database Cluster - - Sink Connector - - ClickHouse Database Cluster - -SourceTables: - DatabaseType: [MySQL, PostgreSQL, MariaDB, MongoDB] - DatabaseVersions: - MySQL: [8.0] - PostgreSQL: null - MariaDB: null - MongoDB: null - DatabaseClusterConfiguration: - MySQL: null - PostgreSQL: null - MariaDB: null - MongoDB: null - EngineType: - MySQL: [InnoDB] - PostgreSQL: null - MariaDB: null - MongoDB: null - Schema: - TableName: - length: - MySQL: 64 characters - PostgreSQL: null - MariaDB: null - MongoDB: null - encodings: - ASCII: null - UTF-8: null - otherEncodings: [armscii8, big5, binary, cp1250, cp1251, cp1256, cp1257, cp850, cp852, cp866, cp932, dec8, - eucjpms, euckr, gb18030, gb2312, gbk, geostd8, greek, hebrew, hp8, keybcs2, koi8r, koi8u, latin1, - latin2, latin5, latin7, macce, macroman, sjis, swe7, tis620, ucs2, ujis, utf16, utf16le, utf32, utf8mb3, utf8mb4] - Partitioning: - MySQL: [RANGE, LIST, COLUMNS, HASH, KEY, Subpartitioning] - Columns: - DefaultValues: [Numeric Types, Date and Time Types, String Types, ENUM Types, SET Types, BOOLEAN, Binary Types] - Type: - MySQL: [Calculated columns, Materialized columns, Primary Key Columns, Foreign Key Columns, Index Columns, Unique Columns, Auto-Increment Columns, Timestamp/DateTime Columns, ENUM and SET Columns, Spatial Columns] - Name: - length: - mysql: null - PostgreSQL: null - MariaDB: null - MongoDB: null - ASCII: null - UTF-8: null - otherEncodings: [armscii8, big5, binary, cp1250, cp1251, cp1256, cp1257, cp850, cp852, cp866, cp932, dec8, - eucjpms, euckr, gb18030, gb2312, gbk, geostd8, greek, hebrew, hp8, keybcs2, koi8r, koi8u, latin1, - latin2, latin5, latin7, macce, macroman, sjis, swe7, tis620, ucs2, ujis, utf16, utf16le, utf32, utf8mb3, utf8mb4] - DataType: - - DECIMAL(2,1) - - DECIMAL(30, 10) - - DOUBLE - - DATE - - DATETIME(1-6) - - TIME - - TIME(1-6) - - INT - - INT - - INT UNSIGNED - - INT UNSIGNED - - BIGINT - - BIGINT - - BIGINT UNSIGNED NOT NULL - - TINYINT - - TINYINT UNSIGNED - - SMALLINT - - SMALLINT UNSIGNED - - MEDIUMINT - - MEDIUMINT NOT NULL - - MEDIUMINT UNSIGNED - - CHAR - - TEXT - - VARCHAR(1-32766) - - BLOB - - MEDIUMBLOB - - LONGBLOB - - BINARY - - VARBINARY(4) - DataValue: - Numeric: [Min, Max, 0, -infinity, +infinity, nan, random value] - Decimal: [Min value based on precision and scale, Max value based on precision and scale, 0, -0.0001, "0.0001", - -Max value based on precision and scale, +Max value based on precision and scale, "NaN", - A random value within precision and scale] - String: - bytes: [null bytes, ...] - UTF-8: [] - ASCII: [] - otherEncodings: [armscii8, big5, binary, cp1250, cp1251, cp1256, cp1257, cp850, cp852, cp866, cp932, dec8, - eucjpms, euckr, gb18030, gb2312, gbk, geostd8, greek, hebrew, hp8, keybcs2, koi8r, koi8u, latin1, - latin2, latin5, latin7, macce, macroman, sjis, swe7, tis620, ucs2, ujis, utf16, utf16le, - utf32, utf8mb3, utf8mb4] - - TableOperations: +clickhouse-sink-connector: + Services: + - Source Database Cluster + - Sink Connector + - ClickHouse Database Cluster + SourceTables: + DatabaseType: [MySQL, PostgreSQL, MariaDB, MongoDB] MySQL: - - INSERT - - UPDATE - - DELETE - - SELECT - - ALTERs: - - ADD COLUMN - - ADD COLUMN NULL/NOT NULL - - ADD COLUMN DEFAULT - - ADD COLUMN FIRST, AFTER - - DROP COLUMN - - MODIFY COLUMN data_type - - MODIFY COLUMN data_type NULL/NOT NULL - - MODIFY COLUMN data_type DEFAULT - - MODIFY COLUMN FIRST, AFTER - - MODIFY COLUMN old_name new_name datatype NULL/NOT NULL - - RENAME COLUMN col1 to col2 - - CHANGE COLUMN FIRST, AFTER - - ALTER COLUMN col_name ADD DEFAULT - - ALTER COLUMN col_name ADD DROP DEFAULT - - ADD PRIMARY KEY - MariaDB: null - PostgreSQL: null - MongoDB: null - -SinkConnector: - Version: [latest] - Configuration: The full list is inside configurations below - -DestinationTables: - DatabaseType: [ClickHouse] - DatabaseClusterConfiguration: - - One node - - Sharded cluster secure - - Cluster with Multiple Shards - - Cluster with Replication (but a single shard) - - Cluster with Multiple Shards and Replicas - - Secure Cluster - DatabaseVersion: [22.8, 23.3, 23,11, latest] - EngineType: [ReplacingMergeTree, ReplicatedReplacingMergeTree] - Schema: - TableName: - length: - MySQL: 64 characters - PostgreSQL: null - MariaDB: null - MongoDB: null - encodings: - ASCII: null - UTF-8: null - otherEncodings: [armscii8, big5, binary, cp1250, cp1251, cp1256, cp1257, cp850, cp852, cp866, cp932, dec8, - eucjpms, euckr, gb18030, gb2312, gbk, geostd8, greek, hebrew, hp8, keybcs2, koi8r, koi8u, latin1, - latin2, latin5, latin7, macce, macroman, sjis, swe7, tis620, ucs2, ujis, utf16, utf16le, utf32, utf8mb3, utf8mb4] - TableOperations: - - INSERT - - UPDATE - - DELETE - - SELECT - - ALTER - -SystemActions: - Network: - - Internal network interruptions in source database cluster - - Network interruptions from source database to sink connector - - Network interruptions from sink connector to clickhouse - - Internal network interruptions in clickhouse database cluster - Process: - Die: - - Internal processes die in source database cluster - - Sink connector dies - - Internal processes die in clickhouse database cluster - Restarted: - - Restart of some or all nodes in source database cluster - - Restart of sink connector - - Restart of some or all nodes in clickhouse database cluster - Disk: - OutOfSpace: - - Out of disk space on some node in source database cluster - - Out of disk space where sink connector is running - - Out of disk space on some node in clickhouse database cluster - Corruptions: - - Corruption on a disk used by some node in source database cluster - - Corruption on a disk where sink connector is running - - Corruption on a disk used by some node in clickhouse database cluster + DatabaseVersion: 8.0 + DatabaseClusterConfiguration: + - MySQL Cluster (NDB Cluster) + - InnoDB Cluster + - Replication Clusters + - Galera Cluster + EngineType: [InnoDB, MyISAM, MEMORY, CSV, ARCHIVE, BLACKHOLE, FEDERATED, EXAMPLE] + Schema: + TableName: + Length: 64 characters + Encodings: + - ASCII + - UTF-8 + - otherEncodings: [armscii8, big5, binary, cp1250, cp1251, cp1256, cp1257, cp850, cp852, cp866, cp932, dec8, + eucjpms, euckr, gb18030, gb2312, gbk, geostd8, greek, hebrew, hp8, keybcs2, koi8r, koi8u, latin1, + latin2, latin5, latin7, macce, macroman, sjis, swe7, tis620, ucs2, ujis, utf16, utf16le, utf32, utf8mb3, utf8mb4] + Partitioning: + - RANGE + - LIST + - COLUMNS + - HASH + - KEY + - Subpartitioning + Columns: + DefaultValues: [Numeric Types, Date and Time Types, String Types, ENUM Types, SET Types, BOOLEAN, Binary Types] + Type: [Calculated Columns, Materialized Columns, Primary Key Columns, Foreign Key Columns, + Index Columns, Unique Columns, Auto-Increment Columns, Timestamp/DateTime Columns, + ENUM and SET Columns, Spatial Columns] + Name: + Length: 64 characters + Encodings: + - ASCII + - UTF-8 + - OtherEncodings: [armscii8, big5, binary, cp1250, cp1251, cp1256, cp1257, cp850, cp852, cp866, cp932, dec8, + eucjpms, euckr, gb18030, gb2312, gbk, geostd8, greek, hebrew, hp8, keybcs2, koi8r, koi8u, latin1, + latin2, latin5, latin7, macce, macroman, sjis, swe7, tis620, ucs2, ujis, utf16, utf16le, utf32, utf8mb3, utf8mb4] + DataType: + - DECIMAL(2,1) + - DECIMAL(30, 10) + - DOUBLE + - DATE + - DATETIME(1-6) + - TIME(1-6) + - INT + - INT UNSIGNED + - BIGINT + - BIGINT UNSIGNED NOT NULL + - TINYINT + - TINYINT UNSIGNED + - SMALLINT + - SMALLINT UNSIGNED + - MEDIUMINT + - MEDIUMINT UNSIGNED + - CHAR + - TEXT + - VARCHAR(1-32766) + - BLOB + - MEDIUMBLOB + - LONGBLOB + - BINARY + - VARBINARY(4) + DataValue: + Numeric: [Min, Max, 0, -infinity, +infinity, nan, random value] + Decimal: [Min value based on precision and scale, Max value based on precision and scale, 0, -0.0001, + "0.0001", -Max value based on precision and scale, +Max value based on precision and scale, + "NaN", A random value within precision and scale] + String: + Bytes: [null bytes] + UTF-8: [C0 Controls and Basic Latin, Latin Extended-A, Spacing Modifiers, Diacritical Marks, + Greek and Coptic, Cyrillic Basic, Currency Symbols, Mathematical Operators, Miscellaneous Symbols, Dingbats] + ASCII: [All ASCII characters] + otherEncodings: [armscii8, big5, binary, cp1250, cp1251, cp1256, cp1257, cp850, cp852, cp866, cp932, dec8, + eucjpms, euckr, gb18030, gb2312, gbk, geostd8, greek, hebrew, hp8, keybcs2, koi8r, koi8u, latin1, + latin2, latin5, latin7, macce, macroman, sjis, swe7, tis620, ucs2, ujis, utf16, utf16le, + utf32, utf8mb3, utf8mb4] + TableOperations: + - INSERT + - UPDATE + - DELETE + - SELECT + - ALTER: + - ADD COLUMN + - ADD COLUMN NULL/NOT NULL + - ADD COLUMN DEFAULT + - ADD COLUMN FIRST, AFTER + - DROP COLUMN + - MODIFY COLUMN data_type + - MODIFY COLUMN data_type NULL/NOT NULL + - MODIFY COLUMN data_type DEFAULT + - MODIFY COLUMN FIRST, AFTER + - MODIFY COLUMN old_name new_name datatype NULL/NOT NULL + - RENAME COLUMN col1 to col2 + - CHANGE COLUMN FIRST, AFTER + - ALTER COLUMN col_name ADD DEFAULT + - ALTER COLUMN col_name ADD DROP DEFAULT + - ADD PRIMARY KEY + PostgreSQL: + DatabaseVersion: null + DatabaseClusterConfiguration: null + EngineType: null + Schema: + TableName: + Length: null + Encodings: null + Partitioning: null + Columns: + DefaultValues: null + Type: null + Name: + Length: null + Encodings: null + DataType: null + DataValue: + Numeric: [] + Decimal: [] + String: + Bytes: [] + UTF-8: [] + ASCII: [] + otherEncodings: null + TableOperations: null + MariaDB: + DatabaseVersion: null + DatabaseClusterConfiguration: null + EngineType: null + Schema: + TableName: + Length: null + Encodings: null + Partitioning: null + Columns: + DefaultValues: null + Type: null + Name: + Length: null + Encodings: null + DataType: null + DataValue: + Numeric: [] + Decimal: [] + String: + Bytes: [] + UTF-8: [] + ASCII: [] + otherEncodings: null + TableOperations: null + MongoDB: + DatabaseVersion: null + DatabaseClusterConfiguration: null + EngineType: null + Schema: + TableName: + Length: null + Encodings: null + Partitioning: null + Columns: + DefaultValues: null + Type: null + Name: + Length: null + Encodings: null + DataType: null + DataValue: + Numeric: [] + Decimal: [] + String: + Bytes: [] + UTF-8: [] + ASCII: [] + otherEncodings: null + TableOperations: null + + SinkConnector: + Version: [latest] + Configuration: The full list is inside configurations below + + DestinationTables: + DatabaseType: [ClickHouse] + DatabaseClusterConfiguration: + - One node + - Sharded cluster secure + - Cluster with Multiple Shards + - Cluster with Replication (but a single shard) + - Cluster with Multiple Shards and Replicas + - Secure Cluster + DatabaseVersion: [22.8, 23.3, 23.11, 23.12] + EngineType: [ReplacingMergeTree, ReplicatedReplacingMergeTree] + TableOperations: + - INSERT + - UPDATE + - DELETE + - SELECT + - ALTER + + SystemActions: + Network: + - Internal network interruptions in source database cluster + - Network interruptions from source database to sink connector + - Network interruptions from sink connector to clickhouse + - Internal network interruptions in clickhouse database cluster + Process: + Die: + - Internal processes die in source database cluster + - Sink connector dies + - Internal processes die in clickhouse database cluster + Restarted: + - Restart of some or all nodes in source database cluster + - Restart of sink connector + - Restart of some or all nodes in clickhouse database cluster + Disk: + OutOfSpace: + - Out of disk space on some node in source database cluster + - Out of disk space where sink connector is running + - Out of disk space on some node in clickhouse database cluster + Corruptions: + - Corruption on a disk used by some node in source database cluster + - Corruption on a disk where sink connector is running + - Corruption on a disk used by some node in clickhouse database cluster ``` ## Configuration @@ -449,13 +508,13 @@ version: 1.0 ### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Consistency -[Altinity Sink Connector] SHALL support consistent data replication from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support consistent data replication from [MySQL] to [ClickHouse]. ### Multiple MySQL Masters #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Consistency.MultipleMySQLMasters -[Altinity Sink Connector] SHALL support consistent data replication from [MySQL] to [CLickHouse] when one or more MySQL +[Altinity Sink Connector] SHALL support consistent data replication from [MySQL] to [ClickHouse] when one or more MySQL masters are going down. ### Deduplication @@ -477,7 +536,7 @@ version: 1.0 #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.OnlyOnceGuarantee version: 1.0 -[Altinity Sink Connector] SHALL support [MySQL] replication to [CLickHouse] with only-once guarantee. +[Altinity Sink Connector] SHALL support [MySQL] replication to [ClickHouse] with only-once guarantee. Block level de-duplication SHALL be used if it is going to replicated tables but the publisher SHALL publish only once. @@ -487,14 +546,9 @@ The following cases SHALL be supported: 2. [MySQL] database event stream provider crash 3. [MySQL] restart 3. [ClickHouse] server crash -4. [Clickhouse] server restart -7. [Debezium] server crash -8. [Debezium] server restart -9. [Altinity Sink Connector] server crash -10. [Altinity Sink Connector] server restart -11. [Schemaregistry] server crash -12. [Schemaregistry] server restart -13. [Zookeeper] read only mode +4. [ClickHouse] server restart +5. [Altinity Sink Connector] server crash +6. [Altinity Sink Connector] server restart ## Transactions @@ -518,8 +572,18 @@ version: 1.0 ### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MySQLStorageEngines.ReplacingMergeTree version: 1.0 -[Altinity Sink Connector] SHALL support replication of tables that use "InnoDB" [MySQL] storage engine to -"ReplacingMergeTree" [ClickHouse] table engine. +[Altinity Sink Connector] SHALL support replication of tables that use the following storage engines in MySQL: + +| Storage Engines | +|-----------------| +| InnoDB | +| MyISAM | +| MEMORY | +| CSV | +| ARCHIVE | +| BLACKHOLE | +| FEDERATED | +| EXAMPLE | #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MySQLStorageEngines.ReplacingMergeTree.VirtualColumnNames version: 1.0 @@ -547,6 +611,32 @@ version: 1.0 [Altinity Sink Connector] SHALL support data replication to [ClickHouse] of tables with any datatypes that [MySQL] supports. +| MySQL Datatypes | +|--------------------| +| DECIMAL | +| DOUBLE | +| DATE | +| DATETIME(1-6) | +| TIME(1-6) | +| INT | +| INT UNSIGNED | +| BIGINT | +| BIGINT UNSIGNED | +| TINYINT | +| TINYINT UNSIGNED | +| SMALLINT | +| SMALLINT UNSIGNED | +| MEDIUMINT | +| MEDIUMINT UNSIGNED | +| CHAR | +| TEXT | +| VARCHAR(1-32766) | +| BLOB | +| MEDIUMBLOB | +| LONGBLOB | +| BINARY | +| VARBINARY | + ### Integer Types #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.DataTypes.IntegerTypes @@ -575,7 +665,7 @@ Integer data types connection table: #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.DataTypes.Decimal version: 1.0 -[Altinity Sink Connector] SHALL support data replication to [CLickHouse] of tables that contain columns with +[Altinity Sink Connector] SHALL support data replication to [ClickHouse] of tables that contain columns with 'Decimal' data types as they supported by [MySQL]. [ClickHouse]'s 'Decimal32(S)', 'Decimal64(S)', 'Decimal128(S)', 'Decimal256(S)' also can be @@ -592,7 +682,7 @@ Data types connection table: #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.DataTypes.Double version: 1.0 -[Altinity Sink Connector] SHALL support data replication to [CLickHouse] of tables that contain columns with +[Altinity Sink Connector] SHALL support data replication to [ClickHouse] of tables that contain columns with 'Double' data types as they supported by [MySQL]. Data types connection table: @@ -606,7 +696,7 @@ Data types connection table: #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.DataTypes.DateTime version: 1.0 -[Altinity Sink Connector] SHALL support data replication to [CLickHouse] of tables that contain columns with 'Data' and 'Time' +[Altinity Sink Connector] SHALL support data replication to [ClickHouse] of tables that contain columns with 'Data' and 'Time' data types as they supported by [MySQL]. Data types connection table: @@ -625,7 +715,7 @@ Data types connection table: #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.DataTypes.Binary version: 1.0 -[Altinity Sink Connector] SHALL support data replication to [CLickHouse] replication of tables that contain columns with 'Binary' +[Altinity Sink Connector] SHALL support data replication to [ClickHouse] replication of tables that contain columns with 'Binary' data types as they supported by [MySQL]. Data types connection table: @@ -640,7 +730,7 @@ Data types connection table: #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.DataTypes.String version: 1.0 -[Altinity Sink Connector] SHALL support data replication to [CLickHouse] of tables that contain columns with 'String' +[Altinity Sink Connector] SHALL support data replication to [ClickHouse] of tables that contain columns with 'String' data types as they supported by [MySQL]. Data types connection table: @@ -657,7 +747,7 @@ Data types connection table: #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.DataTypes.BlobTypes version: 1.0 -[Altinity Sink Connector] SHALL support data replication to [CLickHouse] of tables that contain columns with 'Blob' [MySQL] +[Altinity Sink Connector] SHALL support data replication to [ClickHouse] of tables that contain columns with 'Blob' [MySQL] data types and correctly unhex() them. ```sql @@ -677,7 +767,7 @@ Data types connection table: #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.DataTypes.Nullable version: 1.0 -[Altinity Sink Connector] SHALL support data replication to [CLickHouse] of tables that contain columns with NULL [MySQL] +[Altinity Sink Connector] SHALL support data replication to [ClickHouse] of tables that contain columns with NULL [MySQL] data types if this expected `Nullable(DataType)` construction should be used. For example, [MySQL] `VARCHAR(*)` maps to [ClickHouse] `Nullable(String)` and MySQL @@ -688,7 +778,7 @@ For example, [MySQL] `VARCHAR(*)` maps to [ClickHouse] `Nullable(String)` and My #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.DataTypes.EnumToEnum version: 1.0 -[Altinity Sink Connector] SHALL support data replication to [CLickHouse] of tables that contain columns with 'ENUM' +[Altinity Sink Connector] SHALL support data replication to [ClickHouse] of tables that contain columns with 'ENUM' data types as they supported by [MySQL]. Data types connection table: @@ -700,7 +790,7 @@ Data types connection table: #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.DataTypes.EnumToString version: 1.0 -[Altinity Sink Connector] SHALL support data replication to [CLickHouse] of tables that contain columns with 'ENUM' +[Altinity Sink Connector] SHALL support data replication to [ClickHouse] of tables that contain columns with 'ENUM' data types as they supported by [MySQL]. Data types connection table: @@ -714,7 +804,7 @@ Data types connection table: #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.DataTypes.JSON version: 1.0 -[Altinity Sink Connector] SHALL support data replication to [CLickHouse] of tables that contain columns with 'JSON' +[Altinity Sink Connector] SHALL support data replication to [ClickHouse] of tables that contain columns with 'JSON' data types as they supported by [MySQL]. Data types connection table: @@ -728,7 +818,7 @@ Data types connection table: #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.DataTypes.Year version: 1.0 -[Altinity Sink Connector] SHALL support data replication to [CLickHouse] of tables that contain columns with 'Year' +[Altinity Sink Connector] SHALL support data replication to [ClickHouse] of tables that contain columns with 'Year' data types as they supported by [MySQL]. Data types connection table: @@ -742,7 +832,7 @@ Data types connection table: #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.DataTypes.Bytes version: 1.0 -[Altinity Sink Connector] SHALL support data replication to [CLickHouse] of tables that contain columns with 'BIT(m)' +[Altinity Sink Connector] SHALL support data replication to [ClickHouse] of tables that contain columns with 'BIT(m)' data types where m: 2 - 64 as they supported by [MySQL]. Data types connection table: @@ -759,35 +849,49 @@ Data types connection table: #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Queries.Inserts version: 1.0 -[Altinity Sink Connector] SHALL support new data inserts replication from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support new data inserts replication from [MySQL] to [ClickHouse]. ##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Queries.Inserts.PartitionLimits version: 1.0 -[Altinity Sink Connector] SHALL support correct data inserts replication from [MySQL] to [CLickHouse] when partition +[Altinity Sink Connector] SHALL support correct data inserts replication from [MySQL] to [ClickHouse] when partition limits are hitting or avoid such situations. +##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Queries.Inserts.PartitionTypes +version: 1.0 + +[Altinity Sink Connector] SHALL support correct data inserts replication from [MySQL] to [ClickHouse] when the table in source table is partitioned with the following logic. + +| Partition Type | +|----------------| +| RANGE | +| LIST | +| COLUMNS | +| HASH | +| KEY | +| Subpartition | + ### Updates #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Queries.Updates version: 1.0 -[Altinity Sink Connector] SHALL support data updates replication from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support data updates replication from [MySQL] to [ClickHouse]. ### Deletes #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Queries.Deletes version: 1.0 -[Altinity Sink Connector] SHALL support data deletes replication from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support data deletes replication from [MySQL] to [ClickHouse]. ## Table Schema Creation ### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.TableSchemaCreation version: 1.0 -[Altinity Sink Connector]SHALL support the following ways to replicate schema from [MySQL] to [CLickHouse]: +[Altinity Sink Connector]SHALL support the following ways to replicate schema from [MySQL] to [ClickHouse]: * auto-create option * `clickhouse_loader` script * `chump` utility @@ -797,26 +901,26 @@ version: 1.0 #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.TableSchemaCreation.AutoCreate version: 1.0 -[Altinity Sink Connector] SHALL support auto table creation from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support auto table creation from [MySQL] to [ClickHouse]. ##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.TableSchemaCreation.MultipleAutoCreate version: 1.0 -[Altinity Sink Connector] SHALL support auto creation of multiple tables from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support auto creation of multiple tables from [MySQL] to [ClickHouse]. ### Auto Drop #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.TableSchemaCreation.AutoDrop version: 1.0 -[Altinity Sink Connector] SHALL support `DROP TABLE` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `DROP TABLE` query from [MySQL] to [ClickHouse]. ## Alter ### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter version: 1.0 -[Altinity Sink Connector] SHALL support all `ALTER` queries. +[Altinity Sink Connector] SHALL support the following `ALTER` queries. | MySQL | ClickHouse | |--------------------------------------------------------|-----------------------------------------------------------------| @@ -842,112 +946,112 @@ version: 1.0 #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.AddIndex version: 1.0 -[Altinity Sink Connector] SHALL support `ADD INDEX` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `ADD INDEX` query from [MySQL] to [ClickHouse]. ### Add Key #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.AddKey version: 1.0 -[Altinity Sink Connector] SHALL support `ADD Key` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `ADD Key` query from [MySQL] to [ClickHouse]. ### Add FullText #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.AddFullText version: 1.0 -[Altinity Sink Connector] SHALL support `ADD FULLTEXT` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `ADD FULLTEXT` query from [MySQL] to [ClickHouse]. ### Add Special #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.AddSpecial version: 1.0 -[Altinity Sink Connector] SHALL support `ADD SPECIAL` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `ADD SPECIAL` query from [MySQL] to [ClickHouse]. ### Drop Check #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.DropCheck version: 1.0 -[Altinity Sink Connector] SHALL support `DROP CHECK` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `DROP CHECK` query from [MySQL] to [ClickHouse]. ### Drop Default #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.DropDefault version: 1.0 -[Altinity Sink Connector] SHALL support `DROP DEFAULT` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `DROP DEFAULT` query from [MySQL] to [ClickHouse]. ### Check #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.Check version: 1.0 -[Altinity Sink Connector] SHALL support `ALTER CHECK` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `ALTER CHECK` query from [MySQL] to [ClickHouse]. ### Constraint #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.Constraint version: 1.0 -[Altinity Sink Connector] SHALL support `ALTER CONSTRAINT` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `ALTER CONSTRAINT` query from [MySQL] to [ClickHouse]. ### Index #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.Index version: 1.0 -[Altinity Sink Connector] SHALL support `ALTER INDEX` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `ALTER INDEX` query from [MySQL] to [ClickHouse]. ### Character Set #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.CharacterSet version: 1.0 -[Altinity Sink Connector] SHALL support `ALTER CHARACTER SET` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `ALTER CHARACTER SET` query from [MySQL] to [ClickHouse]. ### Convert To Character Set #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.ConvertToCharacterSet version: 1.0 -[Altinity Sink Connector] SHALL support `ALTER CONVERT TO CHARACTER SET` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `ALTER CONVERT TO CHARACTER SET` query from [MySQL] to [ClickHouse]. ### Algorithm #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.Algorithm version: 1.0 -[Altinity Sink Connector] SHALL support `ALTER ALGORITHM` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `ALTER ALGORITHM` query from [MySQL] to [ClickHouse]. ### Force #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.Force version: 1.0 -[Altinity Sink Connector] SHALL support `ALTER FORCE` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `ALTER FORCE` query from [MySQL] to [ClickHouse]. ### Lock #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.Lock version: 1.0 -[Altinity Sink Connector] SHALL support `ALTER LOCK` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `ALTER LOCK` query from [MySQL] to [ClickHouse]. ### Unlock #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.Unlock version: 1.0 -[Altinity Sink Connector] SHALL support `ALTER UNLOCK` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `ALTER UNLOCK` query from [MySQL] to [ClickHouse]. ### Validation #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.Validation version: 1.0 -[Altinity Sink Connector] SHALL support `ALTER VALIDATION` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `ALTER VALIDATION` query from [MySQL] to [ClickHouse]. ### Columns @@ -956,28 +1060,28 @@ version: 1.0 ##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.Columns.Add version: 1.0 -[Altinity Sink Connector] SHALL support `ADD COLUMN` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `ADD COLUMN` query from [MySQL] to [ClickHouse]. ##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.Columns.Add.NullNotNull version: 1.0 -[Altinity Sink Connector] SHALL support `ADD COLUMN NULL/NOT NULL` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `ADD COLUMN NULL/NOT NULL` query from [MySQL] to [ClickHouse]. ##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.Columns.Add.Default version: 1.0 -[Altinity Sink Connector] SHALL support `ADD COLUMN DEFAULT` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `ADD COLUMN DEFAULT` query from [MySQL] to [ClickHouse]. ##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.Columns.Add.FirstAfter version: 1.0 -[Altinity Sink Connector] SHALL support `ADD COLUMN FIRST, AFTER` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `ADD COLUMN FIRST, AFTER` query from [MySQL] to [ClickHouse]. ##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.Columns.Add.Multiple version: 1.0 -[Altinity Sink Connector] SHALL support multiple `ADD COLUMN` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support multiple `ADD COLUMN` query from [MySQL] to [ClickHouse]. @@ -986,83 +1090,83 @@ version: 1.0 ##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.Columns.Modify version: 1.0 -[Altinity Sink Connector] SHALL support `MODIFY COLUMN data_type` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `MODIFY COLUMN data_type` query from [MySQL] to [ClickHouse]. ##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.Columns.Modify.NullNotNull version: 1.0 -[Altinity Sink Connector] SHALL support `MODIFY COLUMN data_type NULL/NOT NULL` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `MODIFY COLUMN data_type NULL/NOT NULL` query from [MySQL] to [ClickHouse]. ##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.Columns.Modify.Default version: 1.0 -[Altinity Sink Connector] SHALL support `MODIFY COLUMN data_type DEFAULT` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `MODIFY COLUMN data_type DEFAULT` query from [MySQL] to [ClickHouse]. ##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.Columns.Modify.FirstAfter version: 1.0 -[Altinity Sink Connector] SHALL support `MODIFY COLUMN data_type FIRST, AFTER` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `MODIFY COLUMN data_type FIRST, AFTER` query from [MySQL] to [ClickHouse]. ##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.Columns.Modify.Multiple version: 1.0 -[Altinity Sink Connector] SHALL support multiple `MODIFY COLUMN` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support multiple `MODIFY COLUMN` query from [MySQL] to [ClickHouse]. #### Change ##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.Columns.Change.NullNotNullOldNew version: 1.0 -[Altinity Sink Connector] SHALL support `CHANGE COLUMN old_name new_name datatype NULL/NOT NULL` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `CHANGE COLUMN old_name new_name datatype NULL/NOT NULL` query from [MySQL] to [ClickHouse]. ##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.Columns.Change.FirstAfter version: 1.0 -[Altinity Sink Connector] SHALL support `CHANGE COLUMN FIRST, AFTER` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `CHANGE COLUMN FIRST, AFTER` query from [MySQL] to [ClickHouse]. ##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.Columns.Change.Multiple version: 1.0 -[Altinity Sink Connector] SHALL support multiple `CHANGE COLUMN` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support multiple `CHANGE COLUMN` query from [MySQL] to [ClickHouse]. #### Drop ##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.Columns.Drop version: 1.0 -[Altinity Sink Connector] SHALL support `DROP COLUMN` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `DROP COLUMN` query from [MySQL] to [ClickHouse]. ##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.Columns.Drop.Multiple version: 1.0 -[Altinity Sink Connector] SHALL support multiple `DROP COLUMN` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support multiple `DROP COLUMN` query from [MySQL] to [ClickHouse]. #### Rename ##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.Columns.Rename version: 1.0 -[Altinity Sink Connector] SHALL support `RENAME COLUMN col1 to col2` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `RENAME COLUMN col1 to col2` query from [MySQL] to [ClickHouse]. ##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.Columns.Rename.Multiple version: 1.0 -[Altinity Sink Connector] SHALL support multiple `RENAME COLUMN col1 to col2` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support multiple `RENAME COLUMN col1 to col2` query from [MySQL] to [ClickHouse]. ### Add Constraint #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.AddConstraint version: 1.0 -[Altinity Sink Connector] SHALL support `ADD CONSTRAINT` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `ADD CONSTRAINT` query from [MySQL] to [ClickHouse]. ### Drop Constraint #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.DropConstraint version: 1.0 -[Altinity Sink Connector] SHALL support `DROP CONSTRAINT` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `DROP CONSTRAINT` query from [MySQL] to [ClickHouse]. ## Primary Key @@ -1071,7 +1175,7 @@ version: 1.0 #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.PrimaryKey.No version: 1.0 -[Altinity Sink Connector] query SHALL support [MySQL] data replication to [CLickHouse] on queries to tables +[Altinity Sink Connector] query SHALL support [MySQL] data replication to [ClickHouse] on queries to tables with no `PRIMARY KEY`. ### Simple Primary Key @@ -1079,7 +1183,7 @@ with no `PRIMARY KEY`. #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.PrimaryKey.Simple version: 1.0 -[Altinity Sink Connector] query SHALL support [MySQL] data replication to [CLickHouse] on queries with the same order +[Altinity Sink Connector] query SHALL support [MySQL] data replication to [ClickHouse] on queries with the same order as simple `PRIMARY KEY` does. ### Composite Primary Key @@ -1087,7 +1191,7 @@ as simple `PRIMARY KEY` does. #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.PrimaryKey.Composite version: 1.0 -[Altinity Sink Connector] query SHALL support [MySQL] data replication to [CLickHouse] on queries with the same order +[Altinity Sink Connector] query SHALL support [MySQL] data replication to [ClickHouse] on queries with the same order as composite `PRIMARY KEY` does. ## Multiple Upstream Servers @@ -1095,21 +1199,21 @@ as composite `PRIMARY KEY` does. ### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleUpstreamServers version: 1.0 -[Altinity Sink Connector] SHALL support [MySQL] replication to [CLickHouse] from multiple [MySQL] upstream servers. +[Altinity Sink Connector] SHALL support [MySQL] replication to [ClickHouse] from multiple [MySQL] upstream servers. ## Multiple Downstream Servers ### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDownstreamServers version: 1.0 -[Altinity Sink Connector] SHALL support [MySQL] replication to [CLickHouse] when using multiple downstream [ClickHouse] servers. +[Altinity Sink Connector] SHALL support [MySQL] replication to [ClickHouse] when using multiple downstream [ClickHouse] servers. ## Archival Mode ### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ArchivalMode version: 1.0 -[Altinity Sink Connector] SHALL support [MySQL] replication to [CLickHouse] with archival mode that +[Altinity Sink Connector] SHALL support [MySQL] replication to [ClickHouse] with archival mode that SHALL ignore deletes for some or all tables in [ClickHouse]. ## Bootstrapping Mode @@ -1117,7 +1221,7 @@ SHALL ignore deletes for some or all tables in [ClickHouse]. ### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.BootstrappingMode version: 1.0 -[Altinity Sink Connector] SHALL support [MySQL] replication to [CLickHouse] with +[Altinity Sink Connector] SHALL support [MySQL] replication to [ClickHouse] with bootstrapping mode for the initial replication of very large tables that bypasses event stream by using [MySQL] dump files. @@ -1126,7 +1230,7 @@ that bypasses event stream by using [MySQL] dump files. ### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.BinlogPosition version: 1.0 -[Altinity Sink Connector] SHALL support ability to start replication to [CLickHouse] +[Altinity Sink Connector] SHALL support ability to start replication to [ClickHouse] from specific [MySQL] binlog position. ## Column Mapping And Transformation Rules @@ -1134,7 +1238,7 @@ from specific [MySQL] binlog position. ### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ColumnMappingAndTransformationRules version: 1.0 -[Altinity Sink Connector] SHALL support [MySQL] replication to [CLickHouse] with support for +[Altinity Sink Connector] SHALL support [MySQL] replication to [ClickHouse] with support for defining column mapping and transformations rules. ## Columns Inconsistency @@ -1142,29 +1246,29 @@ defining column mapping and transformations rules. ### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ColumnsInconsistency version: 1.0 -[Altinity Sink Connector] SHALL support [MySQL] replication to [CLickHouse] replica table when it has fewer columns. -[MySQL] replication to [CLickHouse] is not available in all other cases of columns inconsistency . +[Altinity Sink Connector] SHALL support [MySQL] replication to [ClickHouse] replica table when it has fewer columns. +[MySQL] replication to [ClickHouse] is not available in all other cases of columns inconsistency . ## Latency ### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Latency version: 1.0 -[Altinity Sink Connector] SHALL support [MySQL] replication to [CLickHouse] with latency as close as possible to real-time. +[Altinity Sink Connector] SHALL support [MySQL] replication to [ClickHouse] with latency as close as possible to real-time. ## Performance ### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Performance version: 1.0 -[Altinity Sink Connector] SHALL support [MySQL] replication to [CLickHouse] more than 100,000 rows/sec. +[Altinity Sink Connector] SHALL support [MySQL] replication to [ClickHouse] more than 100,000 rows/sec. ### Large Daily Data Volumes #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Performance.LargeDailyDataVolumes version: 1.0 -[Altinity Sink Connector] SHALL support [MySQL] replication to [CLickHouse] with large daily data volumes of at least 20-30TB per day. +[Altinity Sink Connector] SHALL support [MySQL] replication to [ClickHouse] with large daily data volumes of at least 20-30TB per day. ## Settings @@ -1229,7 +1333,83 @@ version: 1.0 [Altinity Sink Connector] SHALL retry replication if the ClickHouse instance is stopped/killed during the active replication from source to destination tables. [Altinity Sink Connector] SHALL continue to retry to replicate data into a source table until the ClickHouse instance is not available again. -## Prometheus +## System Actions + +### Handling Network Interruptions + +#### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.SystemActions.Network +version: 1.0 + +[Altinity Sink Connector] SHALL be able to recover replication after network related interruptions happen, so that the data on the destination table is not lost. + + +| List of possible network related interruptions | +|---------------------------------------------------------------| +| Internal network interruptions in source database cluster | +| Network interruptions from source database to sink connector | +| Network interruptions from sink connector to clickhouse | +| Internal network interruptions in clickhouse database cluster | + + +### Handling Process Interruptions + +#### Behaviour When Different Processes Were Killed + +##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.SystemActions.Process.Die +version: 1.0 + +[Altinity Sink Connector] SHALL output an error and keep the existing data when one of the following or all of the +scenarios related to processes being killed happen: + +| Scenarios Related to Processes Being Killed | +|-------------------------------------------------------| +| Internal processes die in source database cluster | +| Sink connector dies | +| Internal processes die in clickhouse database cluster | + +#### Behaviour When Different Processes Were Restarted + +##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.SystemActions.Process.Restarted +version: 1.0 + +[Altinity Sink Connector] SHALL continue replication without losing any data after one of the following or +all of the scenarios related to processes being restarted happen: + +| Scenarios Related to Processes Being Restarted | +|-------------------------------------------------------------| +| Restart of some or all nodes in source database cluster | +| Restart of sink connector | +| Restart of some or all nodes in clickhouse database cluster | + +### Behaviour When There Are Issues With Disk + +#### Disk Is out of Space + +##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.SystemActions.Disk.OutOfSpace +version: 1.0 + +[Altinity Sink Connector] SHALL output an error when: + +- Out of disk space on some node in source database cluster +- Out of disk space where sink connector is running +- Out of disk space on some node in clickhouse database cluster + +The error SHALL be shown so that the data on the source and destination tables is not lost due to the disk related issues. + +#### Disk Is Corrupted + +##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.SystemActions.Disk.Corrupted +version: 1.0 + +[Altinity Sink Connector] SHALL output an error when: + +- Corruption on a disk used by some node in source database cluster +- Corruption on a disk where sink connector is running +- Corruption on a disk used by some node in clickhouse database cluster + +The error SHALL be shown so that the data on the source and destination tables is not lost due to the disk related issues. + +## Prometheus ### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Prometheus version: 1.0 diff --git a/sink-connector-lightweight/tests/integration/requirements/requirements.py b/sink-connector-lightweight/tests/integration/requirements/requirements.py index 60101a960..b1754882d 100644 --- a/sink-connector-lightweight/tests/integration/requirements/requirements.py +++ b/sink-connector-lightweight/tests/integration/requirements/requirements.py @@ -80,7 +80,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support [MySQL] replication to [CLickHouse] with only-once guarantee.\n" + "[Altinity Sink Connector] SHALL support [MySQL] replication to [ClickHouse] with only-once guarantee.\n" "Block level de-duplication SHALL be used if it is going to replicated tables\n" "but the publisher SHALL publish only once.\n" "\n" @@ -90,14 +90,9 @@ "2. [MySQL] database event stream provider crash\n" "3. [MySQL] restart\n" "3. [ClickHouse] server crash\n" - "4. [Clickhouse] server restart\n" - "7. [Debezium] server crash\n" - "8. [Debezium] server restart\n" - "9. [Altinity Sink Connector] server crash\n" - "10. [Altinity Sink Connector] server restart\n" - "11. [Schemaregistry] server crash\n" - "12. [Schemaregistry] server restart\n" - "13. [Zookeeper] read only mode\n" + "4. [ClickHouse] server restart\n" + "5. [Altinity Sink Connector] server crash\n" + "6. [Altinity Sink Connector] server restart\n" "\n" ), link=None, @@ -149,8 +144,18 @@ type=None, uid=None, description=( - '[Altinity Sink Connector] SHALL support replication of tables that use "InnoDB" [MySQL] storage engine to\n' - '"ReplacingMergeTree" [ClickHouse] table engine.\n' + "[Altinity Sink Connector] SHALL support replication of tables that use the following storage engines in MySQL:\n" + "\n" + "| Storage Engines |\n" + "|-----------------|\n" + "| InnoDB |\n" + "| MyISAM |\n" + "| MEMORY |\n" + "| CSV |\n" + "| ARCHIVE |\n" + "| BLACKHOLE |\n" + "| FEDERATED |\n" + "| EXAMPLE |\n" "\n" ), link=None, @@ -220,6 +225,32 @@ description=( "[Altinity Sink Connector] SHALL support data replication to [ClickHouse] of tables with any datatypes that [MySQL] supports.\n" "\n" + "| MySQL Datatypes |\n" + "|--------------------|\n" + "| DECIMAL |\n" + "| DOUBLE |\n" + "| DATE |\n" + "| DATETIME(1-6) |\n" + "| TIME(1-6) |\n" + "| INT |\n" + "| INT UNSIGNED |\n" + "| BIGINT |\n" + "| BIGINT UNSIGNED |\n" + "| TINYINT |\n" + "| TINYINT UNSIGNED |\n" + "| SMALLINT |\n" + "| SMALLINT UNSIGNED |\n" + "| MEDIUMINT |\n" + "| MEDIUMINT UNSIGNED |\n" + "| CHAR |\n" + "| TEXT |\n" + "| VARCHAR(1-32766) |\n" + "| BLOB |\n" + "| MEDIUMBLOB |\n" + "| LONGBLOB |\n" + "| BINARY |\n" + "| VARBINARY |\n" + "\n" ), link=None, level=2, @@ -266,7 +297,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support data replication to [CLickHouse] of tables that contain columns with\n" + "[Altinity Sink Connector] SHALL support data replication to [ClickHouse] of tables that contain columns with\n" "'Decimal' data types as they supported by [MySQL].\n" "\n" "[ClickHouse]'s 'Decimal32(S)', 'Decimal64(S)', 'Decimal128(S)', 'Decimal256(S)' also can be\n" @@ -292,7 +323,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support data replication to [CLickHouse] of tables that contain columns with\n" + "[Altinity Sink Connector] SHALL support data replication to [ClickHouse] of tables that contain columns with\n" "'Double' data types as they supported by [MySQL].\n" "\n" "Data types connection table:\n" @@ -315,7 +346,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support data replication to [CLickHouse] of tables that contain columns with 'Data' and 'Time'\n" + "[Altinity Sink Connector] SHALL support data replication to [ClickHouse] of tables that contain columns with 'Data' and 'Time'\n" "data types as they supported by [MySQL].\n" "\n" "Data types connection table:\n" @@ -343,7 +374,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support data replication to [CLickHouse] replication of tables that contain columns with 'Binary'\n" + "[Altinity Sink Connector] SHALL support data replication to [ClickHouse] replication of tables that contain columns with 'Binary'\n" "data types as they supported by [MySQL].\n" "\n" "Data types connection table:\n" @@ -367,7 +398,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support data replication to [CLickHouse] of tables that contain columns with 'String'\n" + "[Altinity Sink Connector] SHALL support data replication to [ClickHouse] of tables that contain columns with 'String'\n" "data types as they supported by [MySQL].\n" "\n" "Data types connection table:\n" @@ -393,7 +424,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support data replication to [CLickHouse] of tables that contain columns with 'Blob' [MySQL]\n" + "[Altinity Sink Connector] SHALL support data replication to [ClickHouse] of tables that contain columns with 'Blob' [MySQL]\n" "data types and correctly unhex() them.\n" "\n" "```sql\n" @@ -422,7 +453,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support data replication to [CLickHouse] of tables that contain columns with NULL [MySQL]\n" + "[Altinity Sink Connector] SHALL support data replication to [ClickHouse] of tables that contain columns with NULL [MySQL]\n" "data types if this expected `Nullable(DataType)` construction should be used.\n" "\n" "For example, [MySQL] `VARCHAR(*)` maps to [ClickHouse] `Nullable(String)` and MySQL\n" @@ -442,7 +473,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support data replication to [CLickHouse] of tables that contain columns with 'ENUM'\n" + "[Altinity Sink Connector] SHALL support data replication to [ClickHouse] of tables that contain columns with 'ENUM'\n" "data types as they supported by [MySQL].\n" "\n" "Data types connection table:\n" @@ -465,7 +496,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support data replication to [CLickHouse] of tables that contain columns with 'ENUM'\n" + "[Altinity Sink Connector] SHALL support data replication to [ClickHouse] of tables that contain columns with 'ENUM'\n" "data types as they supported by [MySQL].\n" "\n" "Data types connection table:\n" @@ -488,7 +519,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support data replication to [CLickHouse] of tables that contain columns with 'JSON'\n" + "[Altinity Sink Connector] SHALL support data replication to [ClickHouse] of tables that contain columns with 'JSON'\n" "data types as they supported by [MySQL].\n" "\n" "Data types connection table:\n" @@ -511,7 +542,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support data replication to [CLickHouse] of tables that contain columns with 'Year'\n" + "[Altinity Sink Connector] SHALL support data replication to [ClickHouse] of tables that contain columns with 'Year'\n" "data types as they supported by [MySQL].\n" "\n" "Data types connection table:\n" @@ -534,7 +565,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support data replication to [CLickHouse] of tables that contain columns with 'BIT(m)'\n" + "[Altinity Sink Connector] SHALL support data replication to [ClickHouse] of tables that contain columns with 'BIT(m)'\n" "data types where m: 2 - 64 as they supported by [MySQL].\n" "\n" "Data types connection table:\n" @@ -558,7 +589,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support new data inserts replication from [MySQL] to [CLickHouse].\n" + "[Altinity Sink Connector] SHALL support new data inserts replication from [MySQL] to [ClickHouse].\n" "\n" ), link=None, @@ -574,16 +605,41 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support correct data inserts replication from [MySQL] to [CLickHouse] when partition \n" + "[Altinity Sink Connector] SHALL support correct data inserts replication from [MySQL] to [ClickHouse] when partition \n" "limits are hitting or avoid such situations.\n" "\n" - "\n" ), link=None, level=4, num="12.1.1.1", ) +RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_Queries_Inserts_PartitionTypes = Requirement( + name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Queries.Inserts.PartitionTypes", + version="1.0", + priority=None, + group=None, + type=None, + uid=None, + description=( + "[Altinity Sink Connector] SHALL support correct data inserts replication from [MySQL] to [ClickHouse] when the table in source table is partitioned with the following logic.\n" + "\n" + "| Partition Type |\n" + "|----------------|\n" + "| RANGE |\n" + "| LIST |\n" + "| COLUMNS |\n" + "| HASH |\n" + "| KEY |\n" + "| Subpartition |\n" + "\n" + "\n" + ), + link=None, + level=4, + num="12.1.1.2", +) + RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_Queries_Updates = Requirement( name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Queries.Updates", version="1.0", @@ -592,7 +648,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support data updates replication from [MySQL] to [CLickHouse].\n" + "[Altinity Sink Connector] SHALL support data updates replication from [MySQL] to [ClickHouse].\n" "\n" ), link=None, @@ -608,7 +664,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support data deletes replication from [MySQL] to [CLickHouse].\n" + "[Altinity Sink Connector] SHALL support data deletes replication from [MySQL] to [ClickHouse].\n" "\n" ), link=None, @@ -624,7 +680,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector]SHALL support the following ways to replicate schema from [MySQL] to [CLickHouse]:\n" + "[Altinity Sink Connector]SHALL support the following ways to replicate schema from [MySQL] to [ClickHouse]:\n" "* auto-create option\n" "* `clickhouse_loader` script\n" "* `chump` utility\n" @@ -643,7 +699,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support auto table creation from [MySQL] to [CLickHouse].\n" + "[Altinity Sink Connector] SHALL support auto table creation from [MySQL] to [ClickHouse].\n" "\n" ), link=None, @@ -659,7 +715,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support auto creation of multiple tables from [MySQL] to [CLickHouse].\n" + "[Altinity Sink Connector] SHALL support auto creation of multiple tables from [MySQL] to [ClickHouse].\n" "\n" ), link=None, @@ -675,7 +731,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support `DROP TABLE` query from [MySQL] to [CLickHouse].\n" + "[Altinity Sink Connector] SHALL support `DROP TABLE` query from [MySQL] to [ClickHouse].\n" "\n" ), link=None, @@ -691,7 +747,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support all `ALTER` queries.\n" + "[Altinity Sink Connector] SHALL support the following `ALTER` queries.\n" "\n" "| MySQL | ClickHouse |\n" "|--------------------------------------------------------|-----------------------------------------------------------------|\n" @@ -726,7 +782,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support `ADD INDEX` query from [MySQL] to [CLickHouse].\n" + "[Altinity Sink Connector] SHALL support `ADD INDEX` query from [MySQL] to [ClickHouse].\n" "\n" ), link=None, @@ -742,7 +798,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support `ADD Key` query from [MySQL] to [CLickHouse].\n" + "[Altinity Sink Connector] SHALL support `ADD Key` query from [MySQL] to [ClickHouse].\n" "\n" ), link=None, @@ -758,7 +814,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support `ADD FULLTEXT` query from [MySQL] to [CLickHouse].\n" + "[Altinity Sink Connector] SHALL support `ADD FULLTEXT` query from [MySQL] to [ClickHouse].\n" "\n" ), link=None, @@ -774,7 +830,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support `ADD SPECIAL` query from [MySQL] to [CLickHouse].\n" + "[Altinity Sink Connector] SHALL support `ADD SPECIAL` query from [MySQL] to [ClickHouse].\n" "\n" ), link=None, @@ -790,7 +846,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support `DROP CHECK` query from [MySQL] to [CLickHouse].\n" + "[Altinity Sink Connector] SHALL support `DROP CHECK` query from [MySQL] to [ClickHouse].\n" "\n" ), link=None, @@ -806,7 +862,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support `DROP DEFAULT` query from [MySQL] to [CLickHouse].\n" + "[Altinity Sink Connector] SHALL support `DROP DEFAULT` query from [MySQL] to [ClickHouse].\n" "\n" ), link=None, @@ -822,7 +878,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support `ALTER CHECK` query from [MySQL] to [CLickHouse].\n" + "[Altinity Sink Connector] SHALL support `ALTER CHECK` query from [MySQL] to [ClickHouse].\n" "\n" ), link=None, @@ -838,7 +894,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support `ALTER CONSTRAINT` query from [MySQL] to [CLickHouse].\n" + "[Altinity Sink Connector] SHALL support `ALTER CONSTRAINT` query from [MySQL] to [ClickHouse].\n" "\n" ), link=None, @@ -854,7 +910,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support `ALTER INDEX` query from [MySQL] to [CLickHouse].\n" + "[Altinity Sink Connector] SHALL support `ALTER INDEX` query from [MySQL] to [ClickHouse].\n" "\n" ), link=None, @@ -870,7 +926,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support `ALTER CHARACTER SET` query from [MySQL] to [CLickHouse].\n" + "[Altinity Sink Connector] SHALL support `ALTER CHARACTER SET` query from [MySQL] to [ClickHouse].\n" "\n" ), link=None, @@ -886,7 +942,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support `ALTER CONVERT TO CHARACTER SET` query from [MySQL] to [CLickHouse].\n" + "[Altinity Sink Connector] SHALL support `ALTER CONVERT TO CHARACTER SET` query from [MySQL] to [ClickHouse].\n" "\n" ), link=None, @@ -902,7 +958,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support `ALTER ALGORITHM` query from [MySQL] to [CLickHouse].\n" + "[Altinity Sink Connector] SHALL support `ALTER ALGORITHM` query from [MySQL] to [ClickHouse].\n" "\n" ), link=None, @@ -918,7 +974,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support `ALTER FORCE` query from [MySQL] to [CLickHouse].\n" + "[Altinity Sink Connector] SHALL support `ALTER FORCE` query from [MySQL] to [ClickHouse].\n" "\n" ), link=None, @@ -934,7 +990,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support `ALTER LOCK` query from [MySQL] to [CLickHouse].\n" + "[Altinity Sink Connector] SHALL support `ALTER LOCK` query from [MySQL] to [ClickHouse].\n" "\n" ), link=None, @@ -950,7 +1006,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support `ALTER UNLOCK` query from [MySQL] to [CLickHouse].\n" + "[Altinity Sink Connector] SHALL support `ALTER UNLOCK` query from [MySQL] to [ClickHouse].\n" "\n" ), link=None, @@ -966,7 +1022,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support `ALTER VALIDATION` query from [MySQL] to [CLickHouse].\n" + "[Altinity Sink Connector] SHALL support `ALTER VALIDATION` query from [MySQL] to [ClickHouse].\n" "\n" ), link=None, @@ -982,7 +1038,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support `ADD COLUMN` query from [MySQL] to [CLickHouse].\n" + "[Altinity Sink Connector] SHALL support `ADD COLUMN` query from [MySQL] to [ClickHouse].\n" "\n" ), link=None, @@ -998,7 +1054,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support `ADD COLUMN NULL/NOT NULL` query from [MySQL] to [CLickHouse].\n" + "[Altinity Sink Connector] SHALL support `ADD COLUMN NULL/NOT NULL` query from [MySQL] to [ClickHouse].\n" "\n" ), link=None, @@ -1014,7 +1070,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support `ADD COLUMN DEFAULT` query from [MySQL] to [CLickHouse].\n" + "[Altinity Sink Connector] SHALL support `ADD COLUMN DEFAULT` query from [MySQL] to [ClickHouse].\n" "\n" "\n" ), @@ -1031,7 +1087,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support `ADD COLUMN FIRST, AFTER` query from [MySQL] to [CLickHouse].\n" + "[Altinity Sink Connector] SHALL support `ADD COLUMN FIRST, AFTER` query from [MySQL] to [ClickHouse].\n" "\n" ), link=None, @@ -1047,7 +1103,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support multiple `ADD COLUMN` query from [MySQL] to [CLickHouse].\n" + "[Altinity Sink Connector] SHALL support multiple `ADD COLUMN` query from [MySQL] to [ClickHouse].\n" "\n" "\n" "\n" @@ -1065,7 +1121,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support `MODIFY COLUMN data_type` query from [MySQL] to [CLickHouse].\n" + "[Altinity Sink Connector] SHALL support `MODIFY COLUMN data_type` query from [MySQL] to [ClickHouse].\n" "\n" "\n" ), @@ -1082,7 +1138,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support `MODIFY COLUMN data_type NULL/NOT NULL` query from [MySQL] to [CLickHouse].\n" + "[Altinity Sink Connector] SHALL support `MODIFY COLUMN data_type NULL/NOT NULL` query from [MySQL] to [ClickHouse].\n" "\n" ), link=None, @@ -1098,7 +1154,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support `MODIFY COLUMN data_type DEFAULT` query from [MySQL] to [CLickHouse].\n" + "[Altinity Sink Connector] SHALL support `MODIFY COLUMN data_type DEFAULT` query from [MySQL] to [ClickHouse].\n" "\n" ), link=None, @@ -1114,7 +1170,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support `MODIFY COLUMN data_type FIRST, AFTER` query from [MySQL] to [CLickHouse].\n" + "[Altinity Sink Connector] SHALL support `MODIFY COLUMN data_type FIRST, AFTER` query from [MySQL] to [ClickHouse].\n" "\n" ), link=None, @@ -1130,7 +1186,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support multiple `MODIFY COLUMN` query from [MySQL] to [CLickHouse].\n" + "[Altinity Sink Connector] SHALL support multiple `MODIFY COLUMN` query from [MySQL] to [ClickHouse].\n" "\n" ), link=None, @@ -1146,7 +1202,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support `CHANGE COLUMN old_name new_name datatype NULL/NOT NULL` query from [MySQL] to [CLickHouse].\n" + "[Altinity Sink Connector] SHALL support `CHANGE COLUMN old_name new_name datatype NULL/NOT NULL` query from [MySQL] to [ClickHouse].\n" "\n" ), link=None, @@ -1162,7 +1218,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support `CHANGE COLUMN FIRST, AFTER` query from [MySQL] to [CLickHouse].\n" + "[Altinity Sink Connector] SHALL support `CHANGE COLUMN FIRST, AFTER` query from [MySQL] to [ClickHouse].\n" "\n" ), link=None, @@ -1178,7 +1234,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support multiple `CHANGE COLUMN` query from [MySQL] to [CLickHouse].\n" + "[Altinity Sink Connector] SHALL support multiple `CHANGE COLUMN` query from [MySQL] to [ClickHouse].\n" "\n" ), link=None, @@ -1194,7 +1250,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support `DROP COLUMN` query from [MySQL] to [CLickHouse].\n" + "[Altinity Sink Connector] SHALL support `DROP COLUMN` query from [MySQL] to [ClickHouse].\n" "\n" ), link=None, @@ -1210,7 +1266,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support multiple `DROP COLUMN` query from [MySQL] to [CLickHouse].\n" + "[Altinity Sink Connector] SHALL support multiple `DROP COLUMN` query from [MySQL] to [ClickHouse].\n" "\n" ), link=None, @@ -1226,7 +1282,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support `RENAME COLUMN col1 to col2` query from [MySQL] to [CLickHouse].\n" + "[Altinity Sink Connector] SHALL support `RENAME COLUMN col1 to col2` query from [MySQL] to [ClickHouse].\n" "\n" ), link=None, @@ -1242,7 +1298,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support multiple `RENAME COLUMN col1 to col2` query from [MySQL] to [CLickHouse].\n" + "[Altinity Sink Connector] SHALL support multiple `RENAME COLUMN col1 to col2` query from [MySQL] to [ClickHouse].\n" "\n" ), link=None, @@ -1258,7 +1314,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support `ADD CONSTRAINT` query from [MySQL] to [CLickHouse].\n" + "[Altinity Sink Connector] SHALL support `ADD CONSTRAINT` query from [MySQL] to [ClickHouse].\n" "\n" ), link=None, @@ -1274,7 +1330,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support `DROP CONSTRAINT` query from [MySQL] to [CLickHouse].\n" + "[Altinity Sink Connector] SHALL support `DROP CONSTRAINT` query from [MySQL] to [ClickHouse].\n" "\n" ), link=None, @@ -1290,7 +1346,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] query SHALL support [MySQL] data replication to [CLickHouse] on queries to tables\n" + "[Altinity Sink Connector] query SHALL support [MySQL] data replication to [ClickHouse] on queries to tables\n" "with no `PRIMARY KEY`.\n" "\n" ), @@ -1307,7 +1363,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] query SHALL support [MySQL] data replication to [CLickHouse] on queries with the same order\n" + "[Altinity Sink Connector] query SHALL support [MySQL] data replication to [ClickHouse] on queries with the same order\n" "as simple `PRIMARY KEY` does.\n" "\n" ), @@ -1324,7 +1380,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] query SHALL support [MySQL] data replication to [CLickHouse] on queries with the same order \n" + "[Altinity Sink Connector] query SHALL support [MySQL] data replication to [ClickHouse] on queries with the same order \n" "as composite `PRIMARY KEY` does.\n" "\n" ), @@ -1341,7 +1397,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support [MySQL] replication to [CLickHouse] from multiple [MySQL] upstream servers.\n" + "[Altinity Sink Connector] SHALL support [MySQL] replication to [ClickHouse] from multiple [MySQL] upstream servers.\n" "\n" ), link=None, @@ -1357,7 +1413,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support [MySQL] replication to [CLickHouse] when using multiple downstream [ClickHouse] servers.\n" + "[Altinity Sink Connector] SHALL support [MySQL] replication to [ClickHouse] when using multiple downstream [ClickHouse] servers.\n" "\n" ), link=None, @@ -1373,7 +1429,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support [MySQL] replication to [CLickHouse] with archival mode that\n" + "[Altinity Sink Connector] SHALL support [MySQL] replication to [ClickHouse] with archival mode that\n" "SHALL ignore deletes for some or all tables in [ClickHouse].\n" "\n" ), @@ -1390,7 +1446,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support [MySQL] replication to [CLickHouse] with \n" + "[Altinity Sink Connector] SHALL support [MySQL] replication to [ClickHouse] with \n" "bootstrapping mode for the initial replication of very large tables\n" "that bypasses event stream by using [MySQL] dump files.\n" "\n" @@ -1408,7 +1464,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support ability to start replication to [CLickHouse] \n" + "[Altinity Sink Connector] SHALL support ability to start replication to [ClickHouse] \n" "from specific [MySQL] binlog position.\n" "\n" ), @@ -1425,7 +1481,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support [MySQL] replication to [CLickHouse] with support for\n" + "[Altinity Sink Connector] SHALL support [MySQL] replication to [ClickHouse] with support for\n" "defining column mapping and transformations rules.\n" "\n" ), @@ -1442,8 +1498,8 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support [MySQL] replication to [CLickHouse] replica table when it has fewer columns.\n" - "[MySQL] replication to [CLickHouse] is not available in all other cases of columns inconsistency .\n" + "[Altinity Sink Connector] SHALL support [MySQL] replication to [ClickHouse] replica table when it has fewer columns.\n" + "[MySQL] replication to [ClickHouse] is not available in all other cases of columns inconsistency .\n" "\n" ), link=None, @@ -1459,7 +1515,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support [MySQL] replication to [CLickHouse] with latency as close as possible to real-time.\n" + "[Altinity Sink Connector] SHALL support [MySQL] replication to [ClickHouse] with latency as close as possible to real-time.\n" "\n" ), link=None, @@ -1475,7 +1531,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support [MySQL] replication to [CLickHouse] more than 100,000 rows/sec.\n" + "[Altinity Sink Connector] SHALL support [MySQL] replication to [ClickHouse] more than 100,000 rows/sec.\n" "\n" ), link=None, @@ -1491,7 +1547,7 @@ type=None, uid=None, description=( - "[Altinity Sink Connector] SHALL support [MySQL] replication to [CLickHouse] with large daily data volumes of at least 20-30TB per day.\n" + "[Altinity Sink Connector] SHALL support [MySQL] replication to [ClickHouse] with large daily data volumes of at least 20-30TB per day.\n" "\n" ), link=None, @@ -1599,6 +1655,121 @@ num="28.1.1", ) +RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_SystemActions_Network = Requirement( + name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.SystemActions.Network", + version="1.0", + priority=None, + group=None, + type=None, + uid=None, + description=( + "[Altinity Sink Connector] SHALL be able to recover replication after network related interruptions happen, so that the data on the destination table is not lost.\n" + "\n" + "\n" + "| List of possible network related interruptions |\n" + "|---------------------------------------------------------------|\n" + "| Internal network interruptions in source database cluster |\n" + "| Network interruptions from source database to sink connector |\n" + "| Network interruptions from sink connector to clickhouse |\n" + "| Internal network interruptions in clickhouse database cluster |\n" + "\n" + "\n" + ), + link=None, + level=3, + num="29.1.1", +) + +RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_SystemActions_Process_Die = Requirement( + name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.SystemActions.Process.Die", + version="1.0", + priority=None, + group=None, + type=None, + uid=None, + description=( + "[Altinity Sink Connector] SHALL output an error and keep the existing data when one of the following or all of the \n" + "scenarios related to processes being killed happen:\n" + "\n" + "| Scenarios Related to Processes Being Killed |\n" + "|-------------------------------------------------------|\n" + "| Internal processes die in source database cluster |\n" + "| Sink connector dies |\n" + "| Internal processes die in clickhouse database cluster |\n" + "\n" + ), + link=None, + level=4, + num="29.2.1.1", +) + +RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_SystemActions_Process_Restarted = Requirement( + name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.SystemActions.Process.Restarted", + version="1.0", + priority=None, + group=None, + type=None, + uid=None, + description=( + "[Altinity Sink Connector] SHALL continue replication without losing any data after one of the following or \n" + "all of the scenarios related to processes being restarted happen:\n" + "\n" + "| Scenarios Related to Processes Being Restarted |\n" + "|-------------------------------------------------------------|\n" + "| Restart of some or all nodes in source database cluster |\n" + "| Restart of sink connector |\n" + "| Restart of some or all nodes in clickhouse database cluster |\n" + "\n" + ), + link=None, + level=4, + num="29.2.2.1", +) + +RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_SystemActions_Disk_OutOfSpace = Requirement( + name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.SystemActions.Disk.OutOfSpace", + version="1.0", + priority=None, + group=None, + type=None, + uid=None, + description=( + "[Altinity Sink Connector] SHALL output an error when:\n" + "\n" + "- Out of disk space on some node in source database cluster\n" + "- Out of disk space where sink connector is running\n" + "- Out of disk space on some node in clickhouse database cluster\n" + "\n" + "The error SHALL be shown so that the data on the source and destination tables is not lost due to the disk related issues.\n" + "\n" + ), + link=None, + level=4, + num="29.3.1.1", +) + +RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_SystemActions_Disk_Corrupted = Requirement( + name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.SystemActions.Disk.Corrupted", + version="1.0", + priority=None, + group=None, + type=None, + uid=None, + description=( + "[Altinity Sink Connector] SHALL output an error when:\n" + "\n" + "- Corruption on a disk used by some node in source database cluster\n" + "- Corruption on a disk where sink connector is running\n" + "- Corruption on a disk used by some node in clickhouse database cluster\n" + "\n" + "The error SHALL be shown so that the data on the source and destination tables is not lost due to the disk related issues.\n" + "\n" + ), + link=None, + level=4, + num="29.3.2.1", +) + RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_Prometheus = Requirement( name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Prometheus", version="1.0", @@ -1619,7 +1790,7 @@ ), link=None, level=2, - num="29.1", + num="30.1", ) SRS030_MySQL_to_ClickHouse_Replication = Specification( @@ -1815,6 +1986,11 @@ level=4, num="12.1.1.1", ), + Heading( + name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Queries.Inserts.PartitionTypes", + level=4, + num="12.1.1.2", + ), Heading(name="Updates", level=2, num="12.2"), Heading( name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Queries.Updates", @@ -2172,11 +2348,50 @@ level=3, num="28.1.1", ), - Heading(name="Prometheus ", level=1, num="29"), + Heading(name="System Actions", level=1, num="29"), + Heading(name="Handling Network Interruptions", level=2, num="29.1"), + Heading( + name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.SystemActions.Network", + level=3, + num="29.1.1", + ), + Heading(name="Handling Process Interruptions", level=2, num="29.2"), + Heading( + name="Behaviour When Different Processes Were Killed", level=3, num="29.2.1" + ), + Heading( + name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.SystemActions.Process.Die", + level=4, + num="29.2.1.1", + ), + Heading( + name="Behaviour When Different Processes Were Restarted", + level=3, + num="29.2.2", + ), + Heading( + name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.SystemActions.Process.Restarted", + level=4, + num="29.2.2.1", + ), + Heading(name="Behaviour When There Are Issues With Disk", level=2, num="29.3"), + Heading(name="Disk Is out of Space", level=3, num="29.3.1"), + Heading( + name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.SystemActions.Disk.OutOfSpace", + level=4, + num="29.3.1.1", + ), + Heading(name="Disk Is Corrupted", level=3, num="29.3.2"), + Heading( + name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.SystemActions.Disk.Corrupted", + level=4, + num="29.3.2.1", + ), + Heading(name="Prometheus", level=1, num="30"), Heading( name="RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Prometheus", level=2, - num="29.1", + num="30.1", ), ), requirements=( @@ -2207,6 +2422,7 @@ RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_DataTypes_Bytes, RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_Queries_Inserts, RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_Queries_Inserts_PartitionLimits, + RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_Queries_Inserts_PartitionTypes, RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_Queries_Updates, RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_Queries_Deletes, RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_TableSchemaCreation, @@ -2267,6 +2483,11 @@ RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_TableNames_Invalid, RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_ColumnNames_Special, RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_Interruption_ClickHouse_Instance_Stopped, + RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_SystemActions_Network, + RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_SystemActions_Process_Die, + RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_SystemActions_Process_Restarted, + RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_SystemActions_Disk_OutOfSpace, + RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_SystemActions_Disk_Corrupted, RQ_SRS_030_ClickHouse_MySQLToClickHouseReplication_Prometheus, ), content=""" @@ -2335,6 +2556,7 @@ * 12.1 [Inserts](#inserts) * 12.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Queries.Inserts](#rqsrs-030clickhousemysqltoclickhousereplicationqueriesinserts) * 12.1.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Queries.Inserts.PartitionLimits](#rqsrs-030clickhousemysqltoclickhousereplicationqueriesinsertspartitionlimits) + * 12.1.1.2 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Queries.Inserts.PartitionTypes](#rqsrs-030clickhousemysqltoclickhousereplicationqueriesinsertspartitiontypes) * 12.2 [Updates](#updates) * 12.2.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Queries.Updates](#rqsrs-030clickhousemysqltoclickhousereplicationqueriesupdates) * 12.3 [Deletes](#deletes) @@ -2448,8 +2670,21 @@ * 28 [Replication Interruption](#replication-interruption) * 28.1 [Retry Replication When ClickHouse Instance Is Not Active](#retry-replication-when-clickhouse-instance-is-not-active) * 28.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Interruption.ClickHouse.Instance.Stopped](#rqsrs-030clickhousemysqltoclickhousereplicationinterruptionclickhouseinstancestopped) -* 29 [Prometheus ](#prometheus-) - * 29.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Prometheus](#rqsrs-030clickhousemysqltoclickhousereplicationprometheus) +* 29 [System Actions](#system-actions) + * 29.1 [Handling Network Interruptions](#handling-network-interruptions) + * 29.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.SystemActions.Network](#rqsrs-030clickhousemysqltoclickhousereplicationsystemactionsnetwork) + * 29.2 [Handling Process Interruptions](#handling-process-interruptions) + * 29.2.1 [Behaviour When Different Processes Were Killed](#behaviour-when-different-processes-were-killed) + * 29.2.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.SystemActions.Process.Die](#rqsrs-030clickhousemysqltoclickhousereplicationsystemactionsprocessdie) + * 29.2.2 [Behaviour When Different Processes Were Restarted](#behaviour-when-different-processes-were-restarted) + * 29.2.2.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.SystemActions.Process.Restarted](#rqsrs-030clickhousemysqltoclickhousereplicationsystemactionsprocessrestarted) + * 29.3 [Behaviour When There Are Issues With Disk](#behaviour-when-there-are-issues-with-disk) + * 29.3.1 [Disk Is out of Space](#disk-is-out-of-space) + * 29.3.1.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.SystemActions.Disk.OutOfSpace](#rqsrs-030clickhousemysqltoclickhousereplicationsystemactionsdiskoutofspace) + * 29.3.2 [Disk Is Corrupted](#disk-is-corrupted) + * 29.3.2.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.SystemActions.Disk.Corrupted](#rqsrs-030clickhousemysqltoclickhousereplicationsystemactionsdiskcorrupted) +* 30 [Prometheus](#prometheus) + * 30.1 [RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Prometheus](#rqsrs-030clickhousemysqltoclickhousereplicationprometheus) ## Introduction @@ -2469,187 +2704,232 @@ ## Test Schema ```yaml -Services: - - Source Database Cluster - - Sink Connector - - ClickHouse Database Cluster - -SourceTables: - DatabaseType: [MySQL, PostgreSQL, MariaDB, MongoDB] - DatabaseVersions: - MySQL: [8.0] - PostgreSQL: null - MariaDB: null - MongoDB: null - DatabaseClusterConfiguration: - MySQL: null - PostgreSQL: null - MariaDB: null - MongoDB: null - EngineType: - MySQL: [InnoDB] - PostgreSQL: null - MariaDB: null - MongoDB: null - Schema: - TableName: - length: - MySQL: 64 characters - PostgreSQL: null - MariaDB: null - MongoDB: null - encodings: - ASCII: null - UTF-8: null - otherEncodings: [armscii8, big5, binary, cp1250, cp1251, cp1256, cp1257, cp850, cp852, cp866, cp932, dec8, - eucjpms, euckr, gb18030, gb2312, gbk, geostd8, greek, hebrew, hp8, keybcs2, koi8r, koi8u, latin1, - latin2, latin5, latin7, macce, macroman, sjis, swe7, tis620, ucs2, ujis, utf16, utf16le, utf32, utf8mb3, utf8mb4] - Partitioning: - MySQL: [RANGE, LIST, COLUMNS, HASH, KEY, Subpartitioning] - Columns: - DefaultValues: [Numeric Types, Date and Time Types, String Types, ENUM Types, SET Types, BOOLEAN, Binary Types] - Type: - MySQL: [Calculated columns, Materialized columns, Primary Key Columns, Foreign Key Columns, Index Columns, Unique Columns, Auto-Increment Columns, Timestamp/DateTime Columns, ENUM and SET Columns, Spatial Columns] - Name: - length: - mysql: null - PostgreSQL: null - MariaDB: null - MongoDB: null - ASCII: null - UTF-8: null - otherEncodings: [armscii8, big5, binary, cp1250, cp1251, cp1256, cp1257, cp850, cp852, cp866, cp932, dec8, - eucjpms, euckr, gb18030, gb2312, gbk, geostd8, greek, hebrew, hp8, keybcs2, koi8r, koi8u, latin1, - latin2, latin5, latin7, macce, macroman, sjis, swe7, tis620, ucs2, ujis, utf16, utf16le, utf32, utf8mb3, utf8mb4] - DataType: - - DECIMAL(2,1) - - DECIMAL(30, 10) - - DOUBLE - - DATE - - DATETIME(1-6) - - TIME - - TIME(1-6) - - INT - - INT - - INT UNSIGNED - - INT UNSIGNED - - BIGINT - - BIGINT - - BIGINT UNSIGNED NOT NULL - - TINYINT - - TINYINT UNSIGNED - - SMALLINT - - SMALLINT UNSIGNED - - MEDIUMINT - - MEDIUMINT NOT NULL - - MEDIUMINT UNSIGNED - - CHAR - - TEXT - - VARCHAR(1-32766) - - BLOB - - MEDIUMBLOB - - LONGBLOB - - BINARY - - VARBINARY(4) - DataValue: - Numeric: [Min, Max, 0, -infinity, +infinity, nan, random value] - Decimal: [Min value based on precision and scale, Max value based on precision and scale, 0, -0.0001, "0.0001", - -Max value based on precision and scale, +Max value based on precision and scale, "NaN", - A random value within precision and scale] - String: - bytes: [null bytes, ...] - UTF-8: [] - ASCII: [] - otherEncodings: [armscii8, big5, binary, cp1250, cp1251, cp1256, cp1257, cp850, cp852, cp866, cp932, dec8, - eucjpms, euckr, gb18030, gb2312, gbk, geostd8, greek, hebrew, hp8, keybcs2, koi8r, koi8u, latin1, - latin2, latin5, latin7, macce, macroman, sjis, swe7, tis620, ucs2, ujis, utf16, utf16le, - utf32, utf8mb3, utf8mb4] - - TableOperations: +clickhouse-sink-connector: + Services: + - Source Database Cluster + - Sink Connector + - ClickHouse Database Cluster + SourceTables: + DatabaseType: [MySQL, PostgreSQL, MariaDB, MongoDB] MySQL: - - INSERT - - UPDATE - - DELETE - - SELECT - - ALTERs: - - ADD COLUMN - - ADD COLUMN NULL/NOT NULL - - ADD COLUMN DEFAULT - - ADD COLUMN FIRST, AFTER - - DROP COLUMN - - MODIFY COLUMN data_type - - MODIFY COLUMN data_type NULL/NOT NULL - - MODIFY COLUMN data_type DEFAULT - - MODIFY COLUMN FIRST, AFTER - - MODIFY COLUMN old_name new_name datatype NULL/NOT NULL - - RENAME COLUMN col1 to col2 - - CHANGE COLUMN FIRST, AFTER - - ALTER COLUMN col_name ADD DEFAULT - - ALTER COLUMN col_name ADD DROP DEFAULT - - ADD PRIMARY KEY - MariaDB: null - PostgreSQL: null - MongoDB: null - -SinkConnector: - Version: [latest] - Configuration: The full list is inside configurations below - -DestinationTables: - DatabaseType: [ClickHouse] - DatabaseClusterConfiguration: - - One node - - Sharded cluster secure - - Cluster with Multiple Shards - - Cluster with Replication (but a single shard) - - Cluster with Multiple Shards and Replicas - - Secure Cluster - DatabaseVersion: [22.8, 23.3, 23,11, latest] - EngineType: [ReplacingMergeTree, ReplicatedReplacingMergeTree] - Schema: - TableName: - length: - MySQL: 64 characters - PostgreSQL: null - MariaDB: null - MongoDB: null - encodings: - ASCII: null - UTF-8: null - otherEncodings: [armscii8, big5, binary, cp1250, cp1251, cp1256, cp1257, cp850, cp852, cp866, cp932, dec8, - eucjpms, euckr, gb18030, gb2312, gbk, geostd8, greek, hebrew, hp8, keybcs2, koi8r, koi8u, latin1, - latin2, latin5, latin7, macce, macroman, sjis, swe7, tis620, ucs2, ujis, utf16, utf16le, utf32, utf8mb3, utf8mb4] - TableOperations: - - INSERT - - UPDATE - - DELETE - - SELECT - - ALTER - -SystemActions: - Network: - - Internal network interruptions in source database cluster - - Network interruptions from source database to sink connector - - Network interruptions from sink connector to clickhouse - - Internal network interruptions in clickhouse database cluster - Process: - Die: - - Internal processes die in source database cluster - - Sink connector dies - - Internal processes die in clickhouse database cluster - Restarted: - - Restart of some or all nodes in source database cluster - - Restart of sink connector - - Restart of some or all nodes in clickhouse database cluster - Disk: - OutOfSpace: - - Out of disk space on some node in source database cluster - - Out of disk space where sink connector is running - - Out of disk space on some node in clickhouse database cluster - Corruptions: - - Corruption on a disk used by some node in source database cluster - - Corruption on a disk where sink connector is running - - Corruption on a disk used by some node in clickhouse database cluster + DatabaseVersion: 8.0 + DatabaseClusterConfiguration: + - MySQL Cluster (NDB Cluster) + - InnoDB Cluster + - Replication Clusters + - Galera Cluster + EngineType: [InnoDB, MyISAM, MEMORY, CSV, ARCHIVE, BLACKHOLE, FEDERATED, EXAMPLE] + Schema: + TableName: + Length: 64 characters + Encodings: + - ASCII + - UTF-8 + - otherEncodings: [armscii8, big5, binary, cp1250, cp1251, cp1256, cp1257, cp850, cp852, cp866, cp932, dec8, + eucjpms, euckr, gb18030, gb2312, gbk, geostd8, greek, hebrew, hp8, keybcs2, koi8r, koi8u, latin1, + latin2, latin5, latin7, macce, macroman, sjis, swe7, tis620, ucs2, ujis, utf16, utf16le, utf32, utf8mb3, utf8mb4] + Partitioning: + - RANGE + - LIST + - COLUMNS + - HASH + - KEY + - Subpartitioning + Columns: + DefaultValues: [Numeric Types, Date and Time Types, String Types, ENUM Types, SET Types, BOOLEAN, Binary Types] + Type: [Calculated Columns, Materialized Columns, Primary Key Columns, Foreign Key Columns, + Index Columns, Unique Columns, Auto-Increment Columns, Timestamp/DateTime Columns, + ENUM and SET Columns, Spatial Columns] + Name: + Length: 64 characters + Encodings: + - ASCII + - UTF-8 + - OtherEncodings: [armscii8, big5, binary, cp1250, cp1251, cp1256, cp1257, cp850, cp852, cp866, cp932, dec8, + eucjpms, euckr, gb18030, gb2312, gbk, geostd8, greek, hebrew, hp8, keybcs2, koi8r, koi8u, latin1, + latin2, latin5, latin7, macce, macroman, sjis, swe7, tis620, ucs2, ujis, utf16, utf16le, utf32, utf8mb3, utf8mb4] + DataType: + - DECIMAL(2,1) + - DECIMAL(30, 10) + - DOUBLE + - DATE + - DATETIME(1-6) + - TIME(1-6) + - INT + - INT UNSIGNED + - BIGINT + - BIGINT UNSIGNED NOT NULL + - TINYINT + - TINYINT UNSIGNED + - SMALLINT + - SMALLINT UNSIGNED + - MEDIUMINT + - MEDIUMINT UNSIGNED + - CHAR + - TEXT + - VARCHAR(1-32766) + - BLOB + - MEDIUMBLOB + - LONGBLOB + - BINARY + - VARBINARY(4) + DataValue: + Numeric: [Min, Max, 0, -infinity, +infinity, nan, random value] + Decimal: [Min value based on precision and scale, Max value based on precision and scale, 0, -0.0001, + "0.0001", -Max value based on precision and scale, +Max value based on precision and scale, + "NaN", A random value within precision and scale] + String: + Bytes: [null bytes] + UTF-8: [C0 Controls and Basic Latin, Latin Extended-A, Spacing Modifiers, Diacritical Marks, + Greek and Coptic, Cyrillic Basic, Currency Symbols, Mathematical Operators, Miscellaneous Symbols, Dingbats] + ASCII: [All ASCII characters] + otherEncodings: [armscii8, big5, binary, cp1250, cp1251, cp1256, cp1257, cp850, cp852, cp866, cp932, dec8, + eucjpms, euckr, gb18030, gb2312, gbk, geostd8, greek, hebrew, hp8, keybcs2, koi8r, koi8u, latin1, + latin2, latin5, latin7, macce, macroman, sjis, swe7, tis620, ucs2, ujis, utf16, utf16le, + utf32, utf8mb3, utf8mb4] + TableOperations: + - INSERT + - UPDATE + - DELETE + - SELECT + - ALTER: + - ADD COLUMN + - ADD COLUMN NULL/NOT NULL + - ADD COLUMN DEFAULT + - ADD COLUMN FIRST, AFTER + - DROP COLUMN + - MODIFY COLUMN data_type + - MODIFY COLUMN data_type NULL/NOT NULL + - MODIFY COLUMN data_type DEFAULT + - MODIFY COLUMN FIRST, AFTER + - MODIFY COLUMN old_name new_name datatype NULL/NOT NULL + - RENAME COLUMN col1 to col2 + - CHANGE COLUMN FIRST, AFTER + - ALTER COLUMN col_name ADD DEFAULT + - ALTER COLUMN col_name ADD DROP DEFAULT + - ADD PRIMARY KEY + PostgreSQL: + DatabaseVersion: null + DatabaseClusterConfiguration: null + EngineType: null + Schema: + TableName: + Length: null + Encodings: null + Partitioning: null + Columns: + DefaultValues: null + Type: null + Name: + Length: null + Encodings: null + DataType: null + DataValue: + Numeric: [] + Decimal: [] + String: + Bytes: [] + UTF-8: [] + ASCII: [] + otherEncodings: null + TableOperations: null + MariaDB: + DatabaseVersion: null + DatabaseClusterConfiguration: null + EngineType: null + Schema: + TableName: + Length: null + Encodings: null + Partitioning: null + Columns: + DefaultValues: null + Type: null + Name: + Length: null + Encodings: null + DataType: null + DataValue: + Numeric: [] + Decimal: [] + String: + Bytes: [] + UTF-8: [] + ASCII: [] + otherEncodings: null + TableOperations: null + MongoDB: + DatabaseVersion: null + DatabaseClusterConfiguration: null + EngineType: null + Schema: + TableName: + Length: null + Encodings: null + Partitioning: null + Columns: + DefaultValues: null + Type: null + Name: + Length: null + Encodings: null + DataType: null + DataValue: + Numeric: [] + Decimal: [] + String: + Bytes: [] + UTF-8: [] + ASCII: [] + otherEncodings: null + TableOperations: null + + SinkConnector: + Version: [latest] + Configuration: The full list is inside configurations below + + DestinationTables: + DatabaseType: [ClickHouse] + DatabaseClusterConfiguration: + - One node + - Sharded cluster secure + - Cluster with Multiple Shards + - Cluster with Replication (but a single shard) + - Cluster with Multiple Shards and Replicas + - Secure Cluster + DatabaseVersion: [22.8, 23.3, 23.11, 23.12] + EngineType: [ReplacingMergeTree, ReplicatedReplacingMergeTree] + TableOperations: + - INSERT + - UPDATE + - DELETE + - SELECT + - ALTER + + SystemActions: + Network: + - Internal network interruptions in source database cluster + - Network interruptions from source database to sink connector + - Network interruptions from sink connector to clickhouse + - Internal network interruptions in clickhouse database cluster + Process: + Die: + - Internal processes die in source database cluster + - Sink connector dies + - Internal processes die in clickhouse database cluster + Restarted: + - Restart of some or all nodes in source database cluster + - Restart of sink connector + - Restart of some or all nodes in clickhouse database cluster + Disk: + OutOfSpace: + - Out of disk space on some node in source database cluster + - Out of disk space where sink connector is running + - Out of disk space on some node in clickhouse database cluster + Corruptions: + - Corruption on a disk used by some node in source database cluster + - Corruption on a disk where sink connector is running + - Corruption on a disk used by some node in clickhouse database cluster ``` ## Configuration @@ -2721,13 +3001,13 @@ ### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Consistency -[Altinity Sink Connector] SHALL support consistent data replication from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support consistent data replication from [MySQL] to [ClickHouse]. ### Multiple MySQL Masters #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Consistency.MultipleMySQLMasters -[Altinity Sink Connector] SHALL support consistent data replication from [MySQL] to [CLickHouse] when one or more MySQL +[Altinity Sink Connector] SHALL support consistent data replication from [MySQL] to [ClickHouse] when one or more MySQL masters are going down. ### Deduplication @@ -2749,7 +3029,7 @@ #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.OnlyOnceGuarantee version: 1.0 -[Altinity Sink Connector] SHALL support [MySQL] replication to [CLickHouse] with only-once guarantee. +[Altinity Sink Connector] SHALL support [MySQL] replication to [ClickHouse] with only-once guarantee. Block level de-duplication SHALL be used if it is going to replicated tables but the publisher SHALL publish only once. @@ -2759,14 +3039,9 @@ 2. [MySQL] database event stream provider crash 3. [MySQL] restart 3. [ClickHouse] server crash -4. [Clickhouse] server restart -7. [Debezium] server crash -8. [Debezium] server restart -9. [Altinity Sink Connector] server crash -10. [Altinity Sink Connector] server restart -11. [Schemaregistry] server crash -12. [Schemaregistry] server restart -13. [Zookeeper] read only mode +4. [ClickHouse] server restart +5. [Altinity Sink Connector] server crash +6. [Altinity Sink Connector] server restart ## Transactions @@ -2790,8 +3065,18 @@ ### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MySQLStorageEngines.ReplacingMergeTree version: 1.0 -[Altinity Sink Connector] SHALL support replication of tables that use "InnoDB" [MySQL] storage engine to -"ReplacingMergeTree" [ClickHouse] table engine. +[Altinity Sink Connector] SHALL support replication of tables that use the following storage engines in MySQL: + +| Storage Engines | +|-----------------| +| InnoDB | +| MyISAM | +| MEMORY | +| CSV | +| ARCHIVE | +| BLACKHOLE | +| FEDERATED | +| EXAMPLE | #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MySQLStorageEngines.ReplacingMergeTree.VirtualColumnNames version: 1.0 @@ -2819,6 +3104,32 @@ [Altinity Sink Connector] SHALL support data replication to [ClickHouse] of tables with any datatypes that [MySQL] supports. +| MySQL Datatypes | +|--------------------| +| DECIMAL | +| DOUBLE | +| DATE | +| DATETIME(1-6) | +| TIME(1-6) | +| INT | +| INT UNSIGNED | +| BIGINT | +| BIGINT UNSIGNED | +| TINYINT | +| TINYINT UNSIGNED | +| SMALLINT | +| SMALLINT UNSIGNED | +| MEDIUMINT | +| MEDIUMINT UNSIGNED | +| CHAR | +| TEXT | +| VARCHAR(1-32766) | +| BLOB | +| MEDIUMBLOB | +| LONGBLOB | +| BINARY | +| VARBINARY | + ### Integer Types #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.DataTypes.IntegerTypes @@ -2847,7 +3158,7 @@ #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.DataTypes.Decimal version: 1.0 -[Altinity Sink Connector] SHALL support data replication to [CLickHouse] of tables that contain columns with +[Altinity Sink Connector] SHALL support data replication to [ClickHouse] of tables that contain columns with 'Decimal' data types as they supported by [MySQL]. [ClickHouse]'s 'Decimal32(S)', 'Decimal64(S)', 'Decimal128(S)', 'Decimal256(S)' also can be @@ -2864,7 +3175,7 @@ #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.DataTypes.Double version: 1.0 -[Altinity Sink Connector] SHALL support data replication to [CLickHouse] of tables that contain columns with +[Altinity Sink Connector] SHALL support data replication to [ClickHouse] of tables that contain columns with 'Double' data types as they supported by [MySQL]. Data types connection table: @@ -2878,7 +3189,7 @@ #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.DataTypes.DateTime version: 1.0 -[Altinity Sink Connector] SHALL support data replication to [CLickHouse] of tables that contain columns with 'Data' and 'Time' +[Altinity Sink Connector] SHALL support data replication to [ClickHouse] of tables that contain columns with 'Data' and 'Time' data types as they supported by [MySQL]. Data types connection table: @@ -2897,7 +3208,7 @@ #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.DataTypes.Binary version: 1.0 -[Altinity Sink Connector] SHALL support data replication to [CLickHouse] replication of tables that contain columns with 'Binary' +[Altinity Sink Connector] SHALL support data replication to [ClickHouse] replication of tables that contain columns with 'Binary' data types as they supported by [MySQL]. Data types connection table: @@ -2912,7 +3223,7 @@ #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.DataTypes.String version: 1.0 -[Altinity Sink Connector] SHALL support data replication to [CLickHouse] of tables that contain columns with 'String' +[Altinity Sink Connector] SHALL support data replication to [ClickHouse] of tables that contain columns with 'String' data types as they supported by [MySQL]. Data types connection table: @@ -2929,7 +3240,7 @@ #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.DataTypes.BlobTypes version: 1.0 -[Altinity Sink Connector] SHALL support data replication to [CLickHouse] of tables that contain columns with 'Blob' [MySQL] +[Altinity Sink Connector] SHALL support data replication to [ClickHouse] of tables that contain columns with 'Blob' [MySQL] data types and correctly unhex() them. ```sql @@ -2949,7 +3260,7 @@ #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.DataTypes.Nullable version: 1.0 -[Altinity Sink Connector] SHALL support data replication to [CLickHouse] of tables that contain columns with NULL [MySQL] +[Altinity Sink Connector] SHALL support data replication to [ClickHouse] of tables that contain columns with NULL [MySQL] data types if this expected `Nullable(DataType)` construction should be used. For example, [MySQL] `VARCHAR(*)` maps to [ClickHouse] `Nullable(String)` and MySQL @@ -2960,7 +3271,7 @@ #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.DataTypes.EnumToEnum version: 1.0 -[Altinity Sink Connector] SHALL support data replication to [CLickHouse] of tables that contain columns with 'ENUM' +[Altinity Sink Connector] SHALL support data replication to [ClickHouse] of tables that contain columns with 'ENUM' data types as they supported by [MySQL]. Data types connection table: @@ -2972,7 +3283,7 @@ #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.DataTypes.EnumToString version: 1.0 -[Altinity Sink Connector] SHALL support data replication to [CLickHouse] of tables that contain columns with 'ENUM' +[Altinity Sink Connector] SHALL support data replication to [ClickHouse] of tables that contain columns with 'ENUM' data types as they supported by [MySQL]. Data types connection table: @@ -2986,7 +3297,7 @@ #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.DataTypes.JSON version: 1.0 -[Altinity Sink Connector] SHALL support data replication to [CLickHouse] of tables that contain columns with 'JSON' +[Altinity Sink Connector] SHALL support data replication to [ClickHouse] of tables that contain columns with 'JSON' data types as they supported by [MySQL]. Data types connection table: @@ -3000,7 +3311,7 @@ #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.DataTypes.Year version: 1.0 -[Altinity Sink Connector] SHALL support data replication to [CLickHouse] of tables that contain columns with 'Year' +[Altinity Sink Connector] SHALL support data replication to [ClickHouse] of tables that contain columns with 'Year' data types as they supported by [MySQL]. Data types connection table: @@ -3014,7 +3325,7 @@ #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.DataTypes.Bytes version: 1.0 -[Altinity Sink Connector] SHALL support data replication to [CLickHouse] of tables that contain columns with 'BIT(m)' +[Altinity Sink Connector] SHALL support data replication to [ClickHouse] of tables that contain columns with 'BIT(m)' data types where m: 2 - 64 as they supported by [MySQL]. Data types connection table: @@ -3031,35 +3342,49 @@ #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Queries.Inserts version: 1.0 -[Altinity Sink Connector] SHALL support new data inserts replication from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support new data inserts replication from [MySQL] to [ClickHouse]. ##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Queries.Inserts.PartitionLimits version: 1.0 -[Altinity Sink Connector] SHALL support correct data inserts replication from [MySQL] to [CLickHouse] when partition +[Altinity Sink Connector] SHALL support correct data inserts replication from [MySQL] to [ClickHouse] when partition limits are hitting or avoid such situations. +##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Queries.Inserts.PartitionTypes +version: 1.0 + +[Altinity Sink Connector] SHALL support correct data inserts replication from [MySQL] to [ClickHouse] when the table in source table is partitioned with the following logic. + +| Partition Type | +|----------------| +| RANGE | +| LIST | +| COLUMNS | +| HASH | +| KEY | +| Subpartition | + ### Updates #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Queries.Updates version: 1.0 -[Altinity Sink Connector] SHALL support data updates replication from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support data updates replication from [MySQL] to [ClickHouse]. ### Deletes #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Queries.Deletes version: 1.0 -[Altinity Sink Connector] SHALL support data deletes replication from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support data deletes replication from [MySQL] to [ClickHouse]. ## Table Schema Creation ### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.TableSchemaCreation version: 1.0 -[Altinity Sink Connector]SHALL support the following ways to replicate schema from [MySQL] to [CLickHouse]: +[Altinity Sink Connector]SHALL support the following ways to replicate schema from [MySQL] to [ClickHouse]: * auto-create option * `clickhouse_loader` script * `chump` utility @@ -3069,26 +3394,26 @@ #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.TableSchemaCreation.AutoCreate version: 1.0 -[Altinity Sink Connector] SHALL support auto table creation from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support auto table creation from [MySQL] to [ClickHouse]. ##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.TableSchemaCreation.MultipleAutoCreate version: 1.0 -[Altinity Sink Connector] SHALL support auto creation of multiple tables from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support auto creation of multiple tables from [MySQL] to [ClickHouse]. ### Auto Drop #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.TableSchemaCreation.AutoDrop version: 1.0 -[Altinity Sink Connector] SHALL support `DROP TABLE` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `DROP TABLE` query from [MySQL] to [ClickHouse]. ## Alter ### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter version: 1.0 -[Altinity Sink Connector] SHALL support all `ALTER` queries. +[Altinity Sink Connector] SHALL support the following `ALTER` queries. | MySQL | ClickHouse | |--------------------------------------------------------|-----------------------------------------------------------------| @@ -3114,112 +3439,112 @@ #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.AddIndex version: 1.0 -[Altinity Sink Connector] SHALL support `ADD INDEX` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `ADD INDEX` query from [MySQL] to [ClickHouse]. ### Add Key #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.AddKey version: 1.0 -[Altinity Sink Connector] SHALL support `ADD Key` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `ADD Key` query from [MySQL] to [ClickHouse]. ### Add FullText #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.AddFullText version: 1.0 -[Altinity Sink Connector] SHALL support `ADD FULLTEXT` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `ADD FULLTEXT` query from [MySQL] to [ClickHouse]. ### Add Special #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.AddSpecial version: 1.0 -[Altinity Sink Connector] SHALL support `ADD SPECIAL` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `ADD SPECIAL` query from [MySQL] to [ClickHouse]. ### Drop Check #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.DropCheck version: 1.0 -[Altinity Sink Connector] SHALL support `DROP CHECK` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `DROP CHECK` query from [MySQL] to [ClickHouse]. ### Drop Default #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.DropDefault version: 1.0 -[Altinity Sink Connector] SHALL support `DROP DEFAULT` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `DROP DEFAULT` query from [MySQL] to [ClickHouse]. ### Check #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.Check version: 1.0 -[Altinity Sink Connector] SHALL support `ALTER CHECK` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `ALTER CHECK` query from [MySQL] to [ClickHouse]. ### Constraint #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.Constraint version: 1.0 -[Altinity Sink Connector] SHALL support `ALTER CONSTRAINT` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `ALTER CONSTRAINT` query from [MySQL] to [ClickHouse]. ### Index #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.Index version: 1.0 -[Altinity Sink Connector] SHALL support `ALTER INDEX` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `ALTER INDEX` query from [MySQL] to [ClickHouse]. ### Character Set #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.CharacterSet version: 1.0 -[Altinity Sink Connector] SHALL support `ALTER CHARACTER SET` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `ALTER CHARACTER SET` query from [MySQL] to [ClickHouse]. ### Convert To Character Set #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.ConvertToCharacterSet version: 1.0 -[Altinity Sink Connector] SHALL support `ALTER CONVERT TO CHARACTER SET` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `ALTER CONVERT TO CHARACTER SET` query from [MySQL] to [ClickHouse]. ### Algorithm #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.Algorithm version: 1.0 -[Altinity Sink Connector] SHALL support `ALTER ALGORITHM` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `ALTER ALGORITHM` query from [MySQL] to [ClickHouse]. ### Force #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.Force version: 1.0 -[Altinity Sink Connector] SHALL support `ALTER FORCE` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `ALTER FORCE` query from [MySQL] to [ClickHouse]. ### Lock #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.Lock version: 1.0 -[Altinity Sink Connector] SHALL support `ALTER LOCK` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `ALTER LOCK` query from [MySQL] to [ClickHouse]. ### Unlock #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.Unlock version: 1.0 -[Altinity Sink Connector] SHALL support `ALTER UNLOCK` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `ALTER UNLOCK` query from [MySQL] to [ClickHouse]. ### Validation #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.Validation version: 1.0 -[Altinity Sink Connector] SHALL support `ALTER VALIDATION` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `ALTER VALIDATION` query from [MySQL] to [ClickHouse]. ### Columns @@ -3228,28 +3553,28 @@ ##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.Columns.Add version: 1.0 -[Altinity Sink Connector] SHALL support `ADD COLUMN` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `ADD COLUMN` query from [MySQL] to [ClickHouse]. ##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.Columns.Add.NullNotNull version: 1.0 -[Altinity Sink Connector] SHALL support `ADD COLUMN NULL/NOT NULL` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `ADD COLUMN NULL/NOT NULL` query from [MySQL] to [ClickHouse]. ##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.Columns.Add.Default version: 1.0 -[Altinity Sink Connector] SHALL support `ADD COLUMN DEFAULT` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `ADD COLUMN DEFAULT` query from [MySQL] to [ClickHouse]. ##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.Columns.Add.FirstAfter version: 1.0 -[Altinity Sink Connector] SHALL support `ADD COLUMN FIRST, AFTER` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `ADD COLUMN FIRST, AFTER` query from [MySQL] to [ClickHouse]. ##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.Columns.Add.Multiple version: 1.0 -[Altinity Sink Connector] SHALL support multiple `ADD COLUMN` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support multiple `ADD COLUMN` query from [MySQL] to [ClickHouse]. @@ -3258,83 +3583,83 @@ ##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.Columns.Modify version: 1.0 -[Altinity Sink Connector] SHALL support `MODIFY COLUMN data_type` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `MODIFY COLUMN data_type` query from [MySQL] to [ClickHouse]. ##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.Columns.Modify.NullNotNull version: 1.0 -[Altinity Sink Connector] SHALL support `MODIFY COLUMN data_type NULL/NOT NULL` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `MODIFY COLUMN data_type NULL/NOT NULL` query from [MySQL] to [ClickHouse]. ##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.Columns.Modify.Default version: 1.0 -[Altinity Sink Connector] SHALL support `MODIFY COLUMN data_type DEFAULT` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `MODIFY COLUMN data_type DEFAULT` query from [MySQL] to [ClickHouse]. ##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.Columns.Modify.FirstAfter version: 1.0 -[Altinity Sink Connector] SHALL support `MODIFY COLUMN data_type FIRST, AFTER` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `MODIFY COLUMN data_type FIRST, AFTER` query from [MySQL] to [ClickHouse]. ##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.Columns.Modify.Multiple version: 1.0 -[Altinity Sink Connector] SHALL support multiple `MODIFY COLUMN` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support multiple `MODIFY COLUMN` query from [MySQL] to [ClickHouse]. #### Change ##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.Columns.Change.NullNotNullOldNew version: 1.0 -[Altinity Sink Connector] SHALL support `CHANGE COLUMN old_name new_name datatype NULL/NOT NULL` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `CHANGE COLUMN old_name new_name datatype NULL/NOT NULL` query from [MySQL] to [ClickHouse]. ##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.Columns.Change.FirstAfter version: 1.0 -[Altinity Sink Connector] SHALL support `CHANGE COLUMN FIRST, AFTER` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `CHANGE COLUMN FIRST, AFTER` query from [MySQL] to [ClickHouse]. ##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.Columns.Change.Multiple version: 1.0 -[Altinity Sink Connector] SHALL support multiple `CHANGE COLUMN` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support multiple `CHANGE COLUMN` query from [MySQL] to [ClickHouse]. #### Drop ##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.Columns.Drop version: 1.0 -[Altinity Sink Connector] SHALL support `DROP COLUMN` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `DROP COLUMN` query from [MySQL] to [ClickHouse]. ##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.Columns.Drop.Multiple version: 1.0 -[Altinity Sink Connector] SHALL support multiple `DROP COLUMN` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support multiple `DROP COLUMN` query from [MySQL] to [ClickHouse]. #### Rename ##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.Columns.Rename version: 1.0 -[Altinity Sink Connector] SHALL support `RENAME COLUMN col1 to col2` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `RENAME COLUMN col1 to col2` query from [MySQL] to [ClickHouse]. ##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.Columns.Rename.Multiple version: 1.0 -[Altinity Sink Connector] SHALL support multiple `RENAME COLUMN col1 to col2` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support multiple `RENAME COLUMN col1 to col2` query from [MySQL] to [ClickHouse]. ### Add Constraint #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.AddConstraint version: 1.0 -[Altinity Sink Connector] SHALL support `ADD CONSTRAINT` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `ADD CONSTRAINT` query from [MySQL] to [ClickHouse]. ### Drop Constraint #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Alter.DropConstraint version: 1.0 -[Altinity Sink Connector] SHALL support `DROP CONSTRAINT` query from [MySQL] to [CLickHouse]. +[Altinity Sink Connector] SHALL support `DROP CONSTRAINT` query from [MySQL] to [ClickHouse]. ## Primary Key @@ -3343,7 +3668,7 @@ #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.PrimaryKey.No version: 1.0 -[Altinity Sink Connector] query SHALL support [MySQL] data replication to [CLickHouse] on queries to tables +[Altinity Sink Connector] query SHALL support [MySQL] data replication to [ClickHouse] on queries to tables with no `PRIMARY KEY`. ### Simple Primary Key @@ -3351,7 +3676,7 @@ #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.PrimaryKey.Simple version: 1.0 -[Altinity Sink Connector] query SHALL support [MySQL] data replication to [CLickHouse] on queries with the same order +[Altinity Sink Connector] query SHALL support [MySQL] data replication to [ClickHouse] on queries with the same order as simple `PRIMARY KEY` does. ### Composite Primary Key @@ -3359,7 +3684,7 @@ #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.PrimaryKey.Composite version: 1.0 -[Altinity Sink Connector] query SHALL support [MySQL] data replication to [CLickHouse] on queries with the same order +[Altinity Sink Connector] query SHALL support [MySQL] data replication to [ClickHouse] on queries with the same order as composite `PRIMARY KEY` does. ## Multiple Upstream Servers @@ -3367,21 +3692,21 @@ ### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleUpstreamServers version: 1.0 -[Altinity Sink Connector] SHALL support [MySQL] replication to [CLickHouse] from multiple [MySQL] upstream servers. +[Altinity Sink Connector] SHALL support [MySQL] replication to [ClickHouse] from multiple [MySQL] upstream servers. ## Multiple Downstream Servers ### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.MultipleDownstreamServers version: 1.0 -[Altinity Sink Connector] SHALL support [MySQL] replication to [CLickHouse] when using multiple downstream [ClickHouse] servers. +[Altinity Sink Connector] SHALL support [MySQL] replication to [ClickHouse] when using multiple downstream [ClickHouse] servers. ## Archival Mode ### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ArchivalMode version: 1.0 -[Altinity Sink Connector] SHALL support [MySQL] replication to [CLickHouse] with archival mode that +[Altinity Sink Connector] SHALL support [MySQL] replication to [ClickHouse] with archival mode that SHALL ignore deletes for some or all tables in [ClickHouse]. ## Bootstrapping Mode @@ -3389,7 +3714,7 @@ ### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.BootstrappingMode version: 1.0 -[Altinity Sink Connector] SHALL support [MySQL] replication to [CLickHouse] with +[Altinity Sink Connector] SHALL support [MySQL] replication to [ClickHouse] with bootstrapping mode for the initial replication of very large tables that bypasses event stream by using [MySQL] dump files. @@ -3398,7 +3723,7 @@ ### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.BinlogPosition version: 1.0 -[Altinity Sink Connector] SHALL support ability to start replication to [CLickHouse] +[Altinity Sink Connector] SHALL support ability to start replication to [ClickHouse] from specific [MySQL] binlog position. ## Column Mapping And Transformation Rules @@ -3406,7 +3731,7 @@ ### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ColumnMappingAndTransformationRules version: 1.0 -[Altinity Sink Connector] SHALL support [MySQL] replication to [CLickHouse] with support for +[Altinity Sink Connector] SHALL support [MySQL] replication to [ClickHouse] with support for defining column mapping and transformations rules. ## Columns Inconsistency @@ -3414,29 +3739,29 @@ ### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.ColumnsInconsistency version: 1.0 -[Altinity Sink Connector] SHALL support [MySQL] replication to [CLickHouse] replica table when it has fewer columns. -[MySQL] replication to [CLickHouse] is not available in all other cases of columns inconsistency . +[Altinity Sink Connector] SHALL support [MySQL] replication to [ClickHouse] replica table when it has fewer columns. +[MySQL] replication to [ClickHouse] is not available in all other cases of columns inconsistency . ## Latency ### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Latency version: 1.0 -[Altinity Sink Connector] SHALL support [MySQL] replication to [CLickHouse] with latency as close as possible to real-time. +[Altinity Sink Connector] SHALL support [MySQL] replication to [ClickHouse] with latency as close as possible to real-time. ## Performance ### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Performance version: 1.0 -[Altinity Sink Connector] SHALL support [MySQL] replication to [CLickHouse] more than 100,000 rows/sec. +[Altinity Sink Connector] SHALL support [MySQL] replication to [ClickHouse] more than 100,000 rows/sec. ### Large Daily Data Volumes #### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Performance.LargeDailyDataVolumes version: 1.0 -[Altinity Sink Connector] SHALL support [MySQL] replication to [CLickHouse] with large daily data volumes of at least 20-30TB per day. +[Altinity Sink Connector] SHALL support [MySQL] replication to [ClickHouse] with large daily data volumes of at least 20-30TB per day. ## Settings @@ -3501,7 +3826,83 @@ [Altinity Sink Connector] SHALL retry replication if the ClickHouse instance is stopped/killed during the active replication from source to destination tables. [Altinity Sink Connector] SHALL continue to retry to replicate data into a source table until the ClickHouse instance is not available again. -## Prometheus +## System Actions + +### Handling Network Interruptions + +#### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.SystemActions.Network +version: 1.0 + +[Altinity Sink Connector] SHALL be able to recover replication after network related interruptions happen, so that the data on the destination table is not lost. + + +| List of possible network related interruptions | +|---------------------------------------------------------------| +| Internal network interruptions in source database cluster | +| Network interruptions from source database to sink connector | +| Network interruptions from sink connector to clickhouse | +| Internal network interruptions in clickhouse database cluster | + + +### Handling Process Interruptions + +#### Behaviour When Different Processes Were Killed + +##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.SystemActions.Process.Die +version: 1.0 + +[Altinity Sink Connector] SHALL output an error and keep the existing data when one of the following or all of the +scenarios related to processes being killed happen: + +| Scenarios Related to Processes Being Killed | +|-------------------------------------------------------| +| Internal processes die in source database cluster | +| Sink connector dies | +| Internal processes die in clickhouse database cluster | + +#### Behaviour When Different Processes Were Restarted + +##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.SystemActions.Process.Restarted +version: 1.0 + +[Altinity Sink Connector] SHALL continue replication without losing any data after one of the following or +all of the scenarios related to processes being restarted happen: + +| Scenarios Related to Processes Being Restarted | +|-------------------------------------------------------------| +| Restart of some or all nodes in source database cluster | +| Restart of sink connector | +| Restart of some or all nodes in clickhouse database cluster | + +### Behaviour When There Are Issues With Disk + +#### Disk Is out of Space + +##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.SystemActions.Disk.OutOfSpace +version: 1.0 + +[Altinity Sink Connector] SHALL output an error when: + +- Out of disk space on some node in source database cluster +- Out of disk space where sink connector is running +- Out of disk space on some node in clickhouse database cluster + +The error SHALL be shown so that the data on the source and destination tables is not lost due to the disk related issues. + +#### Disk Is Corrupted + +##### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.SystemActions.Disk.Corrupted +version: 1.0 + +[Altinity Sink Connector] SHALL output an error when: + +- Corruption on a disk used by some node in source database cluster +- Corruption on a disk where sink connector is running +- Corruption on a disk used by some node in clickhouse database cluster + +The error SHALL be shown so that the data on the source and destination tables is not lost due to the disk related issues. + +## Prometheus ### RQ.SRS-030.ClickHouse.MySQLToClickHouseReplication.Prometheus version: 1.0