-
Notifications
You must be signed in to change notification settings - Fork 55
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #103 from Altinity/sakila_db_fixes
Integrate sakila employees database
- Loading branch information
Showing
10 changed files
with
230 additions
and
161 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,4 +1,8 @@ | ||
| DATABASE=sbtest | ||
| ./debezium-delete.sh && ./debezium-connector-setup-database.sh $DATABASE && ./sink-delete.sh && ./sink-connector-setup-database.sh $DATABASE | ||
| ./debezium-delete.sh && ./debezium-connector-setup-database.sh $DATABASE && ./sink-delete.sh && ./sink-connector-setup-database.sh $DATABASE | ||
|
|
||
| docker exec -it clickhouse clickhouse-client -uroot --password root -mn --query "drop database if exists $DATABASE;create database $DATABASE;" | ||
|
|
||
|
|
||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,131 +1,131 @@ | ||
| use test; | ||
|
|
||
| CREATE TABLE employees | ||
| ( | ||
| `emp_no` Int32, | ||
| `birth_date` Date32, | ||
| `first_name` String, | ||
| `last_name` String, | ||
| `gender` String, | ||
| `hire_date` Date32, | ||
| `_offset` Nullable(UInt64), | ||
| `_key` Nullable(String), | ||
| `_topic` Nullable(String), | ||
| `_partition` Nullable(UInt64), | ||
| `_timestamp` Nullable(DateTime), | ||
| `_timestamp_ms` Nullable(DateTime64(3)), | ||
| `_ts_ms` Nullable(DateTime64(3)), | ||
| `_server_id` Nullable(Int32), | ||
| `_gtid` Nullable(Int32), | ||
| `_binlog_file` Nullable(String), | ||
| `_binlog_pos` Nullable(Int32), | ||
| `_binlog_row` Nullable(Int32), | ||
| `_server_thread` Nullable(Int32), | ||
| `_version` UInt64 | ||
| ) | ||
| ENGINE = ReplacingMergeTree(_version) | ||
| PRIMARY KEY emp_no | ||
| ORDER BY emp_no; | ||
|
|
||
| alter table employees | ||
| add column `salary` Nullable(UInt64), | ||
| add column num_years Nullable(UInt8), | ||
| add column bonus Nullable(UInt32), | ||
| add column small_value Nullable(UInt16), | ||
| add column int_value Nullable(UInt32), | ||
|
|
||
| add column discount Nullable(Int64), | ||
| add column num_years_signed Nullable(Int8), | ||
| add column bonus_signed Nullable(Int32), | ||
| add column small_value_signed Nullable(Int16), | ||
| add column int_value_signed Nullable(Int32); | ||
|
|
||
| alter table employees add column last_modified_date_time Nullable(String); | ||
| alter table employees add column last_access_time Nullable(String); | ||
|
|
||
| alter table employees add column `married_status` Nullable(String); | ||
| alter table employees add column `perDiemRate` Nullable(Decimal(30, 12)); | ||
|
|
||
| alter table employees add column `hourlyRate` Nullable(Float64); | ||
| alter table employees add column `jobDescription` Nullable(String); | ||
|
|
||
| alter table employees add column `raw_data` Nullable(String); | ||
|
|
||
| -- alter table employees add column `updated_time` Nullable(DateTime); | ||
|
|
||
| alter table employees add column `bytes_date` Nullable(String); | ||
|
|
||
| alter table employees add column `binary_test_column` String; | ||
|
|
||
| alter table employees add column `blob_med` String; | ||
|
|
||
| alter table employees add column `blob_new` String; | ||
|
|
||
| CREATE TABLE products( | ||
| `productCode` String, | ||
| `productName` String, | ||
| `productLine` String, | ||
| `productScale` String, | ||
| `productVendor` String, | ||
| `productDescription` String, | ||
| `quantityInStock` Int32, | ||
| `buyPrice` Decimal(10,2), | ||
| `MSRP` Decimal(10,2), | ||
| `raw_data` String, | ||
| `sign` Int8 | ||
| ) | ||
| ENGINE = CollapsingMergeTree(sign) | ||
| PRIMARY KEY productCode | ||
| order by productCode; | ||
|
|
||
| CREATE TABLE t1( | ||
| `intcol1` UInt64, | ||
| `charcol1` String | ||
| ) | ||
| ENGINE = MergeTree | ||
| PRIMARY KEY intcol1; | ||
|
|
||
| -- Sysbench tables --- | ||
| --CREATE TABLE sbtest1( | ||
| --`id` UInt64, | ||
| --`k` UInt64, | ||
| --`c` String, | ||
| --`pad` String, | ||
| --`sign` Int8, | ||
| -- `ver` UInt64 | ||
| -- | ||
| --CREATE TABLE employees | ||
| --( | ||
| -- `emp_no` Int32, | ||
| -- `birth_date` Date32, | ||
| -- `first_name` String, | ||
| -- `last_name` String, | ||
| -- `gender` String, | ||
| -- `hire_date` Date32, | ||
| -- `_offset` Nullable(UInt64), | ||
| -- `_key` Nullable(String), | ||
| -- `_topic` Nullable(String), | ||
| -- `_partition` Nullable(UInt64), | ||
| -- `_timestamp` Nullable(DateTime), | ||
| -- `_timestamp_ms` Nullable(DateTime64(3)), | ||
| -- `_ts_ms` Nullable(DateTime64(3)), | ||
| -- `_server_id` Nullable(Int32), | ||
| -- `_gtid` Nullable(Int32), | ||
| -- `_binlog_file` Nullable(String), | ||
| -- `_binlog_pos` Nullable(Int32), | ||
| -- `_binlog_row` Nullable(Int32), | ||
| -- `_server_thread` Nullable(Int32), | ||
| -- `_version` UInt64 | ||
| --) | ||
| --ENGINE = ReplacingMergeTree(ver) | ||
| ---- ENGINE = CollapsingMergeTree(sign) | ||
| --ORDER by(id, k); | ||
|
|
||
| -- Metadata tables -- | ||
| --CREATE TABLE topic_offset_metadata( | ||
| --`_topic` String, | ||
| --`_partition` UInt64, | ||
| --`_offset` SimpleAggregateFunction(max, UInt64) | ||
| --ENGINE = ReplacingMergeTree(_version) | ||
| --PRIMARY KEY emp_no | ||
| --ORDER BY emp_no; | ||
| -- | ||
| --alter table employees | ||
| --add column `salary` Nullable(UInt64), | ||
| --add column num_years Nullable(UInt8), | ||
| --add column bonus Nullable(UInt32), | ||
| --add column small_value Nullable(UInt16), | ||
| --add column int_value Nullable(UInt32), | ||
| -- | ||
| --add column discount Nullable(Int64), | ||
| --add column num_years_signed Nullable(Int8), | ||
| --add column bonus_signed Nullable(Int32), | ||
| --add column small_value_signed Nullable(Int16), | ||
| --add column int_value_signed Nullable(Int32); | ||
| -- | ||
| --alter table employees add column last_modified_date_time Nullable(String); | ||
| --alter table employees add column last_access_time Nullable(String); | ||
| -- | ||
| --alter table employees add column `married_status` Nullable(String); | ||
| --alter table employees add column `perDiemRate` Nullable(Decimal(30, 12)); | ||
| -- | ||
| --alter table employees add column `hourlyRate` Nullable(Float64); | ||
| --alter table employees add column `jobDescription` Nullable(String); | ||
| -- | ||
| --alter table employees add column `raw_data` Nullable(String); | ||
| -- | ||
| ---- alter table employees add column `updated_time` Nullable(DateTime); | ||
| -- | ||
| --alter table employees add column `bytes_date` Nullable(String); | ||
| -- | ||
| --alter table employees add column `binary_test_column` String; | ||
| -- | ||
| --alter table employees add column `blob_med` String; | ||
| -- | ||
| --alter table employees add column `blob_new` String; | ||
| -- | ||
| --CREATE TABLE products( | ||
| -- `productCode` String, | ||
| -- `productName` String, | ||
| -- `productLine` String, | ||
| -- `productScale` String, | ||
| -- `productVendor` String, | ||
| -- `productDescription` String, | ||
| -- `quantityInStock` Int32, | ||
| -- `buyPrice` Decimal(10,2), | ||
| -- `MSRP` Decimal(10,2), | ||
| -- `raw_data` String, | ||
| -- `sign` Int8 | ||
| --) | ||
| --ENGINE = AggregatingMergeTree | ||
| --ORDER BY (_topic, _partition); | ||
|
|
||
| SET allow_experimental_object_type = 1; | ||
|
|
||
| -- Postgres tables -- | ||
| CREATE TABLE Employee( | ||
| `EmployeeId` UInt64, | ||
| `LastName` String, | ||
| `FirstName` String, | ||
| `Title` String, | ||
| `ReportsTo` UInt64, | ||
| `BirthDate` DateTime64, | ||
| `HireDate` DateTime64, | ||
| `Address` String, | ||
| `City` String, | ||
| `State` String, | ||
| `Country` String, | ||
| `PostalCode` String, | ||
| `Phone` String, | ||
| `Fax` String, | ||
| `Email` String, | ||
| `json_data` String) | ||
| ENGINE = MergeTree | ||
| PRIMARY KEY EmployeeId; | ||
| --ENGINE = CollapsingMergeTree(sign) | ||
| --PRIMARY KEY productCode | ||
| --order by productCode; | ||
| -- | ||
| --CREATE TABLE t1( | ||
| --`intcol1` UInt64, | ||
| --`charcol1` String | ||
| --) | ||
| --ENGINE = MergeTree | ||
| --PRIMARY KEY intcol1; | ||
| -- | ||
| ---- Sysbench tables --- | ||
| ----CREATE TABLE sbtest1( | ||
| ----`id` UInt64, | ||
| ----`k` UInt64, | ||
| ----`c` String, | ||
| ----`pad` String, | ||
| ----`sign` Int8, | ||
| ---- `ver` UInt64 | ||
| ----) | ||
| ----ENGINE = ReplacingMergeTree(ver) | ||
| ------ ENGINE = CollapsingMergeTree(sign) | ||
| ----ORDER by(id, k); | ||
| -- | ||
| ---- Metadata tables -- | ||
| ----CREATE TABLE topic_offset_metadata( | ||
| ----`_topic` String, | ||
| ----`_partition` UInt64, | ||
| ----`_offset` SimpleAggregateFunction(max, UInt64) | ||
| ----) | ||
| ----ENGINE = AggregatingMergeTree | ||
| ----ORDER BY (_topic, _partition); | ||
| -- | ||
| --SET allow_experimental_object_type = 1; | ||
| -- | ||
| ---- Postgres tables -- | ||
| --CREATE TABLE Employee( | ||
| --`EmployeeId` UInt64, | ||
| --`LastName` String, | ||
| --`FirstName` String, | ||
| --`Title` String, | ||
| --`ReportsTo` UInt64, | ||
| --`BirthDate` DateTime64, | ||
| -- `HireDate` DateTime64, | ||
| -- `Address` String, | ||
| -- `City` String, | ||
| -- `State` String, | ||
| -- `Country` String, | ||
| -- `PostalCode` String, | ||
| -- `Phone` String, | ||
| -- `Fax` String, | ||
| -- `Email` String, | ||
| -- `json_data` String) | ||
| -- ENGINE = MergeTree | ||
| -- PRIMARY KEY EmployeeId; |
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,15 @@ | ||
| cd ../python | ||
| DATABASE=employees | ||
|
|
||
| python db_compare/mysql_table_checksum.py --mysql_host localhost --mysql_user root --mysql_password root --mysql_database $DATABASE --tables_regex "^*" --debug_output | ||
| python db_compare/clickhouse_table_checksum.py --clickhouse_host localhost --clickhouse_user root --clickhouse_password root --clickhouse_database $DATABASE --tables_regex "^*" --debug_output | ||
|
|
||
| for tableName in departments dept_emp dept_manager employees salaries titles | ||
| do | ||
| echo " DIFF TABLE ****${tableName} **** " | ||
| diff out.${tableName}.mysql.txt out.${tableName}.ch.txt | ||
| done | ||
| #diff ../python/ | ||
|
|
||
| #rm -fr *.ch.txt | ||
| #rm -fr *.mysql.txt |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,11 +1,15 @@ | ||
| cd ../python | ||
| #python db_compare/mysql_table_checksum.py --mysql_host localhost --mysql_user root --mysql_password root --mysql_database sakila --tables_regex "^*" --debug_output | ||
| #python db_compare/clickhouse_table_checksum.py --clickhouse_host localhost --clickhouse_user root --clickhouse_password root --clickhouse_database sakila --tables_regex "^*" --debug_output | ||
|
|
||
| diff out.customer.mysql.txt out.customer.ch.txt | ||
| diff out.payment.mysql.txt out.payment.ch.txt | ||
| diff out.rental.mysql.txt out.rental.ch.txt | ||
| python db_compare/mysql_table_checksum.py --mysql_host localhost --mysql_user root --mysql_password root --mysql_database sakila --tables_regex "^*" --debug_output | ||
| python db_compare/clickhouse_table_checksum.py --clickhouse_host localhost --clickhouse_user root --clickhouse_password root --clickhouse_database sakila --tables_regex "^*" --debug_output | ||
|
|
||
|
|
||
| for tableName in actor address category city country customer film film_actor film_category film_text inventory language payment rental staff store | ||
| do | ||
| echo " DIFF TABLE ****${tableName} **** " | ||
| diff out.${tableName}.mysql.txt out.${tableName}.ch.txt | ||
| done | ||
| #diff ../python/ | ||
|
|
||
| rm -fr *.ch.txt | ||
| rm -fr *.mysql.txt | ||
| #rm -fr *.ch.txt | ||
| #rm -fr *.mysql.txt |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,19 @@ | ||
| #!/bin/bash | ||
|
|
||
| set +x | ||
|
|
||
| rm -fr MySQL.tsv | ||
| rm -fr CH.tsv | ||
|
|
||
| if [[ $1 == "bulk_insert" || $1 == "oltp_insert" ]]; then | ||
| docker exec -it clickhouse clickhouse-client --multiquery -uroot --password root --query "use sbtest; select id ,k from sbtest.sbtest1 where _sign !=-1 order by id format TSV" | grep -v "<jemalloc>" >CH.tsv | ||
| else | ||
| docker exec -it clickhouse clickhouse-client --multiquery -uroot --password root --query "use sbtest; select id ,k, c, pad from sbtest.sbtest1 final where _sign !=-1 order by id format TSV" | grep -v "<jemalloc>" >CH.tsv | ||
| fi | ||
| docker exec -it mysql-master mysql -uroot -proot -B -N -e "select * from sbtest.sbtest1 order by id" | grep -v "Using a password on the command line interface" >MySQL.tsv | ||
|
|
||
| diff --strip-trailing-cr MySQL.tsv CH.tsv | ||
|
|
||
| # | ||
| #rm -fr MySQL.tsv | ||
| #rm -fr CH.tsv |
Oops, something went wrong.