Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor TestFlows tests related to Lightweight #543

Merged
merged 63 commits into from
Apr 18, 2024
Merged
Changes from 1 commit
Commits
Show all changes
63 commits
Select commit Hold shift + click to select a range
a121b3b
remove xfails for a test run
Mar 18, 2024
77260f1
fix grammar errors
Selfeer Mar 24, 2024
bab8d4c
Merge branch 'develop' into investigate_kafka_tests
Selfeer Mar 24, 2024
431523b
fix grammar errors remove comments
Selfeer Mar 24, 2024
ae45d19
Restructure test environment to decouple sink-connector process
Selfeer Mar 24, 2024
efa9ab2
add newline
Selfeer Mar 24, 2024
79b91be
fix some text issues
Mar 28, 2024
f1fd11a
fix some text issues
Mar 31, 2024
ce15240
Merge branch 'develop' into investigate_kafka_tests
Apr 1, 2024
ef48a9f
comment out entrypoint
Apr 1, 2024
9423e5e
add step to start sink connector
Apr 1, 2024
3a63367
add step to start sink connector
Apr 1, 2024
9526949
change condition order
Apr 1, 2024
fa21b6c
try to add steps to start and stop sink connector
Apr 3, 2024
8c15759
refactor
Apr 5, 2024
3cbebd1
refactor
Apr 5, 2024
ed810fe
refactor
Apr 5, 2024
1e36c08
refactor
Apr 5, 2024
867758d
refactor
Apr 5, 2024
9749da3
update requirements
Apr 5, 2024
406e52e
refactor
Apr 5, 2024
4ecd256
remove broken tests
Apr 5, 2024
338da97
add cli tests
Apr 5, 2024
f357bef
remove redundant files and tests that might break replication
Apr 8, 2024
c0f9fc2
remove redundant files and tests that might break replication
Apr 8, 2024
2690f1a
update
Apr 9, 2024
8f265a0
update
Apr 10, 2024
10fff5e
Merge branch 'refs/heads/develop' into refactor_testflows_tests
Apr 15, 2024
055d0ea
add dockerfile to install basic linux tools
Apr 15, 2024
dfccea0
update requirements - initial
Apr 15, 2024
6761c32
update requirements
Apr 15, 2024
b77b268
add sink connector logs
Apr 15, 2024
fc03dfe
remove redundant volume
Apr 15, 2024
86892a8
remove redundant volume
Apr 15, 2024
004cc41
update workflows to be able to specify clickhouse version
Apr 15, 2024
45a457d
update workflows to be able to specify clickhouse version
Apr 15, 2024
143d0b0
reduce sleep time
Apr 15, 2024
3716b11
change logs file
Apr 15, 2024
b50ae19
update requirements
Apr 15, 2024
9d73a4d
try to add parallelization to tests
Apr 15, 2024
35adeed
try to add parallelization to tests
Apr 15, 2024
376a2c9
try to increese paralleization to 4
Apr 15, 2024
83e4aca
try to add possibility to update configuration from features
Apr 16, 2024
59041ec
add config
Apr 16, 2024
df68fcb
add step to change the sink configuration
Apr 16, 2024
4fcec26
remove Dockerfile
Apr 16, 2024
d0a1f58
remove Dockerfile
Apr 16, 2024
cdb78c3
update requirements
Apr 16, 2024
5bbba77
update configuration update
Apr 16, 2024
d30d365
update xfails
Apr 16, 2024
5933c95
refactor configuration generation and logs for sink
Apr 16, 2024
b34ec5c
revert some deleted files, part 1
Apr 17, 2024
d13b140
revert some deleted files, part 1
Apr 17, 2024
462926b
revert some deleted files, part 1
Apr 17, 2024
2a45155
revert some deleted files
Apr 17, 2024
9e0293a
revert some deleted files
Apr 17, 2024
f532e84
revert some deleted files
Apr 17, 2024
2a8c7e3
add docstring to SinkConnector class
Apr 18, 2024
7a914b5
Move start sink connector to happen after the database in ClickHouse …
Apr 18, 2024
965261c
update CI/CD file to be able to run the Lightweight tests on PRs with…
Apr 18, 2024
63ba9b5
upload configuration files used for sink connector to artefacts
Apr 18, 2024
a8c6bc7
Merge remote-tracking branch 'origin/refactor_testflows_tests' into r…
Apr 18, 2024
b351ed6
Add xfails
Selfeer Apr 18, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
add step to start sink connector
  • Loading branch information
Selfeer committed Apr 1, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature.
commit 9423e5e8194c6c52105e29317f0fb9640cdf6b67
41 changes: 41 additions & 0 deletions sink-connector-lightweight/tests/integration/helpers/cluster.py
Original file line number Diff line number Diff line change
@@ -527,6 +527,8 @@ def node(self, name):
return ClickHouseNode(self, name, data_base_type="clickhouse")
if name.startswith("mysql"):
return MySQLNode(self, name, data_base_type="mysql")
if name.startswith("clickhouse-sink"):
return SinkConnector(self, name)
return Node(self, name)

def down(self, timeout=300):
@@ -925,6 +927,45 @@ def query(
)


class SinkConnector(DatabaseNode):
"""ClickHouse Sink Connector node."""

sink_connector_cli = "./sink-connector-client "

def start_sink_connector(self, timeout=300):
with Given("I start ClickHouse Sink Connector"):
self.command(
node=self.name,
command=f"java -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005 -jar /app.jar "
f"/config.yml com.altinity.clickhouse.debezium.embedded.ClickHouseDebeziumEmbeddedApplication",
timeout=timeout,
)

def stop_replication(self, timeout=300):
with Given("I stop ClickHouse Sink Connector replication"):
self.command(
node=self.name,
command=f"{self.sink_connector_cli} stop_replica",
timeout=timeout,
)

def start_replication(self):
with Given("I start ClickHouse Sink Connector replication"):
self.command(
node=self.name,
command=f"{self.sink_connector_cli} start_replica",
timeout=300,
)

def change_replica_source(self):
with Given("I change ClickHouse Sink Connector replica source"):
self.command(
node=self.name,
command=f"{self.sink_connector_cli} change_replica_source",
timeout=300,
)


class ClickHouseNode(DatabaseNode):
"""Node with ClickHouse server."""

Original file line number Diff line number Diff line change
@@ -6,7 +6,6 @@

from testflows.core import *


append_path(sys.path, "..")

from integration.helpers.argparser import argparser
@@ -15,7 +14,6 @@
from integration.requirements.requirements import *
from integration.tests.steps.steps_global import *


ffails = {
"schema changes/table recreation with different datatypes": (
Skip,
@@ -122,7 +120,6 @@
),
}


xflags = {}


@@ -182,6 +179,11 @@ def regression(
caller_dir=os.path.join(current_dir()),
)

with And("I start ClickHouse sink connector"):
sink_node = self.context.cluster.node("clickhouse-sink-connector-lt")

sink_node.start_sink_connector()

self.context.cluster = cluster

self.context.env = env
Loading