Wiring for rs3 antithesis runs

.github/workflows/github-e2e-manual.yaml

@@ -10,6 +10,7 @@ on:
         options:
           - scylla
           - mongo
+          - rs3
       test-type:
         description: 'Which test variant to run'
         type: choice

.github/workflows/github-e2e.yaml

@@ -102,7 +102,7 @@ jobs:
           password: ${{ secrets.ANTITHESIS_PASSWORD }}
           github_token: ${{ secrets.ANTITHESIS_GH_PAT }}
           config_image: us-central1-docker.pkg.dev/molten-verve-216720/responsive-repository/e2e-test-config-${{ inputs.test-type }}:${{ steps.kafka_client_version.outputs.KAFKA_CLIENT_VERSION }}
-          images: us-central1-docker.pkg.dev/molten-verve-216720/responsive-repository/e2e-test:${{ steps.kafka_client_version.outputs.KAFKA_CLIENT_VERSION }};docker.io/mongo:5.0
+          images: us-central1-docker.pkg.dev/molten-verve-216720/responsive-repository/e2e-test:${{ steps.kafka_client_version.outputs.KAFKA_CLIENT_VERSION }};docker.io/mongo:5.0;public.ecr.aws/j8q9y0n6/responsiveinc/rs3-antithesis:0.1.0-main.30.0
           description: "responsive antithesis test run"
           email_recipients: "antithesis-responsive-aaaamurlsqy6e3hxnx6ksnec5y@antithesisgroup.slack.com"
           additional_parameters: |-

kafka-client-examples/e2e-test/docker/antithesis/async/Dockerfile

@@ -1,15 +1,17 @@
-from scratch
+FROM scratch
 
 COPY base/config/docker-compose-base.yml /docker-compose-base.yml
 COPY base/config/docker-compose-scylla.yml /docker-compose-scylla.yml
 COPY base/config/docker-compose-mongo.yml /docker-compose-mongo.yml
+COPY base/config/docker-compose-rs3.yml /docker-compose-rs3.yml
 COPY base/config/volumes/scylla.yaml volumes/scylla.yaml
 ADD base/config/volumes/zookeeper /volumes/zookeeper
 ADD base/config/volumes/scylla /volumes/scylla
 ADD base/config/volumes/scylla2 /volumes/scylla2
 ADD base/config/volumes/scylla3 /volumes/scylla3
 ADD base/config/volumes/kafka /volumes/kafka
 
+COPY async/config/rs3.toml volumes/rs3.toml
 COPY async/config/docker-compose.yml /docker-compose.yml
 COPY async/config/volumes/app.properties /volumes/app.properties
 COPY async/config/volumes/log4j2.properties /volumes/log4j2.properties

kafka-client-examples/e2e-test/docker/antithesis/async/config/rs3.toml

@@ -0,0 +1,18 @@
+[server]
+listen_port = 50051
+
+[store_provider]
+provider = "Static"
+
+[store_provider.static_store.slatedb]
+wal_enabled = false
+
+[store_provider.static_store.slatedb.block_cache]
+mode = "Default"
+
+[[store_provider.static_store.stores]]
+id = "b1a45157-e2f0-4698-be0e-5bf3a9b8e9d1"
+pss = [{ id = 0 }, { id = 1 }, { id = 2 }, { id = 3 }, { id = 4 }, { id = 5 }, { id = 6 }, { id = 7 }]
+
+[cloud_provider]
+provider = "InMemory"

kafka-client-examples/e2e-test/docker/antithesis/async/config/volumes/app-rs3.properties

@@ -0,0 +1,21 @@
+#
+# Copyright 2024 Responsive Computing, Inc.
+#
+# This source code is licensed under the Responsive Business Source License Agreement v1.0
+# available at:
+#
+# https://www.responsive.dev/legal/responsive-bsl-10
+#
+# This software requires a valid Commercial License Key for production use. Trial and commercial
+# licenses can be obtained at https://www.responsive.dev
+#
+
+bootstrap.servers=broker:9092
+responsive.org=responsive
+responsive.env=test
+
+responsive.storage.backend.type=RS3
+responsive.rs3.hostname=rs3
+responsive.rs3.port=50051
+responsive.rs3.logical.store.mapping=e2e:b1a45157-e2f0-4698-be0e-5bf3a9b8e9d1
+responsive.rs3.tls.enabled=false

kafka-client-examples/e2e-test/docker/antithesis/base/config/docker-compose-rs3.yml

@@ -0,0 +1,18 @@
+include:
+  - docker-compose-base.yml
+services:
+  rs3:
+    # Remember to update the image in github-e2e.yaml as well
+    image: public.ecr.aws/j8q9y0n6/responsiveinc/rs3-antithesis:0.1.0-main.30.0
+    platform: linux/amd64
+    container_name: rs3
+    hostname: rs3
+    ports:
+      - 50051:50051
+    environment:
+      CONFIG_FILE: /etc/rs3/rs3.toml
+    volumes:
+      - ./volumes/rs3.toml:/etc/rs3/rs3.toml
+    networks:
+      backend:
+        ipv4_address: 10.0.0.60

kafka-client-examples/e2e-test/docker/antithesis/regression/Dockerfile

@@ -1,8 +1,9 @@
-from scratch
+FROM scratch
 
 COPY base/config/docker-compose-base.yml /docker-compose-base.yml
 COPY base/config/docker-compose-scylla.yml /docker-compose-scylla.yml
 COPY base/config/docker-compose-mongo.yml /docker-compose-mongo.yml
+COPY base/config/docker-compose-rs3.yml /docker-compose-rs3.yml
 COPY base/config/volumes/scylla.yaml volumes/scylla.yaml
 ADD base/config/volumes/zookeeper /volumes/zookeeper
 ADD base/config/volumes/scylla /volumes/scylla
@@ -13,6 +14,7 @@ ADD base/config/volumes/mongo2 /volumes/mongo2
 ADD base/config/volumes/mongo3 /volumes/mongo3
 ADD base/config/volumes/kafka /volumes/kafka
 
+COPY regression/config/rs3.toml volumes/rs3.toml
 COPY regression/config/docker-compose.yml /docker-compose.yml
 COPY regression/config/volumes/app.properties /volumes/app.properties
 COPY regression/config/volumes/log4j2.properties /volumes/log4j2.properties

kafka-client-examples/e2e-test/docker/antithesis/regression/config/rs3.toml

@@ -0,0 +1,22 @@
+[server]
+listen_port = 50051
+
+[store_provider]
+provider = "Static"
+
+[store_provider.static_store.slatedb]
+wal_enabled = false
+
+[store_provider.static_store.slatedb.block_cache]
+mode = "Default"
+
+[[store_provider.static_store.stores]]
+id = "b1a45157-e2f0-4698-be0e-5bf3a9b8e9d1"
+pss = [{ id = 0 }, { id = 1 }, { id = 2 }, { id = 3 }, { id = 4 }, { id = 5 }, { id = 6 }, { id = 7 }]
+
+[[store_provider.static_store.stores]]
+id = "013a57c3-16b3-44be-8de4-0d1083671cc6"
+pss = [{ id = 0 }, { id = 1 }, { id = 2 }, { id = 3 }, { id = 4 }, { id = 5 }, { id = 6 }, { id = 7 }]
+
+[cloud_provider]
+provider = "InMemory"

kafka-client-examples/e2e-test/docker/antithesis/regression/config/volumes/app-rs3.properties

@@ -0,0 +1,21 @@
+#
+# Copyright 2024 Responsive Computing, Inc.
+#
+# This source code is licensed under the Responsive Business Source License Agreement v1.0
+# available at:
+#
+# https://www.responsive.dev/legal/responsive-bsl-10
+#
+# This software requires a valid Commercial License Key for production use. Trial and commercial
+# licenses can be obtained at https://www.responsive.dev
+#
+
+bootstrap.servers=broker:9092
+responsive.org=responsive
+responsive.env=test
+
+responsive.storage.backend.type=RS3
+responsive.rs3.hostname=rs3
+responsive.rs3.port=50051
+responsive.rs3.logical.store.mapping=customers_state_store_0000000001:b1a45157-e2f0-4698-be0e-5bf3a9b8e9d1,KSTREAM-REDUCE-STATE-STORE-0000000006:013a57c3-16b3-44be-8de4-0d1083671cc6
+responsive.rs3.tls.enabled=false

kafka-client-examples/e2e-test/src/main/java/dev/responsive/examples/e2etest/E2ETestApplication.java

@@ -34,6 +34,7 @@
 import org.apache.kafka.streams.KafkaStreams;
 import org.apache.kafka.streams.StreamsBuilder;
 import org.apache.kafka.streams.StreamsConfig;
+import org.apache.kafka.streams.Topology;
 import org.apache.kafka.streams.kstream.Consumed;
 import org.apache.kafka.streams.kstream.KStream;
 import org.apache.kafka.streams.kstream.Named;
@@ -140,10 +141,11 @@ private KafkaStreams buildTopology(final Map<String, Object> properties) {
     builderProperties.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG,
         StreamsConfig.EXACTLY_ONCE_V2);
     builderProperties.put(ResponsiveConfig.ASYNC_THREAD_POOL_SIZE_CONFIG, 2);
-    final var streams = new ResponsiveKafkaStreams(
-        builder.build(builderProperties),
-        builderProperties
-    );
+
+    Topology topology = builder.build(builderProperties);
+    LOG.info("Starting Kafka Streams with topology \n{}", topology.describe());
+
+    final var streams = new ResponsiveKafkaStreams(topology, builderProperties);
     streams.setUncaughtExceptionHandler(new UncaughtStreamsAntithesisHandler());
     return streams;
   }

kafka-client-examples/e2e-test/src/main/java/dev/responsive/examples/regression/tests/AbstractKSExampleService.java

@@ -82,7 +82,9 @@ protected final void startUp() throws Exception {
     E2ETestUtils.awaitKeyspace(properties);
     LOG.info("Created keyspace...");
 
-    LOG.info("Starting Kafka Streams...");
+    Topology topology = buildTopology();
+    LOG.info("Starting Kafka Streams with topology \n{}", topology.describe());
+
     kafkaStreams = responsive
         ? new ResponsiveKafkaStreams(buildTopology(), properties)
         : new KafkaStreams(buildTopology(), new StreamsConfig(properties));