Programmatically download Kafka Connect HTTP

This way we don't have to have a huge file in our repository.

scripts/cli

@@ -688,6 +688,14 @@ function install_libs() {
     (cd $CLI_HOME/.. && pnpm i --frozen-lockfile)
 }
 
+function download_kafka_connect_http() {
+    say "Downloading Kafka Connect HTTP plugin..."
+    wget -nv https://github.com/lensesio/stream-reactor/releases/download/8.1.28/kafka-connect-http-8.1.28.zip -O $CLI_HOME/kafka-connect-http.zip
+    unzip -q $CLI_HOME/kafka-connect-http.zip -d $CLI_HOME/scaffold/kafka-connect/kafka-connect-http
+    rm $CLI_HOME/kafka-connect-http.zip
+    say "Kafka Connect HTTP plugin downloaded."
+}
+
 function up_scaffold() {
     scaffold_set_up_network "$PROJECT_NAME-bridge" $DOCKER_NETWORK_SUBNET $DOCKER_NETWORK_GATEWAY
 
@@ -696,6 +704,10 @@ function up_scaffold() {
         profile='--profile nginx'
     fi
 
+    if [[ ! -d "$CLI_HOME/scaffold/kafka-connect/kafka-connect-http" ]]; then
+        download_kafka_connect_http
+    fi
+
     $_DC --compatibility -p $PROJECT_NAME -f $CLI_HOME/scaffold.yaml ${profile} up -d --build
 }
 

scripts/scaffold/kafka-connect/Dockerfile

@@ -6,7 +6,7 @@ RUN yum install -y jq findutils unzip
 
 RUN confluent-hub install snowflakeinc/snowflake-kafka-connector:2.5.0 --no-prompt
 RUN confluent-hub install questdb/kafka-questdb-connector:0.12 --no-prompt
-COPY kafka-connect-http-8.1.25/ /usr/share/confluent-hub-components/kafka-connect-http-8.1.25/
+COPY kafka-connect-http/ /usr/share/confluent-hub-components/kafka-connect-http/
 
 
 VOLUME /storage

scripts/scaffold/kafka-connect/README.md

@@ -0,0 +1,5 @@
+The Kafka HTTP Connector we're using comes from here:
+https://github.com/lensesio/stream-reactor/releases
+
+Its documentation is available here:
+https://docs.lenses.io/latest/connectors/kafka-connectors/sinks/http