Use snowplow/snowplow-enrich-nsq (close #386)

snowplow · Jan 24, 2024 · cde897c · cde897c
1 parent b4eec64
commit cde897c
Show file tree

Hide file tree

Showing 8 changed files with 56 additions and 26 deletions.
diff --git a/README.md b/README.md
@@ -88,7 +88,7 @@ Snowplow Mini runs several distinct applications on the same box which are all l
   * Starts server listening on `http://< sp mini public ip>/` which events can be sent to.
   * Sends "good" events to the `RawEvents` NSQ topic
   * Sends "bad" events to the `BadEvents` NSQ topic
-* Stream Enrich:
+* Enrich:
   * Reads events in from the `RawEvents` NSQ topic
   * Sends events which passed the enrichment process to the `EnrichedEvents` NSQ topic
   * Sends events which failed the enrichment process to the `BadEvents` NSQ topic

diff --git a/provisioning/resources/configs/compositions/docker-compose-aws.yml b/provisioning/resources/configs/compositions/docker-compose-aws.yml
@@ -157,14 +157,13 @@ services:
       - "JDK_JAVA_OPTIONS=-XX:MinRAMPercentage=50 -XX:MaxRAMPercentage=75 -Dlog4j2.formatMsgNoLookups=true"
     mem_limit: ${SP_COLLECTOR_MEM_SIZE}
 
-  stream-enrich:
-    image: snowplow/stream-enrich-nsq:3.8.0-distroless
-    container_name: stream-enrich-nsq
+  enrich:
+    image: snowplow/snowplow-enrich-nsq:3.8.2-distroless
+    container_name: enrich
     command: [
-      "--config", "/snowplow/config/snowplow-stream-enrich.hocon",
-      "--resolver", "file:/snowplow/config/iglu-resolver.json",
-      "--enrichments", "file:/snowplow/config/enrichments",
-      "--force-cached-files-download"
+      "--config", "/snowplow/config/snowplow-enrich.hocon",
+      "--iglu-config", "/snowplow/config/iglu-resolver.json",
+      "--enrichments", "/snowplow/config/enrichments"
     ]
     restart: always
     depends_on:
@@ -175,7 +174,7 @@ services:
       driver: awslogs
       options:
         awslogs-group: "${AWS_LOGS_GROUP}"
-        awslogs-stream: "stream-enrich"
+        awslogs-stream: "enrich"
     environment:
       - "JDK_JAVA_OPTIONS=-XX:MinRAMPercentage=50 -XX:MaxRAMPercentage=75 -Dlog4j2.formatMsgNoLookups=true"
     mem_limit: ${SP_ENRICH_MEM_SIZE}

diff --git a/provisioning/resources/configs/compositions/docker-compose-gcp.yml b/provisioning/resources/configs/compositions/docker-compose-gcp.yml
@@ -133,14 +133,13 @@ services:
       - "JDK_JAVA_OPTIONS=-XX:MinRAMPercentage=50 -XX:MaxRAMPercentage=75 -Dlog4j2.formatMsgNoLookups=true"
     mem_limit: ${SP_COLLECTOR_MEM_SIZE}
 
-  stream-enrich:
-    image: snowplow/stream-enrich-nsq:3.8.0-distroless
-    container_name: stream-enrich-nsq
+  enrich:
+    image: snowplow/snowplow-enrich-nsq:3.8.2-distroless
+    container_name: enrich
     command: [
-      "--config", "/snowplow/config/snowplow-stream-enrich.hocon",
-      "--resolver", "file:/snowplow/config/iglu-resolver.json",
-      "--enrichments", "file:/snowplow/config/enrichments",
-      "--force-cached-files-download"
+      "--config", "/snowplow/config/snowplow-enrich.hocon",
+      "--iglu-config", "/snowplow/config/iglu-resolver.json",
+      "--enrichments", "/snowplow/config/enrichments"
     ]
     restart: always
     depends_on:

diff --git a/provisioning/resources/configs/control-plane-api.toml b/provisioning/resources/configs/control-plane-api.toml
@@ -17,7 +17,7 @@ collector = "snowplow-stream-collector.hocon"
 
 [init_scripts]
 stream_collector = "scala-stream-collector"
-stream_enrich = "stream-enrich"
+enrich = "enrich"
 es_loader_good = "elasticsearch-loader-good"
 es_loader_bad = "elasticsearch-loader-bad"
 iglu = "iglu-server"

diff --git a/provisioning/resources/configs/snowplow-enrich.hocon b/provisioning/resources/configs/snowplow-enrich.hocon
@@ -0,0 +1,33 @@
+{
+  "input": {
+    "topic": "RawEvents"
+    "lookupHost": "nsqlookupd"
+    "lookupPort": 4161
+  }
+
+  "output": {
+    "good": {
+      "topic": "EnrichedEvents"
+      "nsqdHost": "nsqd"
+      "nsqdPort": 4150
+    }
+
+    "bad": {
+      "topic": "BadEnrichedEvents"
+      "nsqdHost": "nsqd"
+      "nsqdPort": 4150
+    }
+
+    "pii": {
+      "type": "Nsq"
+      "topic": "PiiEvents"
+      "nsqdHost": "nsqd"
+      "nsqdPort": 4150
+      "backoffPolicy": {
+        "minBackoff": 100 milliseconds
+        "maxBackoff": 10 seconds
+        "maxRetries": 10
+      }
+    }
+  }
+}
diff --git a/provisioning/resources/control-plane/model.go b/provisioning/resources/control-plane/model.go
@@ -24,7 +24,7 @@ type configFileNames struct {
 
 type initScripts struct {
 	Collector     string `toml:"stream_collector"`
-	Enrich        string `toml:"stream_enrich"`
+	Enrich        string `toml:"enrich"`
 	EsLoaderGood  string `toml:"es_loader_good"`
 	EsLoaderBad   string `toml:"es_loader_bad"`
 	Iglu          string

diff --git a/provisioning/resources/ui/js/components/Overview.tsx b/provisioning/resources/ui/js/components/Overview.tsx
@@ -51,7 +51,7 @@ export class Overview extends React.Component<{}, {}> {
         <ul>
         <li><b>Snowplow Mini 0.17.0</b></li>
           <li>Snowplow Stream Collector NSQ 3.0.0-rc9</li>
-          <li>Snowplow Stream Enrich NSQ 3.8.0</li>
+          <li>Snowplow Stream Enrich NSQ 3.8.2</li>
           <li>Snowplow Elasticsearch Loader 2.1.0</li>
           <li>Snowplow Iglu Server 0.10.0</li>
           <li>Postgres 15.1</li>

diff --git a/provisioning/roles/docker/files/docker-compose.yml b/provisioning/roles/docker/files/docker-compose.yml
@@ -149,14 +149,13 @@ services:
       - "JDK_JAVA_OPTIONS=-XX:MinRAMPercentage=50 -XX:MaxRAMPercentage=75 -Dlog4j2.formatMsgNoLookups=true"
     mem_limit: ${SP_COLLECTOR_MEM_SIZE}
 
-  stream-enrich:
-    image: snowplow/stream-enrich-nsq:3.8.0-distroless
-    container_name: stream-enrich-nsq
+  enrich:
+    image: snowplow/snowplow-enrich-nsq:3.8.2-distroless
+    container_name: enrich
     command: [
-      "--config", "/snowplow/config/snowplow-stream-enrich.hocon",
-      "--resolver", "file:/snowplow/config/iglu-resolver.json",
-      "--enrichments", "file:/snowplow/config/enrichments",
-      "--force-cached-files-download"
+      "--config", "/snowplow/config/snowplow-enrich.hocon",
+      "--iglu-config", "/snowplow/config/iglu-resolver.json",
+      "--enrichments", "/snowplow/config/enrichments"
     ]
     restart: always
     depends_on: