diff --git a/README.md b/README.md index 046e16d9..62289a3f 100644 --- a/README.md +++ b/README.md @@ -88,7 +88,7 @@ Snowplow Mini runs several distinct applications on the same box which are all l * Starts server listening on `http://< sp mini public ip>/` which events can be sent to. * Sends "good" events to the `RawEvents` NSQ topic * Sends "bad" events to the `BadEvents` NSQ topic -* Stream Enrich: +* Enrich: * Reads events in from the `RawEvents` NSQ topic * Sends events which passed the enrichment process to the `EnrichedEvents` NSQ topic * Sends events which failed the enrichment process to the `BadEvents` NSQ topic diff --git a/provisioning/resources/configs/compositions/docker-compose-aws.yml b/provisioning/resources/configs/compositions/docker-compose-aws.yml index a6822c87..4c7541db 100644 --- a/provisioning/resources/configs/compositions/docker-compose-aws.yml +++ b/provisioning/resources/configs/compositions/docker-compose-aws.yml @@ -157,14 +157,13 @@ services: - "JDK_JAVA_OPTIONS=-XX:MinRAMPercentage=50 -XX:MaxRAMPercentage=75 -Dlog4j2.formatMsgNoLookups=true" mem_limit: ${SP_COLLECTOR_MEM_SIZE} - stream-enrich: - image: snowplow/stream-enrich-nsq:3.8.0-distroless - container_name: stream-enrich-nsq + enrich: + image: snowplow/snowplow-enrich-nsq:3.8.2-distroless + container_name: enrich command: [ - "--config", "/snowplow/config/snowplow-stream-enrich.hocon", - "--resolver", "file:/snowplow/config/iglu-resolver.json", - "--enrichments", "file:/snowplow/config/enrichments", - "--force-cached-files-download" + "--config", "/snowplow/config/snowplow-enrich.hocon", + "--iglu-config", "/snowplow/config/iglu-resolver.json", + "--enrichments", "/snowplow/config/enrichments" ] restart: always depends_on: @@ -175,7 +174,7 @@ services: driver: awslogs options: awslogs-group: "${AWS_LOGS_GROUP}" - awslogs-stream: "stream-enrich" + awslogs-stream: "enrich" environment: - "JDK_JAVA_OPTIONS=-XX:MinRAMPercentage=50 -XX:MaxRAMPercentage=75 -Dlog4j2.formatMsgNoLookups=true" mem_limit: ${SP_ENRICH_MEM_SIZE} diff --git a/provisioning/resources/configs/compositions/docker-compose-gcp.yml b/provisioning/resources/configs/compositions/docker-compose-gcp.yml index 2dd51432..e936c10c 100644 --- a/provisioning/resources/configs/compositions/docker-compose-gcp.yml +++ b/provisioning/resources/configs/compositions/docker-compose-gcp.yml @@ -133,14 +133,13 @@ services: - "JDK_JAVA_OPTIONS=-XX:MinRAMPercentage=50 -XX:MaxRAMPercentage=75 -Dlog4j2.formatMsgNoLookups=true" mem_limit: ${SP_COLLECTOR_MEM_SIZE} - stream-enrich: - image: snowplow/stream-enrich-nsq:3.8.0-distroless - container_name: stream-enrich-nsq + enrich: + image: snowplow/snowplow-enrich-nsq:3.8.2-distroless + container_name: enrich command: [ - "--config", "/snowplow/config/snowplow-stream-enrich.hocon", - "--resolver", "file:/snowplow/config/iglu-resolver.json", - "--enrichments", "file:/snowplow/config/enrichments", - "--force-cached-files-download" + "--config", "/snowplow/config/snowplow-enrich.hocon", + "--iglu-config", "/snowplow/config/iglu-resolver.json", + "--enrichments", "/snowplow/config/enrichments" ] restart: always depends_on: diff --git a/provisioning/resources/configs/control-plane-api.toml b/provisioning/resources/configs/control-plane-api.toml index 248e65a1..cb1a6aef 100644 --- a/provisioning/resources/configs/control-plane-api.toml +++ b/provisioning/resources/configs/control-plane-api.toml @@ -17,7 +17,7 @@ collector = "snowplow-stream-collector.hocon" [init_scripts] stream_collector = "scala-stream-collector" -stream_enrich = "stream-enrich" +enrich = "enrich" es_loader_good = "elasticsearch-loader-good" es_loader_bad = "elasticsearch-loader-bad" iglu = "iglu-server" diff --git a/provisioning/resources/configs/snowplow-enrich.hocon b/provisioning/resources/configs/snowplow-enrich.hocon new file mode 100644 index 00000000..e982f47c --- /dev/null +++ b/provisioning/resources/configs/snowplow-enrich.hocon @@ -0,0 +1,33 @@ +{ + "input": { + "topic": "RawEvents" + "lookupHost": "nsqlookupd" + "lookupPort": 4161 + } + + "output": { + "good": { + "topic": "EnrichedEvents" + "nsqdHost": "nsqd" + "nsqdPort": 4150 + } + + "bad": { + "topic": "BadEnrichedEvents" + "nsqdHost": "nsqd" + "nsqdPort": 4150 + } + + "pii": { + "type": "Nsq" + "topic": "PiiEvents" + "nsqdHost": "nsqd" + "nsqdPort": 4150 + "backoffPolicy": { + "minBackoff": 100 milliseconds + "maxBackoff": 10 seconds + "maxRetries": 10 + } + } + } +} diff --git a/provisioning/resources/control-plane/model.go b/provisioning/resources/control-plane/model.go index c75668b3..e0b31ade 100644 --- a/provisioning/resources/control-plane/model.go +++ b/provisioning/resources/control-plane/model.go @@ -24,7 +24,7 @@ type configFileNames struct { type initScripts struct { Collector string `toml:"stream_collector"` - Enrich string `toml:"stream_enrich"` + Enrich string `toml:"enrich"` EsLoaderGood string `toml:"es_loader_good"` EsLoaderBad string `toml:"es_loader_bad"` Iglu string diff --git a/provisioning/resources/ui/js/components/Overview.tsx b/provisioning/resources/ui/js/components/Overview.tsx index 9521c436..8f9ca24c 100644 --- a/provisioning/resources/ui/js/components/Overview.tsx +++ b/provisioning/resources/ui/js/components/Overview.tsx @@ -51,7 +51,7 @@ export class Overview extends React.Component<{}, {}> {