compose: update Clickhouse schema to match casing of proto (#133)

This commit resolves an issue with the mapping between the ClickHouse
schema and the flow.proto schema.

In 3326554, the casing of the proto fields was updated, but the
ClickHouse column names were not also updated, resulting in the
ClickHouse Kafka engine only being able to successfully deserialize
fields that without an underscore.

Also updates the provisioned dashboards. Versions of the tools are also updated.

Co-authored-by: lspgn <lspgn@users.noreply.github.com>

compose/elk/docker-compose.yml

@@ -1,17 +1,21 @@
 version: "3"
 services:
-  goflow:
+  goflow2:
     build:
       context: ../../
       dockerfile: Dockerfile
+      args:
+        VERSION: compose
+        LDFLAGS: -X main.version=compose
+    image: netsampler/goflow2
     user: root # because docker-compose mount as root
     ports:
       - '8080:8080'
       - '6343:6343/udp'
       - '2055:2055/udp'
     command:
       - -transport=file
-      - -transport.file=/var/log/goflow/goflow.log
+      - -transport.file=/var/log/goflow/goflow2.log
       - -format=json
     restart: always
     logging:

compose/kcg/clickhouse/create.sh

@@ -4,7 +4,7 @@ set -e
 clickhouse client -n <<-EOSQL
 
     CREATE DATABASE IF NOT EXISTS dictionaries;
-    
+
     CREATE DICTIONARY IF NOT EXISTS dictionaries.protocols (
         proto UInt8,
         name String,
@@ -17,27 +17,27 @@ clickhouse client -n <<-EOSQL
 
     CREATE TABLE IF NOT EXISTS flows
     (
-        TimeReceived UInt64,
-        TimeFlowStart UInt64,
+        time_received UInt64,
+        time_flow_start UInt64,
 
-        SequenceNum UInt32,
-        SamplingRate UInt64,
-        SamplerAddress FixedString(16),
+        sequence_num UInt32,
+        sampling_rate UInt64,
+        sampler_address FixedString(16),
 
-        SrcAddr FixedString(16),
-        DstAddr FixedString(16),
+        src_addr FixedString(16),
+        dst_addr FixedString(16),
 
-        SrcAS UInt32,
-        DstAS UInt32,
+        src_as UInt32,
+        dst_as UInt32,
 
-        EType UInt32,
-        Proto UInt32,
+        etype UInt32,
+        proto UInt32,
 
-        SrcPort UInt32,
-        DstPort UInt32,
+        src_port UInt32,
+        dst_port UInt32,
 
-        Bytes UInt64,
-        Packets UInt64
+        bytes UInt64,
+        packets UInt64
     ) ENGINE = Kafka()
     SETTINGS
         kafka_broker_list = 'kafka:9092',
@@ -48,78 +48,78 @@ clickhouse client -n <<-EOSQL
 
     CREATE TABLE IF NOT EXISTS flows_raw
     (
-        Date Date,
-        TimeReceived DateTime,
-        TimeFlowStart DateTime,
+        date Date,
+        time_received DateTime,
+        time_flow_start DateTime,
 
-        SequenceNum UInt32,
-        SamplingRate UInt64,
-        SamplerAddress FixedString(16),
+        sequence_num UInt32,
+        sampling_rate UInt64,
+        sampler_address FixedString(16),
 
-        SrcAddr FixedString(16),
-        DstAddr FixedString(16),
+        src_addr FixedString(16),
+        dst_addr FixedString(16),
 
-        SrcAS UInt32,
-        DstAS UInt32,
+        src_as UInt32,
+        dst_as UInt32,
 
-        EType UInt32,
-        Proto UInt32,
+        etype UInt32,
+        proto UInt32,
 
-        SrcPort UInt32,
-        DstPort UInt32,
+        src_port UInt32,
+        dst_port UInt32,
 
-        Bytes UInt64,
-        Packets UInt64
+        bytes UInt64,
+        packets UInt64
     ) ENGINE = MergeTree()
-    PARTITION BY Date
-    ORDER BY TimeReceived;
+    PARTITION BY date
+    ORDER BY time_received;
 
-    CREATE MATERIALIZED VIEW IF NOT EXISTS flows_raw_view TO flows_raw 
+    CREATE MATERIALIZED VIEW IF NOT EXISTS flows_raw_view TO flows_raw
     AS SELECT
-        toDate(TimeReceived) AS Date,
+        toDate(time_received) AS date,
         *
        FROM flows;
 
     CREATE TABLE IF NOT EXISTS flows_5m
     (
-        Date Date,
-        Timeslot DateTime,
+        date Date,
+        timeslot DateTime,
 
-        SrcAS UInt32,
-        DstAS UInt32,
+        src_as UInt32,
+        dst_as UInt32,
 
-        ETypeMap Nested (
-            EType UInt32,
-            Bytes UInt64,
-            Packets UInt64,
-            Count UInt64
+        etypeMap Nested (
+            etype UInt32,
+            bytes UInt64,
+            packets UInt64,
+            count UInt64
         ),
 
-        Bytes UInt64,
-        Packets UInt64,
-        Count UInt64
+        bytes UInt64,
+        packets UInt64,
+        count UInt64
     ) ENGINE = SummingMergeTree()
-    PARTITION BY Date
-    ORDER BY (Date, Timeslot, SrcAS, DstAS, \`ETypeMap.EType\`);
+    PARTITION BY date
+    ORDER BY (date, timeslot, src_as, dst_as, \`etypeMap.etype\`);
 
-    CREATE MATERIALIZED VIEW IF NOT EXISTS flows_5m_view TO flows_5m 
+    CREATE MATERIALIZED VIEW IF NOT EXISTS flows_5m_view TO flows_5m
     AS
         SELECT
-            Date,
-            toStartOfFiveMinute(TimeReceived) AS Timeslot,
-            SrcAS,
-            DstAS,
+            date,
+            toStartOfFiveMinute(time_received) AS timeslot,
+            src_as,
+            dst_as,
 
-            [EType] AS \`ETypeMap.EType\`,
-            [Bytes] AS \`ETypeMap.Bytes\`,
-            [Packets] AS \`ETypeMap.Packets\`,
-            [Count] AS \`ETypeMap.Count\`,
+            [etype] AS \`etypeMap.etype\`,
+            [bytes] AS \`etypeMap.bytes\`,
+            [packets] AS \`etypeMap.packets\`,
+            [count] AS \`etypeMap.count\`,
 
-            sum(Bytes) AS Bytes,
-            sum(Packets) AS Packets,
-            count() AS Count
+            sum(bytes) AS bytes,
+            sum(packets) AS packets,
+            count() AS count
 
         FROM flows_raw
-        GROUP BY Date, Timeslot, SrcAS, DstAS, \`ETypeMap.EType\`;
-        
+        GROUP BY date, timeslot, src_as, dst_as, \`etypeMap.etype\`;
+
 EOSQL

compose/kcg/docker-compose.yml

@@ -1,14 +1,14 @@
 version: "3"
 services:
   zookeeper:
-    image: bitnami/zookeeper:3.6.3
+    image: bitnami/zookeeper:3.7.1
     ports:
       - 2181:2181
     environment:
       - ALLOW_ANONYMOUS_LOGIN=yes
     restart: always
   kafka:
-    image: bitnami/kafka:2.8.0
+    image: bitnami/kafka:3.4.0
     ports:
       - 9092:9092
     environment:
@@ -19,10 +19,11 @@ services:
     depends_on:
       - zookeeper
   grafana:
-    image: grafana/grafana:9.1.7
+    image: grafana/grafana:9.4.3
     environment:
-      - GF_INSTALL_PLUGINS=grafana-clickhouse-datasource
-      - GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS=vertamedia-clickhouse-datasource
+      - GF_INSTALL_PLUGINS=vertamedia-clickhouse-datasource
+    #  - GF_INSTALL_PLUGINS=grafana-clickhouse-datasource
+    #  - GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS=vertamedia-clickhouse-datasource
     ports:
       - 3000:3000
     restart: always
@@ -31,7 +32,7 @@ services:
       - ./grafana/dashboards.yml:/etc/grafana/provisioning/dashboards/dashboards.yml
       - ./grafana/dashboards:/var/lib/grafana/dashboards
   prometheus:
-    image: prom/prometheus:v2.27.0
+    image: prom/prometheus:v2.37.6
     ports:
       - 9090:9090
     restart: always
@@ -41,6 +42,10 @@ services:
     build:
       context: ../../
       dockerfile: Dockerfile
+      args:
+        VERSION: compose
+        LDFLAGS: -X main.version=compose
+    image: netsampler/goflow2
     depends_on:
       - kafka
     ports:
@@ -55,7 +60,7 @@ services:
     - -format=pb
     - -format.protobuf.fixedlen=true
   db:
-    image: clickhouse/clickhouse-server:22.6.9.11-alpine
+    image: clickhouse/clickhouse-server:22.8.14.53-alpine
     ports:
       - 8123:8123
     volumes: