Merge branch 'master' into 22.3

* master: chore: start stack once in cloud tests (#9879) feat(apps): frontend apps (#9831) chore: Fix snapshots on master (#9885) chore(apps): rename plugins to apps (#9755) refactor: Remove constance library dependency, use json-encoded model (#9852) chore(clickhouse): avoid creating kafka_events, events_mv (#9863) fix(insights): Fix timezone date issues (#9678) refactor(plugin-server): refactor the event pipeline (#9829) feat(object storage): add unused object storage (#9846) fix: make kafka health check timeout test reliable (#9857) fix: query elements from start of day (#9827)
PostHog · May 20, 2022 · 7212f0d · 7212f0d
2 parents 9926a52 + 0f91599
commit 7212f0d
Show file tree

Hide file tree

Showing 216 changed files with 4,928 additions and 1,710 deletions.
diff --git a/.github/actions/run-backend-tests/action.yml b/.github/actions/run-backend-tests/action.yml
@@ -37,7 +37,7 @@ runs:
           run: |
               export CLICKHOUSE_SERVER_IMAGE=${{ inputs.clickhouse-server-image }}
               docker-compose -f docker-compose.dev.yml down
-              docker-compose -f docker-compose.dev.yml up -d db clickhouse zookeeper kafka redis &
+              docker-compose -f docker-compose.dev.yml up -d db clickhouse zookeeper kafka redis object_storage &
 
         - name: Set up Python
           uses: actions/setup-python@v2

diff --git a/.github/workflows/ci-backend.yml b/.github/workflows/ci-backend.yml
@@ -24,6 +24,10 @@ env:
     CLICKHOUSE_VERIFY: 'False'
     TEST: 1
     CLICKHOUSE_SERVER_IMAGE_VERSION: ${{ github.event.inputs.clickhouseServerVersion || '' }}
+    OBJECT_STORAGE_ENABLED: 'True'
+    OBJECT_STORAGE_ENDPOINT: 'http://localhost:19000'
+    OBJECT_STORAGE_ACCESS_KEY_ID: 'object_storage_root_user'
+    OBJECT_STORAGE_SECRET_ACCESS_KEY: 'object_storage_root_password'
 
 jobs:
     # Job to decide if we should run backend ci
@@ -293,10 +297,14 @@ jobs:
                   cp -r messaging deploy/
                   cat multi_tenancy_settings.py > deploy/posthog/settings/cloud.py
                   cat requirements.txt >> deploy/requirements.txt
+
+            - name: Add kafka host to /etc/hosts for kafka connectivity
+              run: sudo echo "127.0.0.1 kafka" | sudo tee -a /etc/hosts
+
             - name: Start stack with Docker Compose
               run: |
                   docker-compose -f deploy/docker-compose.dev.yml down
-                  docker-compose -f deploy/docker-compose.dev.yml up -d db clickhouse zookeeper kafka redis &
+                  docker-compose -f deploy/docker-compose.dev.yml up -d db clickhouse zookeeper kafka redis object_storage &
 
             - name: Set up Python 3.8.12
               uses: actions/setup-python@v2
@@ -357,9 +365,6 @@ jobs:
                   python manage.py makemigrations --check --dry-run
                   python manage.py migrate
 
-            - name: Add kafka host to /etc/hosts for kafka connectivity
-              run: sudo echo "127.0.0.1 kafka" | sudo tee -a /etc/hosts
-
             - name: Set up needed files
               run: |
                   cd deploy

diff --git a/.github/workflows/ci-plugin-server.yml b/.github/workflows/ci-plugin-server.yml
@@ -11,6 +11,14 @@ on:
             - 'docker*.yml'
             - '*Dockerfile'
 
+env:
+    OBJECT_STORAGE_ENABLED: true
+    OBJECT_STORAGE_ENDPOINT: 'http://localhost:19000'
+    OBJECT_STORAGE_ACCESS_KEY_ID: 'object_storage_root_user'
+    OBJECT_STORAGE_SECRET_ACCESS_KEY: 'object_storage_root_password'
+    OBJECT_STORAGE_SESSION_RECORDING_FOLDER: 'session_recordings'
+    OBJECT_STORAGE_BUCKET: 'posthog'
+
 jobs:
     code-quality:
         name: Code quality
@@ -73,8 +81,8 @@ jobs:
                   sudo bash -c 'echo "127.0.0.1 kafka zookeeper" >> /etc/hosts'
                   ping -c 1 kafka
                   ping -c 1 zookeeper
-            - name: Start Kafka, ClickHouse, Zookeeper
-              run: docker-compose -f docker-compose.dev.yml up -d zookeeper kafka clickhouse
+            - name: Start Kafka, ClickHouse, Zookeeper, Object Storage
+              run: docker-compose -f docker-compose.dev.yml up -d zookeeper kafka clickhouse object_storage
 
             - name: Set up Python 3.8.12
               uses: actions/setup-python@v2
@@ -154,8 +162,8 @@ jobs:
                   sudo bash -c 'echo "127.0.0.1 kafka zookeeper" >> /etc/hosts'
                   ping -c 1 kafka
                   ping -c 1 zookeeper
-            - name: Start Kafka, ClickHouse, Zookeeper
-              run: docker-compose -f docker-compose.dev.yml up -d zookeeper kafka clickhouse
+            - name: Start Kafka, ClickHouse, Zookeeper, Object Storage
+              run: docker-compose -f docker-compose.dev.yml up -d zookeeper kafka clickhouse object_storage
 
             - name: Set up Python 3.8.12
               uses: actions/setup-python@v2
@@ -236,8 +244,8 @@ jobs:
                   sudo bash -c 'echo "127.0.0.1 kafka zookeeper" >> /etc/hosts'
                   ping -c 1 kafka
                   ping -c 1 zookeeper
-            - name: Start Kafka, ClickHouse, Zookeeper
-              run: docker-compose -f docker-compose.dev.yml up -d zookeeper kafka clickhouse
+            - name: Start Kafka, ClickHouse, Zookeeper, Object Storage
+              run: docker-compose -f docker-compose.dev.yml up -d zookeeper kafka clickhouse object_storage
 
             - name: Set up Python 3.8.12
               uses: actions/setup-python@v2
@@ -318,8 +326,8 @@ jobs:
                   sudo bash -c 'echo "127.0.0.1 kafka zookeeper" >> /etc/hosts'
                   ping -c 1 kafka
                   ping -c 1 zookeeper
-            - name: Start Kafka, ClickHouse, Zookeeper
-              run: docker-compose -f docker-compose.dev.yml up -d zookeeper kafka clickhouse
+            - name: Start Kafka, ClickHouse, Zookeeper, Object Storage
+              run: docker-compose -f docker-compose.dev.yml up -d zookeeper kafka clickhouse object_storage
 
             - name: Set up Python 3.8.12
               uses: actions/setup-python@v2

diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
@@ -18,6 +18,10 @@ env:
     SITE_URL: 'test.posthog.net' # used to test password resets
     NO_RESTART_LOOP: 1
     CLICKHOUSE_SECURE: 0
+    OBJECT_STORAGE_ENABLED: 1
+    OBJECT_STORAGE_ENDPOINT: 'http://localhost:19000'
+    OBJECT_STORAGE_ACCESS_KEY_ID: 'object_storage_root_user'
+    OBJECT_STORAGE_SECRET_ACCESS_KEY: 'object_storage_root_password'
 
 jobs:
     # Job that lists and chunks spec file names and caches node modules
@@ -69,7 +73,7 @@ jobs:
             - name: Start stack with Docker Compose
               run: |
                   docker-compose -f docker-compose.dev.yml down
-                  docker-compose -f docker-compose.dev.yml up -d db clickhouse zookeeper kafka redis &
+                  docker-compose -f docker-compose.dev.yml up -d db clickhouse zookeeper kafka redis object_storage &
             - name: Add kafka host to /etc/hosts for kafka connectivity
               run: sudo echo "127.0.0.1 kafka" | sudo tee -a /etc/hosts
 

diff --git a/.run/Plugin Server.run.xml b/.run/Plugin Server.run.xml
@@ -7,12 +7,13 @@
     </scripts>
     <node-interpreter value="project" />
     <envs>
-      <env name="WORKER_CONCURRENCY" value="2" />
+      <env name="CLICKHOUSE_SECURE" value="False" />
       <env name="DATABASE_URL" value="postgres://posthog:posthog@localhost:5432/posthog" />
       <env name="KAFKA_ENABLED" value="true" />
-      <env name="CLICKHOUSE_SECURE" value="False" />
       <env name="KAFKA_HOSTS" value="localhost:9092" />
+      <env name="WORKER_CONCURRENCY" value="2" />
+      <env name="OBJECT_STORAGE_ENABLED" value="True" />
     </envs>
     <method v="2" />
   </configuration>
-</component>
+</component>
diff --git a/bin/check_kafka_clickhouse_up b/bin/check_kafka_clickhouse_up
@@ -4,10 +4,10 @@ set -e
 
 # Check Kafka
 while true; do
-nc -z localhost 9092 && break || echo "Checking Kafka status..." && sleep 1
+nc -z localhost 9092 && break || echo 'Checking Kafka status...' && sleep 1
 done
 
 # Check ClickHouse
 while true; do
-curl -s -o /dev/null -I 'http://localhost:8123/' && break || echo "Checking ClickHouse status..." && sleep 1
+curl -s -o /dev/null -I 'http://localhost:8123/' && break || echo 'Checking ClickHouse status...' && sleep 1
 done
diff --git a/docker-compose.arm64.yml b/docker-compose.arm64.yml
@@ -78,6 +78,7 @@ services:
             - redis
             - clickhouse
             - kafka
+            - object_storage
     web:
         <<: *worker
         command: '${CH_WEB_SCRIPT:-./ee/bin/docker-ch-dev-web}'
@@ -103,3 +104,17 @@ services:
             - redis
             - clickhouse
             - kafka
+            - object_storage
+
+    object_storage:
+        image: minio/minio
+        ports:
+            - '19000:19000'
+            - '19001:19001'
+        volumes:
+            - ./object_storage:/data
+        environment:
+            MINIO_ROOT_USER: object_storage_root_user
+            MINIO_ROOT_PASSWORD: object_storage_root_password
+        entrypoint: sh
+        command: -c 'mkdir -p /data/posthog && minio server --address ":19000" --console-address ":19001" /data' # create the 'posthog' bucket before starting the service
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
@@ -75,6 +75,7 @@ services:
             - redis
             - clickhouse
             - kafka
+            - object_storage
     web:
         <<: *worker
         command: '${CH_WEB_SCRIPT:-./ee/bin/docker-ch-dev-web}'
@@ -100,3 +101,15 @@ services:
             - redis
             - clickhouse
             - kafka
+            - object_storage
+
+    object_storage:
+        image: minio/minio
+        ports:
+            - '19000:19000'
+            - '19001:19001'
+        environment:
+            MINIO_ROOT_USER: object_storage_root_user
+            MINIO_ROOT_PASSWORD: object_storage_root_password
+        entrypoint: sh
+        command: -c 'mkdir -p /data/posthog && chmod u+rxw /data/posthog && minio server --address ":19000" --console-address ":19001" /data' # create the 'posthog' bucket before starting the service
diff --git a/docker-compose.hobby.yml b/docker-compose.hobby.yml
@@ -84,6 +84,7 @@ services:
             - redis
             - clickhouse
             - kafka
+            - object_storage
     web:
         <<: *worker
         command: /compose/start
@@ -117,6 +118,21 @@ services:
             - redis
             - clickhouse
             - kafka
+            - object_storage
+
+    object_storage:
+        image: minio/minio
+        ports:
+            - '19000:19000'
+            - '19001:19001'
+        volumes:
+            - object_storage:/data
+        environment:
+            MINIO_ROOT_USER: object_storage_root_user
+            MINIO_ROOT_PASSWORD: object_storage_root_password
+        entrypoint: sh
+        command: -c 'mkdir -p /data/posthog && minio server --address ":19000" --console-address ":19001" /data' # create the 'posthog' bucket before starting the service
+
     asyncmigrationscheck:
         <<: *worker
         command: python manage.py run_async_migrations --check
@@ -127,3 +143,4 @@ volumes:
     zookeeper-data:
     zookeeper-datalog:
     zookeeper-logs:
+    object_storage:
diff --git a/docker-compose.test.yml b/docker-compose.test.yml
@@ -23,3 +23,4 @@ services:
             - redis
             - clickhouse
             - kafka
+            - object_storage
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -54,6 +54,7 @@ services:
             - redis
             - clickhouse
             - kafka
+            - object_storage
         environment:
             DATABASE_URL: postgres://posthog:posthog@db:5432/posthog
             REDIS_URL: redis://redis:6379/
@@ -70,6 +71,20 @@ services:
         ports:
             - 8000:8000
             - 80:8000
+
+    object_storage:
+        image: minio/minio
+        ports:
+            - '19000:19000'
+            - '19001:19001'
+        volumes:
+            - ./object_storage:/data
+        environment:
+            MINIO_ROOT_USER: object_storage_root_user
+            MINIO_ROOT_PASSWORD: object_storage_root_password
+        entrypoint: sh
+        command: -c 'mkdir -p /data/posthog && minio server --address ":19000" --console-address ":19001" /data' # create the 'posthog' bucket before starting the service
+
 volumes:
     postgres-data:
 version: '3'
diff --git a/ee/api/test/test_instance_settings.py b/ee/api/test/test_instance_settings.py
@@ -1,11 +1,11 @@
 # TODO: Once Clickhouse is moved out of EE, move test cases to posthog/api/test/test_instance_settings.py
-from constance import config
 from rest_framework import status
 
 from ee.api.test.base import APILicensedTest
 from ee.clickhouse.sql.session_recording_events import SESSION_RECORDING_EVENTS_DATA_TABLE
 from ee.clickhouse.util import ClickhouseTestMixin, snapshot_clickhouse_alter_queries
 from posthog.client import sync_execute
+from posthog.models.instance_setting import get_instance_setting
 from posthog.settings.data_stores import CLICKHOUSE_DATABASE
 
 
@@ -25,7 +25,7 @@ def test_update_recordings_ttl_setting(self):
         self.assertEqual(response.status_code, status.HTTP_200_OK)
         self.assertEqual(response.json()["value"], 5)
 
-        self.assertEqual(config.RECORDINGS_TTL_WEEKS, 5)
+        self.assertEqual(get_instance_setting("RECORDINGS_TTL_WEEKS"), 5)
 
         table_engine = sync_execute(
             "SELECT engine_full FROM system.tables WHERE database = %(database)s AND name = %(table)s",

diff --git a/ee/clickhouse/materialized_columns/columns.py b/ee/clickhouse/materialized_columns/columns.py
@@ -2,13 +2,13 @@
 from datetime import timedelta
 from typing import Dict, List, Literal, Union
 
-from constance import config
 from django.utils.timezone import now
 
 from ee.clickhouse.materialized_columns.util import cache_for
 from ee.clickhouse.replication.utils import clickhouse_is_replicated
 from ee.clickhouse.sql.clickhouse import trim_quotes_expr
 from posthog.client import sync_execute
+from posthog.models.instance_setting import get_instance_setting
 from posthog.models.property import PropertyName, TableWithProperties
 from posthog.models.utils import generate_random_short_suffix
 from posthog.settings import CLICKHOUSE_CLUSTER, CLICKHOUSE_DATABASE, TEST
@@ -32,7 +32,7 @@ def get_materialized_columns(table: TablesWithMaterializedColumns) -> Dict[Prope
     """,
         {"database": CLICKHOUSE_DATABASE, "table": table},
     )
-    if rows and getattr(config, "MATERIALIZED_COLUMNS_ENABLED"):
+    if rows and get_instance_setting("MATERIALIZED_COLUMNS_ENABLED"):
         return {extract_property(comment): column_name for comment, column_name in rows}
     else:
         return {}

diff --git a/ee/clickhouse/migrations/0004_kafka.py b/ee/clickhouse/migrations/0004_kafka.py
@@ -1,11 +1,6 @@
 from infi.clickhouse_orm import migrations
 
-from ee.clickhouse.sql.events import (
-    DISTRIBUTED_EVENTS_TABLE_SQL,
-    EVENTS_TABLE_MV_SQL,
-    KAFKA_EVENTS_TABLE_SQL,
-    WRITABLE_EVENTS_TABLE_SQL,
-)
+from ee.clickhouse.sql.events import DISTRIBUTED_EVENTS_TABLE_SQL, WRITABLE_EVENTS_TABLE_SQL
 from ee.clickhouse.sql.person import (
     KAFKA_PERSONS_DISTINCT_ID_TABLE_SQL,
     KAFKA_PERSONS_TABLE_SQL,
@@ -14,11 +9,26 @@
 )
 from posthog.settings import CLICKHOUSE_REPLICATION
 
+# NOTE: this migration previously created kafka_events and events_mv tables.
+# kafka_events was a Kafka ClickHouse engine table that used Protobuf for
+# serialization. To remove complexity of deployments, Protobuf support has been
+# removed from the PostHog app and the Plugin server. See
+# https://github.com/PostHog/posthog/issues/9207 for detail.
+#
+# These have been superseded by kafka_events_json and events_json_mv. However,
+# we can't simply add a DROP TABLE for the old tables as there may still be
+# events in Kafka that need to be consumed. We'd need some orchestration around
+# this to avoid losing in flight events. See migration
+# ee/clickhouse/migrations/0025_json_events.py for details of the new tables.
+#
+# For new installs however, we don't need to be consider this case, so we can
+# simply not create them.
+#
+# WARNING: this does however mean that you can arrive at different DB states
+# depending on which versions of PostHog you have run.
 operations = [
-    migrations.RunSQL(KAFKA_EVENTS_TABLE_SQL()),
     migrations.RunSQL(KAFKA_PERSONS_TABLE_SQL()),
     migrations.RunSQL(KAFKA_PERSONS_DISTINCT_ID_TABLE_SQL()),
-    migrations.RunSQL(EVENTS_TABLE_MV_SQL()),
     migrations.RunSQL(PERSONS_TABLE_MV_SQL),
     migrations.RunSQL(PERSONS_DISTINCT_ID_TABLE_MV_SQL),
 ]

diff --git a/ee/clickhouse/queries/funnels/base.py b/ee/clickhouse/queries/funnels/base.py
@@ -3,7 +3,6 @@
 from abc import ABC
 from typing import Any, Dict, List, Optional, Tuple, Union, cast
 
-from django.utils import timezone
 from rest_framework.exceptions import ValidationError
 
 from ee.clickhouse.materialized_columns.columns import ColumnName
@@ -116,9 +115,6 @@ def _update_filters(self):
         if not self._filter._date_from:
             data.update({"date_from": relative_date_parse("-7d")})
 
-        if not self._filter._date_to:
-            data.update({"date_to": timezone.now()})
-
         if self._filter.breakdown and not self._filter.breakdown_type:
             data.update({"breakdown_type": "event"})
-Original file line number
+Diff line change
@@ Expand Up / @@ -23,3 +23,4 @@ services: @@
                 - redis
                 - clickhouse
                 - kafka
+                - object_storage