feat(object storage): add unused object storage

.github/actions/run-backend-tests/action.yml

@@ -34,7 +34,7 @@ runs:
           run: |
               export CLICKHOUSE_SERVER_IMAGE=${{ inputs.clickhouse-server-image }}
               docker-compose -f docker-compose.dev.yml down
-              docker-compose -f docker-compose.dev.yml up -d db clickhouse zookeeper kafka redis &
+              docker-compose -f docker-compose.dev.yml up -d db clickhouse zookeeper kafka redis object_storage &
 
         - name: Set up Python
           uses: actions/setup-python@v2

.github/workflows/ci-backend.yml

@@ -24,6 +24,11 @@ env:
     CLICKHOUSE_VERIFY: 'False'
     TEST: 1
     CLICKHOUSE_SERVER_IMAGE_VERSION: ${{ github.event.inputs.clickhouseServerVersion || '' }}
+    OBJECT_STORAGE_ENABLED: 'True'
+    OBJECT_STORAGE_HOST: 'localhost'
+    OBJECT_STORAGE_PORT: '19000'
+    OBJECT_STORAGE_ACCESS_KEY_ID: 'object_storage_root_user'
+    OBJECT_STORAGE_SECRET_ACCESS_KEY: 'object_storage_root_password'
 
 jobs:
     # Job to decide if we should run backend ci
@@ -264,7 +269,7 @@ jobs:
             - name: Start stack with Docker Compose
               run: |
                   docker-compose -f deploy/docker-compose.dev.yml down
-                  docker-compose -f deploy/docker-compose.dev.yml up -d db clickhouse zookeeper kafka redis &
+                  docker-compose -f deploy/docker-compose.dev.yml up -d db clickhouse zookeeper kafka redis object_storage &
 
             - name: Set up Python 3.8.12
               uses: actions/setup-python@v2

.github/workflows/ci-plugin-server.yml

@@ -11,6 +11,15 @@ on:
             - 'docker*.yml'
             - '*Dockerfile'
 
+env:
+    OBJECT_STORAGE_ENABLED: true
+    OBJECT_STORAGE_HOST: 'localhost'
+    OBJECT_STORAGE_PORT: '19000'
+    OBJECT_STORAGE_ACCESS_KEY_ID: 'object_storage_root_user'
+    OBJECT_STORAGE_SECRET_ACCESS_KEY: 'object_storage_root_password'
+    OBJECT_STORAGE_SESSION_RECORDING_FOLDER: 'session_recordings'
+    OBJECT_STORAGE_BUCKET: 'posthog'
+
 jobs:
     code-quality:
         name: Code quality
@@ -73,8 +82,8 @@ jobs:
                   sudo bash -c 'echo "127.0.0.1 kafka zookeeper" >> /etc/hosts'
                   ping -c 1 kafka
                   ping -c 1 zookeeper
-            - name: Start Kafka, ClickHouse, Zookeeper
-              run: docker-compose -f docker-compose.dev.yml up -d zookeeper kafka clickhouse
+            - name: Start Kafka, ClickHouse, Zookeeper, Object Storage
+              run: docker-compose -f docker-compose.dev.yml up -d zookeeper kafka clickhouse object_storage
 
             - name: Set up Python 3.8.12
               uses: actions/setup-python@v2
@@ -154,8 +163,8 @@ jobs:
                   sudo bash -c 'echo "127.0.0.1 kafka zookeeper" >> /etc/hosts'
                   ping -c 1 kafka
                   ping -c 1 zookeeper
-            - name: Start Kafka, ClickHouse, Zookeeper
-              run: docker-compose -f docker-compose.dev.yml up -d zookeeper kafka clickhouse
+            - name: Start Kafka, ClickHouse, Zookeeper, Object Storage
+              run: docker-compose -f docker-compose.dev.yml up -d zookeeper kafka clickhouse object_storage
 
             - name: Set up Python 3.8.12
               uses: actions/setup-python@v2
@@ -236,8 +245,8 @@ jobs:
                   sudo bash -c 'echo "127.0.0.1 kafka zookeeper" >> /etc/hosts'
                   ping -c 1 kafka
                   ping -c 1 zookeeper
-            - name: Start Kafka, ClickHouse, Zookeeper
-              run: docker-compose -f docker-compose.dev.yml up -d zookeeper kafka clickhouse
+            - name: Start Kafka, ClickHouse, Zookeeper, Object Storage
+              run: docker-compose -f docker-compose.dev.yml up -d zookeeper kafka clickhouse object_storage
 
             - name: Set up Python 3.8.12
               uses: actions/setup-python@v2
@@ -318,8 +327,8 @@ jobs:
                   sudo bash -c 'echo "127.0.0.1 kafka zookeeper" >> /etc/hosts'
                   ping -c 1 kafka
                   ping -c 1 zookeeper
-            - name: Start Kafka, ClickHouse, Zookeeper
-              run: docker-compose -f docker-compose.dev.yml up -d zookeeper kafka clickhouse
+            - name: Start Kafka, ClickHouse, Zookeeper, Object Storage
+              run: docker-compose -f docker-compose.dev.yml up -d zookeeper kafka clickhouse object_storage
 
             - name: Set up Python 3.8.12
               uses: actions/setup-python@v2

.github/workflows/e2e.yml

@@ -18,6 +18,11 @@ env:
     SITE_URL: 'test.posthog.net' # used to test password resets
     NO_RESTART_LOOP: 1
     CLICKHOUSE_SECURE: 0
+    OBJECT_STORAGE_ENABLED: 1
+    OBJECT_STORAGE_HOST: 'localhost'
+    OBJECT_STORAGE_PORT: '19000'
+    OBJECT_STORAGE_ACCESS_KEY_ID: 'object_storage_root_user'
+    OBJECT_STORAGE_SECRET_ACCESS_KEY: 'object_storage_root_password'
 
 jobs:
     # Job that lists and chunks spec file names and caches node modules
@@ -69,7 +74,7 @@ jobs:
             - name: Start stack with Docker Compose
               run: |
                   docker-compose -f docker-compose.dev.yml down
-                  docker-compose -f docker-compose.dev.yml up -d db clickhouse zookeeper kafka redis &
+                  docker-compose -f docker-compose.dev.yml up -d db clickhouse zookeeper kafka redis object_storage &
             - name: Add kafka host to /etc/hosts for kafka connectivity
               run: sudo echo "127.0.0.1 kafka" | sudo tee -a /etc/hosts
 
@@ -91,9 +96,7 @@ jobs:
             - name: Install python dependencies
               if: steps.cache-virtualenv.outputs.cache-hit != 'true'
               run: |
-                  python -m pip install --upgrade pip
-                  python -m pip install $(grep -ivE "psycopg2" requirements.txt | cut -d'#' -f1) --no-cache-dir --compile
-                  python -m pip install psycopg2-binary --no-cache-dir --compile
+                  python -m pip install requirements.txt --no-cache-dir --compile
             - uses: actions/setup-node@v1
               with:
                   node-version: 16

.run/Plugin Server.run.xml

@@ -7,12 +7,13 @@
     </scripts>
     <node-interpreter value="project" />
     <envs>
-      <env name="WORKER_CONCURRENCY" value="2" />
+      <env name="CLICKHOUSE_SECURE" value="False" />
       <env name="DATABASE_URL" value="postgres://posthog:posthog@localhost:5432/posthog" />
       <env name="KAFKA_ENABLED" value="true" />
-      <env name="CLICKHOUSE_SECURE" value="False" />
       <env name="KAFKA_HOSTS" value="localhost:9092" />
+      <env name="WORKER_CONCURRENCY" value="2" />
+      <env name="OBJECT_STORAGE_ENABLED" value="True" />
     </envs>
     <method v="2" />
   </configuration>
-</component>
+</component>

docker-compose.arm64.yml

@@ -78,6 +78,7 @@ services:
             - redis
             - clickhouse
             - kafka
+            - object_storage
     web:
         <<: *worker
         command: '${CH_WEB_SCRIPT:-./ee/bin/docker-ch-dev-web}'
@@ -103,3 +104,17 @@ services:
             - redis
             - clickhouse
             - kafka
+            - object_storage
+
+    object_storage:
+        image: minio/minio
+        ports:
+            - '19000:19000'
+            - '19001:19001'
+        volumes:
+            - ./object_storage:/data
+        environment:
+            MINIO_ROOT_USER: object_storage_root_user
+            MINIO_ROOT_PASSWORD: object_storage_root_password
+        entrypoint: sh
+        command: -c 'mkdir -p /data/posthog && minio server --address ":19000" --console-address ":19001" /data' # create the 'posthog' bucket before starting the service

docker-compose.dev.yml

@@ -75,6 +75,7 @@ services:
             - redis
             - clickhouse
             - kafka
+            - object_storage
     web:
         <<: *worker
         command: '${CH_WEB_SCRIPT:-./ee/bin/docker-ch-dev-web}'
@@ -100,3 +101,15 @@ services:
             - redis
             - clickhouse
             - kafka
+            - object_storage
+
+    object_storage:
+        image: minio/minio
+        ports:
+            - '19000:19000'
+            - '19001:19001'
+        environment:
+            MINIO_ROOT_USER: object_storage_root_user
+            MINIO_ROOT_PASSWORD: object_storage_root_password
+        entrypoint: sh
+        command: -c 'mkdir -p /data/posthog && chmod u+rxw /data/posthog && minio server --address ":19000" --console-address ":19001" /data' # create the 'posthog' bucket before starting the service

docker-compose.hobby.yml

@@ -84,6 +84,7 @@ services:
             - redis
             - clickhouse
             - kafka
+            - object_storage
     web:
         <<: *worker
         command: /compose/start
@@ -117,6 +118,21 @@ services:
             - redis
             - clickhouse
             - kafka
+            - object_storage
+
+    object_storage:
+        image: minio/minio
+        ports:
+            - '19000:19000'
+            - '19001:19001'
+        volumes:
+            - object_storage:/data
+        environment:
+            MINIO_ROOT_USER: object_storage_root_user
+            MINIO_ROOT_PASSWORD: object_storage_root_password
+        entrypoint: sh
+        command: -c 'mkdir -p /data/posthog && minio server --address ":19000" --console-address ":19001" /data' # create the 'posthog' bucket before starting the service
+
     asyncmigrationscheck:
         <<: *worker
         command: python manage.py run_async_migrations --check
@@ -127,3 +143,4 @@ volumes:
     zookeeper-data:
     zookeeper-datalog:
     zookeeper-logs:
+    object_storage:

docker-compose.test.yml

@@ -23,3 +23,4 @@ services:
             - redis
             - clickhouse
             - kafka
+            - object_storage

docker-compose.yml

@@ -54,6 +54,7 @@ services:
             - redis
             - clickhouse
             - kafka
+            - object_storage
         environment:
             DATABASE_URL: postgres://posthog:posthog@db:5432/posthog
             REDIS_URL: redis://redis:6379/
@@ -70,6 +71,20 @@ services:
         ports:
             - 8000:8000
             - 80:8000
+
+    object_storage:
+        image: minio/minio
+        ports:
+            - '19000:19000'
+            - '19001:19001'
+        volumes:
+            - ./object_storage:/data
+        environment:
+            MINIO_ROOT_USER: object_storage_root_user
+            MINIO_ROOT_PASSWORD: object_storage_root_password
+        entrypoint: sh
+        command: -c 'mkdir -p /data/posthog && minio server --address ":19000" --console-address ":19001" /data' # create the 'posthog' bucket before starting the service
+
 volumes:
     postgres-data:
 version: '3'

frontend/src/mocks/fixtures/_preflight.json

@@ -9,6 +9,7 @@
     "initiated": true,
     "cloud": false,
     "demo": false,
+    "object_storage": true,
     "realm": "hosted-clickhouse",
     "available_social_auth_providers": {
         "github": false,

frontend/src/scenes/PreflightCheck/__mocks__/preflight.initial.json

@@ -28,6 +28,7 @@
                     "clickhouse": false,
                     "kafka": false,
                     "realm": "hosted",
+                    "object_storage": true,
                     "available_social_auth_providers": {
                         "github": false,
                         "gitlab": false,

frontend/src/scenes/PreflightCheck/preflightLogic.test.ts

@@ -86,6 +86,11 @@ describe('preflightLogic', () => {
                             status: 'warning',
                             caption: 'Set up before ingesting real user data',
                         },
+                        {
+                            id: 'object_storage',
+                            name: 'Object Storage',
+                            status: 'validated',
+                        },
                     ],
                 })
         })
@@ -144,6 +149,11 @@ describe('preflightLogic', () => {
                             status: 'optional',
                             caption: 'Not required for experimentation mode',
                         },
+                        {
+                            id: 'object_storage',
+                            name: 'Object Storage',
+                            status: 'validated',
+                        },
                     ],
                 })
         })
@@ -156,7 +166,7 @@ describe('preflightLogic', () => {
                 .toDispatchActions(['loadPreflightSuccess'])
                 .toMatchValues({
                     checksSummary: {
-                        summaryString: '6 successful, 1 warning, 2 errors',
+                        summaryString: '7 successful, 1 warning, 2 errors',
                         summaryStatus: 'error',
                     },
                 })
@@ -169,7 +179,7 @@ describe('preflightLogic', () => {
                 .toDispatchActions(['loadPreflightSuccess'])
                 .toMatchValues({
                     checksSummary: {
-                        summaryString: '6 successful, 1 warning, 1 error, 1 optional',
+                        summaryString: '7 successful, 1 warning, 1 error, 1 optional',
                         summaryStatus: 'error',
                     },
                 })

frontend/src/scenes/PreflightCheck/preflightLogic.tsx

@@ -76,7 +76,7 @@ export const preflightLogic = kea<
         checks: [
             (s) => [s.preflight, s.preflightMode],
             (preflight, preflightMode) => {
-                return [
+                const preflightItems = [
                     {
                         id: 'database',
                         name: 'Application database · Postgres',
@@ -139,7 +139,21 @@ export const preflightLogic = kea<
                                 ? 'Not required for experimentation mode'
                                 : 'Set up before ingesting real user data',
                     },
-                ] as PreflightItemInterface[]
+                ]
+
+                if (preflight?.object_storage || preflight?.is_debug) {
+                    /** __for now__, only prompt debug users if object storage is unhealthy **/
+                    preflightItems.push({
+                        id: 'object_storage',
+                        name: 'Object Storage',
+                        status: preflight?.object_storage ? 'validated' : 'warning',
+                        caption: preflight?.object_storage
+                            ? undefined
+                            : 'Some features will not work without object storage',
+                    })
+                }
+
+                return preflightItems as PreflightItemInterface[]
             },
         ],
         checksSummary: [

frontend/src/types.ts

@@ -1390,6 +1390,7 @@ export interface PreflightStatus {
     licensed_users_available?: number | null
     site_url?: string
     instance_preferences?: InstancePreferencesInterface
+    object_storage: boolean
 }
 
 export enum ItemMode { // todo: consolidate this and dashboardmode

package.json

@@ -52,9 +52,9 @@
         "arm64:ch-dev:start": "concurrently -n DOCKER,ESBUILD,TYPEGEN -c red,blue,green \"docker-compose -f docker-compose.arm64.yml pull && CH_WEB_SCRIPT=./ee/bin/docker-ch-dev-backend docker-compose -f docker-compose.arm64.yml up\" \"yarn run start-http --host 0.0.0.0\" \"yarn run typegen:watch\"",
         "arm64:ch-dev:clear": "docker compose -f docker-compose.arm64.yml stop && docker compose -f docker-compose.arm64.yml rm -v && docker compose -f docker-compose.arm64.yml down",
         "arm64:services": "yarn arm64:services:stop && yarn arm64:services:clean && yarn arm64:services:start",
-        "arm64:services:start": "docker-compose -f docker-compose.arm64.yml up zookeeper kafka clickhouse",
+        "arm64:services:start": "docker-compose -f docker-compose.arm64.yml up zookeeper kafka clickhouse object_storage",
         "arm64:services:stop": "docker-compose -f docker-compose.arm64.yml down",
-        "arm64:services:clean": "docker-compose -f docker-compose.arm64.yml rm -v zookeeper kafka clickhouse",
+        "arm64:services:clean": "docker-compose -f docker-compose.arm64.yml rm -v zookeeper kafka clickhouse object_storage",
         "dev:migrate:postgres": "export DEBUG=1 && source env/bin/activate && python manage.py migrate",
         "dev:migrate:clickhouse": "export DEBUG=1 && source env/bin/activate && python manage.py migrate_clickhouse",
         "prepare": "husky install"

plugin-server/package.json

@@ -33,9 +33,9 @@
         "prepublishOnly": "yarn build",
         "setup:dev:clickhouse": "cd .. && DEBUG=1 python manage.py migrate_clickhouse",
         "setup:test": "cd .. && TEST=1 python manage.py setup_test_environment",
-        "services:start": "cd .. && docker-compose -f docker-compose.dev.yml up zookeeper kafka clickhouse",
+        "services:start": "cd .. && docker-compose -f docker-compose.dev.yml up zookeeper kafka clickhouse object_storage",
         "services:stop": "cd .. && docker-compose -f docker-compose.dev.yml down",
-        "services:clean": "cd .. && docker-compose -f docker-compose.dev.yml rm -v zookeeper kafka clickhouse",
+        "services:clean": "cd .. && docker-compose -f docker-compose.dev.yml rm -v zookeeper kafka clickhouse object_storage",
         "services": "yarn services:stop && yarn services:clean && yarn services:start"
     },
     "bin": {