Merge branch 'master' of github.com:PostHog/posthog into decide-analy…

…tics4

frontend/src/mocks/utils.ts

@@ -16,12 +16,25 @@ export const mocksToHandlers = (mocks: Mocks): ReturnType<(typeof rest)['get']>[
             const pathWithoutTrailingSlash = path.replace(/\/$/, '')
             response.push(
                 (rest[method] as (typeof rest)['get'])(pathWithoutTrailingSlash, async (req, res, ctx) => {
+                    // We currently support a few ways to specify a mock response:
+                    // 1. A function that returns a tuple of [status, body]
+                    // 2. A function that returns a tuple of [status]
+                    // 3. A function that returns undefined. This represents that a network error has occured
+                    // 4. A function that returns an MSW response
+                    // 5. A JSON serializable object that will be returned as the response body
                     if (typeof handler === 'function') {
-                        const responseArray = await handler(req, res, ctx)
-                        if (responseArray.length === 2 && typeof responseArray[0] === 'number') {
-                            return res(ctx.status(responseArray[0]), ctx.json(responseArray[1] ?? null))
+                        const response = await handler(req, res, ctx)
+                        if (Array.isArray(response)) {
+                            const responseArray = response
+                            if (responseArray.length === 2 && typeof responseArray[0] === 'number') {
+                                return res(ctx.status(responseArray[0]), ctx.json(responseArray[1] ?? null))
+                            }
+                            return res(...responseArray)
+                        } else if (!response) {
+                            return res()
+                        } else {
+                            return response
                         }
-                        return res(...responseArray)
                     } else {
                         return res(ctx.json(handler ?? null))
                     }

posthog/api/test/batch_exports/test_backfill.py

@@ -29,7 +29,7 @@ def test_batch_export_backfill(client: HttpClient):
         "config": {
             "bucket_name": "my-production-s3-bucket",
             "region": "us-east-1",
-            "key_template": "posthog-events/{table_name}.csv",
+            "prefix": "posthog-events/",
             "batch_window_size": 3600,
             "aws_access_key_id": "abc123",
             "aws_secret_access_key": "secret",
@@ -64,7 +64,7 @@ def test_cannot_trigger_backfill_for_another_organization(client: HttpClient):
         "config": {
             "bucket_name": "my-production-s3-bucket",
             "region": "us-east-1",
-            "key_template": "posthog-events/{table_name}.csv",
+            "prefix": "posthog-events/",
             "batch_window_size": 3600,
             "aws_access_key_id": "abc123",
             "aws_secret_access_key": "secret",
@@ -104,7 +104,7 @@ def test_backfill_is_partitioned_by_team_id(client: HttpClient):
         "config": {
             "bucket_name": "my-production-s3-bucket",
             "region": "us-east-1",
-            "key_template": "posthog-events/{table_name}.csv",
+            "prefix": "posthog-events/",
             "batch_window_size": 3600,
             "aws_access_key_id": "abc123",
             "aws_secret_access_key": "secret",

posthog/api/test/batch_exports/test_create.py

@@ -31,7 +31,7 @@ def test_create_batch_export_with_interval_schedule(client: HttpClient):
         "config": {
             "bucket_name": "my-production-s3-bucket",
             "region": "us-east-1",
-            "key_template": "posthog-events/{table_name}.csv",
+            "prefix": "posthog-events/",
             "batch_window_size": 3600,
             "aws_access_key_id": "abc123",
             "aws_secret_access_key": "secret",
@@ -81,7 +81,7 @@ def test_cannot_create_a_batch_export_for_another_organization(client: HttpClien
         "config": {
             "bucket_name": "my-production-s3-bucket",
             "region": "us-east-1",
-            "key_template": "posthog-events/{table_name}.csv",
+            "prefix": "posthog-events/",
             "batch_window_size": 3600,
             "aws_access_key_id": "abc123",
             "aws_secret_access_key": "secret",

posthog/api/test/batch_exports/test_delete.py

@@ -35,7 +35,7 @@ def test_delete_batch_export(client: HttpClient):
         "config": {
             "bucket_name": "my-production-s3-bucket",
             "region": "us-east-1",
-            "key_template": "posthog-events/{table_name}.csv",
+            "prefix": "posthog-events/",
             "batch_window_size": 3600,
             "aws_access_key_id": "abc123",
             "aws_secret_access_key": "secret",
@@ -73,7 +73,7 @@ def test_cannot_delete_export_of_other_organizations(client: HttpClient):
         "config": {
             "bucket_name": "my-production-s3-bucket",
             "region": "us-east-1",
-            "key_template": "posthog-events/{table_name}.csv",
+            "prefix": "posthog-events/",
             "batch_window_size": 3600,
             "aws_access_key_id": "abc123",
             "aws_secret_access_key": "secret",
@@ -116,7 +116,7 @@ def test_deletes_are_partitioned_by_team_id(client: HttpClient):
         "config": {
             "bucket_name": "my-production-s3-bucket",
             "region": "us-east-1",
-            "key_template": "posthog-events/{table_name}.csv",
+            "prefix": "posthog-events/",
             "batch_window_size": 3600,
             "aws_access_key_id": "abc123",
             "aws_secret_access_key": "secret",

posthog/api/test/batch_exports/test_get.py

@@ -24,7 +24,7 @@ def test_can_get_exports_for_your_organizations(client: HttpClient):
         "config": {
             "bucket_name": "my-production-s3-bucket",
             "region": "us-east-1",
-            "key_template": "posthog-events/{table_name}.csv",
+            "prefix": "posthog-events/",
             "batch_window_size": 3600,
             "aws_access_key_id": "abc123",
             "aws_secret_access_key": "secret",
@@ -58,7 +58,7 @@ def test_can_get_exports_for_your_organizations(client: HttpClient):
         assert batch_export["destination"]["config"] == {
             "bucket_name": "my-production-s3-bucket",
             "region": "us-east-1",
-            "key_template": "posthog-events/{table_name}.csv",
+            "prefix": "posthog-events/",
             "batch_window_size": 3600,
         }
 
@@ -71,7 +71,7 @@ def test_cannot_get_exports_for_other_organizations(client: HttpClient):
         "config": {
             "bucket_name": "my-production-s3-bucket",
             "region": "us-east-1",
-            "key_template": "posthog-events/{table_name}.csv",
+            "prefix": "posthog-events/",
             "batch_window_size": 3600,
             "aws_access_key_id": "abc123",
             "aws_secret_access_key": "secret",
@@ -116,7 +116,7 @@ def test_batch_exports_are_partitioned_by_team(client: HttpClient):
         "config": {
             "bucket_name": "my-production-s3-bucket",
             "region": "us-east-1",
-            "key_template": "posthog-events/{table_name}.csv",
+            "prefix": "posthog-events/",
             "batch_window_size": 3600,
             "aws_access_key_id": "abc123",
             "aws_secret_access_key": "secret",

posthog/api/test/batch_exports/test_list.py

@@ -30,7 +30,7 @@ def test_list_batch_exports(client: HttpClient):
         "config": {
             "bucket_name": "my-production-s3-bucket",
             "region": "us-east-1",
-            "key_template": "posthog-events/{table_name}.csv",
+            "prefix": "posthog-events/",
             "batch_window_size": 3600,
             "aws_access_key_id": "abc123",
             "aws_secret_access_key": "secret",
@@ -76,7 +76,7 @@ def test_cannot_list_batch_exports_for_other_organizations(client: HttpClient):
         "config": {
             "bucket_name": "my-production-s3-bucket",
             "region": "us-east-1",
-            "key_template": "posthog-events/{table_name}.csv",
+            "prefix": "posthog-events/",
             "batch_window_size": 3600,
             "aws_access_key_id": "abc123",
             "aws_secret_access_key": "secret",
@@ -116,7 +116,7 @@ def test_list_is_partitioned_by_team(client: HttpClient):
         "config": {
             "bucket_name": "my-production-s3-bucket",
             "region": "us-east-1",
-            "key_template": "posthog-events/{table_name}.csv",
+            "prefix": "posthog-events/",
             "batch_window_size": 3600,
             "aws_access_key_id": "abc123",
             "aws_secret_access_key": "secret",

posthog/api/test/batch_exports/test_pause.py

@@ -35,7 +35,7 @@ def test_pause_and_unpause_batch_export(client: HttpClient):
         "config": {
             "bucket_name": "my-production-s3-bucket",
             "region": "us-east-1",
-            "key_template": "posthog-events/{table_name}.csv",
+            "prefix": "posthog-events/",
             "batch_window_size": 3600,
             "aws_access_key_id": "abc123",
             "aws_secret_access_key": "secret",
@@ -90,7 +90,7 @@ def test_connot_pause_and_unpause_batch_exports_of_other_organizations(client: H
         "config": {
             "bucket_name": "my-production-s3-bucket",
             "region": "us-east-1",
-            "key_template": "posthog-events/{table_name}.csv",
+            "prefix": "posthog-events/",
             "batch_window_size": 3600,
             "aws_access_key_id": "abc123",
             "aws_secret_access_key": "secret",
@@ -157,7 +157,7 @@ def test_pause_and_unpause_are_partitioned_by_team_id(client: HttpClient):
         "config": {
             "bucket_name": "my-production-s3-bucket",
             "region": "us-east-1",
-            "key_template": "posthog-events/{table_name}.csv",
+            "prefix": "posthog-events/",
             "batch_window_size": 3600,
             "aws_access_key_id": "abc123",
             "aws_secret_access_key": "secret",

posthog/api/test/batch_exports/test_runs.py

@@ -30,7 +30,7 @@ def test_can_get_export_runs_for_your_organizations(client: HttpClient):
         "config": {
             "bucket_name": "my-production-s3-bucket",
             "region": "us-east-1",
-            "key_template": "posthog-events/{table_name}.csv",
+            "prefix": "posthog-events/",
             "batch_window_size": 3600,
             "aws_access_key_id": "abc123",
             "aws_secret_access_key": "secret",
@@ -67,7 +67,7 @@ def test_cannot_get_exports_for_other_organizations(client: HttpClient):
         "config": {
             "bucket_name": "my-production-s3-bucket",
             "region": "us-east-1",
-            "key_template": "posthog-events/{table_name}.csv",
+            "prefix": "posthog-events/",
             "batch_window_size": 3600,
             "aws_access_key_id": "abc123",
             "aws_secret_access_key": "secret",
@@ -112,7 +112,7 @@ def test_batch_exports_are_partitioned_by_team(client: HttpClient):
         "config": {
             "bucket_name": "my-production-s3-bucket",
             "region": "us-east-1",
-            "key_template": "posthog-events/{table_name}.csv",
+            "prefix": "posthog-events/",
             "batch_window_size": 3600,
             "aws_access_key_id": "abc123",
             "aws_secret_access_key": "secret",
@@ -160,7 +160,7 @@ def test_batch_export_backfill_creates_a_run(client: HttpClient):
         "config": {
             "bucket_name": "my-production-s3-bucket",
             "region": "us-east-1",
-            "key_template": "posthog-events/{table_name}.csv",
+            "prefix": "posthog-events/",
             "batch_window_size": 3600,
             "aws_access_key_id": "abc123",
             "aws_secret_access_key": "secret",

posthog/api/test/batch_exports/test_update.py

@@ -29,7 +29,7 @@ def test_can_put_config(client: HttpClient):
         "config": {
             "bucket_name": "my-production-s3-bucket",
             "region": "us-east-1",
-            "key_template": "posthog-events/{table_name}.csv",
+            "prefix": "posthog-events/",
             "batch_window_size": 3600,
             "aws_access_key_id": "abc123",
             "aws_secret_access_key": "secret",
@@ -86,7 +86,7 @@ def test_can_patch_config(client: HttpClient):
         "config": {
             "bucket_name": "my-production-s3-bucket",
             "region": "us-east-1",
-            "key_template": "posthog-events/{table_name}.csv",
+            "prefix": "posthog-events/",
             "batch_window_size": 3600,
             "aws_access_key_id": "abc123",
             "aws_secret_access_key": "secret",
@@ -119,7 +119,7 @@ def test_can_patch_config(client: HttpClient):
         "config": {
             "bucket_name": "my-production-s3-bucket",
             "region": "us-east-1",
-            "key_template": "posthog-events/{table_name}.csv",
+            "prefix": "posthog-events/",
         },
     }
 

posthog/batch_exports/service.py

@@ -13,6 +13,7 @@
     Client,
     Schedule,
     ScheduleActionStartWorkflow,
+    ScheduleIntervalSpec,
     ScheduleSpec,
     ScheduleState,
 )
@@ -42,7 +43,7 @@ class S3BatchExportInputs:
 
     bucket_name: str
     region: str
-    key_template: str
+    prefix: str
     batch_window_size: int
     team_id: int
     batch_export_id: str
@@ -60,11 +61,12 @@ class S3BatchExportInputs:
 
 
 @async_to_sync
-async def create_schedule(temporal, id: str, schedule: Schedule):
+async def create_schedule(temporal, id: str, schedule: Schedule, trigger_immediately: bool = False):
     """Create a Temporal Schedule."""
     return await temporal.create_schedule(
         id=id,
         schedule=schedule,
+        trigger_immediately=trigger_immediately,
     )
 
 
@@ -199,6 +201,8 @@ def create_batch_export(team_id: int, interval: str, name: str, destination_data
 
     temporal = sync_connect()
 
+    time_delta_from_interval = dt.timedelta(hours=1) if interval == "hour" else dt.timedelta(days=1)
+
     create_schedule(
         temporal,
         id=str(batch_export.id),
@@ -217,9 +221,12 @@ def create_batch_export(team_id: int, interval: str, name: str, destination_data
                 id=str(batch_export.id),
                 task_queue=settings.TEMPORAL_TASK_QUEUE,
             ),
-            spec=ScheduleSpec(),
+            spec=ScheduleSpec(
+                intervals=[ScheduleIntervalSpec(every=time_delta_from_interval)],
+            ),
             state=state,
         ),
+        trigger_immediately=True,
     )
 
     return batch_export

posthog/temporal/tests/batch_exports/test_run_updates.py

@@ -48,7 +48,7 @@ def destination(team):
         config={
             "bucket_name": "bucket",
             "region": "us-east-1",
-            "key_template": "events.csv",
+            "prefix": "posthog-events/",
             "batch_window_size": 3600,
             "aws_access_key_id": "key_id",
             "aws_secret_access_key": "secret",

posthog/temporal/tests/batch_exports/test_s3_batch_export_workflow.py

@@ -162,7 +162,7 @@ async def test_insert_into_s3_activity_puts_data_into_s3(activity_environment):
     insert_inputs = S3InsertInputs(
         bucket_name=bucket_name,
         region="us-east-1",
-        key_template=f"{prefix}",
+        prefix=prefix,
         team_id=team_id,
         data_interval_start=data_interval_start,
         data_interval_end=data_interval_end,
@@ -229,7 +229,7 @@ async def test_s3_export_workflow_with_minio_bucket(client: HttpClient):
         "config": {
             "bucket_name": "my-production-s3-bucket",
             "region": "us-east-1",
-            "key_template": "posthog-events/{table_name}.csv",
+            "prefix": "posthog-events/",
             "batch_window_size": 3600,
             "aws_access_key_id": "abc123",
             "aws_secret_access_key": "secret",

posthog/temporal/workflows/s3_batch_export.py

@@ -52,7 +52,7 @@ class S3InsertInputs:
 
     bucket_name: str
     region: str
-    key_template: str
+    prefix: str
     team_id: int
     data_interval_start: str
     data_interval_end: str
@@ -133,14 +133,12 @@ async def insert_into_s3_activity(inputs: S3InsertInputs):
 
         activity.logger.info("BatchExporting %s rows to S3", count)
 
-        template_vars = prepare_template_vars(inputs)
-
         query_template = Template(SELECT_QUERY_TEMPLATE.template)
 
         activity.logger.debug(query_template.template)
 
         # Create a multipart upload to S3
-        key = inputs.key_template.format(**template_vars)
+        key = f"{inputs.prefix}/{inputs.data_interval_start}-{inputs.data_interval_end}.jsonl"
         s3_client = boto3.client(
             "s3",
             region_name=inputs.region,
@@ -273,7 +271,7 @@ async def run(self, inputs: S3BatchExportInputs):
         insert_inputs = S3InsertInputs(
             bucket_name=inputs.bucket_name,
             region=inputs.region,
-            key_template=inputs.key_template,
+            prefix=inputs.prefix,
             team_id=inputs.team_id,
             aws_access_key_id=inputs.aws_access_key_id,
             aws_secret_access_key=inputs.aws_secret_access_key,