From d3e6abece019f2064fd0ef3dc7aac16cd3942c7b Mon Sep 17 00:00:00 2001
From: Marius Andra <marius.andra@gmail.com>
Date: Thu, 25 Feb 2021 14:44:02 +0100
Subject: [PATCH 01/10] remove teamless plugins

---
 src/plugins.ts | 28 ++++++++++------------------
 1 file changed, 10 insertions(+), 18 deletions(-)

diff --git a/src/plugins.ts b/src/plugins.ts
index 3e9e2f21..f0dc173a 100644
--- a/src/plugins.ts
+++ b/src/plugins.ts
@@ -40,7 +40,6 @@ export async function setupPlugins(server: PluginsServer): Promise<void> {
     const pluginConfigRows = await getPluginConfigRows(server)
     const foundPluginConfigs = new Map<number, boolean>()
     server.pluginConfigsPerTeam.clear()
-    server.defaultConfigs = []
     for (const row of pluginConfigRows) {
         const plugin = server.plugins.get(row.plugin_id)
         if (!plugin) {
@@ -56,15 +55,16 @@ export async function setupPlugins(server: PluginsServer): Promise<void> {
         server.pluginConfigs.set(row.id, pluginConfig)
 
         if (!row.team_id) {
-            server.defaultConfigs.push(row)
-        } else {
-            let teamConfigs = server.pluginConfigsPerTeam.get(row.team_id)
-            if (!teamConfigs) {
-                teamConfigs = []
-                server.pluginConfigsPerTeam.set(row.team_id, teamConfigs)
-            }
-            teamConfigs.push(pluginConfig)
+            console.error(`🔴 PluginConfig(id=${row.id}) without team_id!`)
+            continue
         }
+
+        let teamConfigs = server.pluginConfigsPerTeam.get(row.team_id)
+        if (!teamConfigs) {
+            teamConfigs = []
+            server.pluginConfigsPerTeam.set(row.team_id, teamConfigs)
+        }
+        teamConfigs.push(pluginConfig)
     }
     for (const [id, pluginConfig] of server.pluginConfigs) {
         if (!foundPluginConfigs.has(id)) {
@@ -86,15 +86,7 @@ export async function setupPlugins(server: PluginsServer): Promise<void> {
 
     const sortFunction = (a: PluginConfig, b: PluginConfig) => a.order - b.order
     for (const teamId of server.pluginConfigsPerTeam.keys()) {
-        if (server.defaultConfigs.length > 0) {
-            const combinedPluginConfigs = [
-                ...(server.pluginConfigsPerTeam.get(teamId) || []),
-                ...server.defaultConfigs,
-            ].sort(sortFunction)
-            server.pluginConfigsPerTeam.set(teamId, combinedPluginConfigs)
-        } else {
-            server.pluginConfigsPerTeam.get(teamId)?.sort(sortFunction)
-        }
+        server.pluginConfigsPerTeam.get(teamId)?.sort(sortFunction)
     }
 }
 

From 18244bd21bcca2cf5932ad366e3407d375dda42b Mon Sep 17 00:00:00 2001
From: Marius Andra <marius.andra@gmail.com>
Date: Thu, 25 Feb 2021 22:36:20 +0100
Subject: [PATCH 02/10] limit start:dev to 4 workers

---
 package.json | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/package.json b/package.json
index 932a397a..0a36c43b 100644
--- a/package.json
+++ b/package.json
@@ -16,8 +16,8 @@
         "benchmark:vm:worker": "node --expose-gc node_modules/.bin/jest --runInBand benchmarks/vm/worker.benchmark.ts",
         "start": "yarn start:dev",
         "start:dist": "node dist/index.js --base-dir ../posthog",
-        "start:dev": "ts-node-dev --exit-child src/index.ts --base-dir ../posthog",
-        "start:dev:ee": "KAFKA_ENABLED=true KAFKA_HOSTS=localhost:9092 yarn start:dev",
+        "start:dev": "WORKER_CONCURRENCY=4 ts-node-dev --exit-child src/index.ts --base-dir ../posthog",
+        "start:dev:ee": "WORKER_CONCURRENCY=4 KAFKA_ENABLED=true KAFKA_HOSTS=localhost:9092 yarn start:dev",
         "build": "yarn clean && yarn compile",
         "clean": "rimraf dist/*",
         "compile:protobuf": "cd src/idl/ && rimraf protos.* && pbjs -t static-module -w commonjs -o protos.js *.proto && pbts -o protos.d.ts protos.js && eslint --fix . && prettier --write .",

From 41a39c78f1621ee68cc525e0c39c16934d680507 Mon Sep 17 00:00:00 2001
From: Marius Andra <marius.andra@gmail.com>
Date: Thu, 25 Feb 2021 22:51:00 +0100
Subject: [PATCH 03/10] remove support for lib.js

---
 src/plugins.ts | 14 ++------------
 src/vm/vm.ts   |  6 ++----
 2 files changed, 4 insertions(+), 16 deletions(-)

diff --git a/src/plugins.ts b/src/plugins.ts
index f0dc173a..3e5d0d61 100644
--- a/src/plugins.ts
+++ b/src/plugins.ts
@@ -129,14 +129,8 @@ async function loadPlugin(server: PluginsServer, pluginConfig: PluginConfig): Pr
             const jsPath = path.resolve(pluginPath, config['main'] || 'index.js')
             const indexJs = fs.readFileSync(jsPath).toString()
 
-            const libPath = path.resolve(pluginPath, config['lib'] || 'lib.js')
-            const libJs = fs.existsSync(libPath) ? fs.readFileSync(libPath).toString() : ''
-            if (libJs) {
-                console.warn(`⚠️ Using "lib.js" is deprecated! Used by: ${plugin.name} (${plugin.url})`)
-            }
-
             try {
-                pluginConfig.vm = await createPluginConfigVM(server, pluginConfig, indexJs, libJs)
+                pluginConfig.vm = await createPluginConfigVM(server, pluginConfig, indexJs)
                 status.info('🔌', `Loaded local plugin "${plugin.name}" from "${pluginPath}"!`)
                 await clearError(server, pluginConfig)
                 return true
@@ -157,14 +151,10 @@ async function loadPlugin(server: PluginsServer, pluginConfig: PluginConfig): Pr
             }
 
             const indexJs = await getFileFromArchive(archive, config['main'] || 'index.js')
-            const libJs = await getFileFromArchive(archive, config['lib'] || 'lib.js')
-            if (libJs) {
-                console.warn(`⚠️ Using "lib.js" is deprecated! Used by: ${plugin.name} (${plugin.url})`)
-            }
 
             if (indexJs) {
                 try {
-                    pluginConfig.vm = await createPluginConfigVM(server, pluginConfig, indexJs, libJs || '')
+                    pluginConfig.vm = await createPluginConfigVM(server, pluginConfig, indexJs)
                     status.info('🔌', `Loaded plugin "${plugin.name}"!`)
                     await clearError(server, pluginConfig)
                     return true
diff --git a/src/vm/vm.ts b/src/vm/vm.ts
index d6efd00b..c51283b5 100644
--- a/src/vm/vm.ts
+++ b/src/vm/vm.ts
@@ -13,11 +13,9 @@ import { secureCode } from './transforms'
 export async function createPluginConfigVM(
     server: PluginsServer,
     pluginConfig: PluginConfig, // NB! might have team_id = 0
-    indexJs: string,
-    libJs = ''
+    indexJs: string
 ): Promise<PluginConfigVMReponse> {
-    const rawCode = libJs ? `${libJs};${indexJs}` : indexJs
-    const securedCode = secureCode(rawCode, server)
+    const securedCode = secureCode(indexJs, server)
 
     // Create virtual machine
     const vm = new VM({

From 6476f3ff3c2a835431b5514de77b90f324ecc8aa Mon Sep 17 00:00:00 2001
From: Marius Andra <marius.andra@gmail.com>
Date: Thu, 25 Feb 2021 22:52:47 +0100
Subject: [PATCH 04/10] remove lib.js test

---
 tests/postgres/vm.test.ts | 25 +------------------------
 1 file changed, 1 insertion(+), 24 deletions(-)

diff --git a/tests/postgres/vm.test.ts b/tests/postgres/vm.test.ts
index 91f00d41..96d8c98d 100644
--- a/tests/postgres/vm.test.ts
+++ b/tests/postgres/vm.test.ts
@@ -35,8 +35,7 @@ afterEach(async () => {
 
 test('empty plugins', async () => {
     const indexJs = ''
-    const libJs = ''
-    const vm = await createPluginConfigVM(mockServer, pluginConfig39, indexJs, libJs)
+    const vm = await createPluginConfigVM(mockServer, pluginConfig39, indexJs)
 
     expect(Object.keys(vm).sort()).toEqual(['methods', 'tasks', 'vm'])
     expect(Object.keys(vm.methods).sort()).toEqual(['processEvent', 'processEventBatch'])
@@ -554,28 +553,6 @@ test('meta.cache incr', async () => {
     expect(event.properties!['counter']).toEqual(3)
 })
 
-test('lib.js (deprecated)', async () => {
-    const indexJs = `
-        async function processEvent (event, meta) {
-            event.event = libraryFunction(event.event)
-            return event
-        }
-    `
-    const libJs = `
-        function libraryFunction (string) {
-            return string.split("").reverse().join("")
-        }
-    `
-    const vm = await createPluginConfigVM(mockServer, pluginConfig39, indexJs, libJs)
-    const event: PluginEvent = {
-        ...defaultEvent,
-        event: 'original event',
-    }
-    await vm.methods.processEvent(event)
-
-    expect(event.event).toEqual('tneve lanigiro')
-})
-
 test('console.log', async () => {
     console.log = jest.fn()
     console.error = jest.fn()

From 8e49135de0a63341179a185ec0846833bbec0e5d Mon Sep 17 00:00:00 2001
From: Marius Andra <marius.andra@gmail.com>
Date: Thu, 25 Feb 2021 23:19:13 +0100
Subject: [PATCH 05/10] add missing script

---
 package.json | 1 +
 1 file changed, 1 insertion(+)

diff --git a/package.json b/package.json
index 0a36c43b..46936a89 100644
--- a/package.json
+++ b/package.json
@@ -7,6 +7,7 @@
     "scripts": {
         "test": "jest --runInBand tests/**/*.test.ts",
         "test:postgres": "jest --runInBand tests/postgres/*.test.ts tests/*.test.ts",
+        "test:clickhouse": "yarn test:clickhouse:1 && yarn test:clickhouse:2",
         "test:clickhouse:1": "jest --runInBand tests/clickhouse/postgres-parity.test.ts tests/clickhouse/e2e.test.ts tests/clickhouse/ingestion-utils.test.ts",
         "test:clickhouse:2": "jest --runInBand tests/clickhouse/process-event.test.ts",
         "benchmark": "yarn run benchmarks:clickhouse && yarn run benchmark:postgres && yarn run benchmarks:vm",

From fdcb95365f77b91e342fcfe29a35af9d5ecf6615 Mon Sep 17 00:00:00 2001
From: Marius Andra <marius.andra@gmail.com>
Date: Thu, 25 Feb 2021 23:19:55 +0100
Subject: [PATCH 06/10] lazy processEvent functions

---
 src/plugins.ts |  6 +++---
 src/types.ts   | 13 +++++++++++--
 src/vm/vm.ts   | 23 ++++++++++++++++++++---
 3 files changed, 34 insertions(+), 8 deletions(-)

diff --git a/src/plugins.ts b/src/plugins.ts
index 3e5d0d61..af71af9b 100644
--- a/src/plugins.ts
+++ b/src/plugins.ts
@@ -130,7 +130,7 @@ async function loadPlugin(server: PluginsServer, pluginConfig: PluginConfig): Pr
             const indexJs = fs.readFileSync(jsPath).toString()
 
             try {
-                pluginConfig.vm = await createPluginConfigVM(server, pluginConfig, indexJs)
+                pluginConfig.vm = createPluginConfigVM(server, pluginConfig, indexJs)
                 status.info('🔌', `Loaded local plugin "${plugin.name}" from "${pluginPath}"!`)
                 await clearError(server, pluginConfig)
                 return true
@@ -154,7 +154,7 @@ async function loadPlugin(server: PluginsServer, pluginConfig: PluginConfig): Pr
 
             if (indexJs) {
                 try {
-                    pluginConfig.vm = await createPluginConfigVM(server, pluginConfig, indexJs)
+                    pluginConfig.vm = createPluginConfigVM(server, pluginConfig, indexJs)
                     status.info('🔌', `Loaded plugin "${plugin.name}"!`)
                     await clearError(server, pluginConfig)
                     return true
@@ -166,7 +166,7 @@ async function loadPlugin(server: PluginsServer, pluginConfig: PluginConfig): Pr
             }
         } else if (plugin.plugin_type === 'source' && plugin.source) {
             try {
-                pluginConfig.vm = await createPluginConfigVM(server, pluginConfig, plugin.source)
+                pluginConfig.vm = createPluginConfigVM(server, pluginConfig, plugin.source)
                 status.info('🔌', `Loaded plugin "${plugin.name}"!`)
                 await clearError(server, pluginConfig)
                 return true
diff --git a/src/types.ts b/src/types.ts
index 4e66541c..e88bb12d 100644
--- a/src/types.ts
+++ b/src/types.ts
@@ -119,7 +119,7 @@ export interface PluginConfig {
     config: Record<string, unknown>
     error?: PluginError
     attachments?: Record<string, PluginAttachment>
-    vm?: PluginConfigVMReponse | null
+    vm?: PluginConfigLazyVMResponse | null
 }
 
 export interface PluginJsonConfig {
@@ -156,7 +156,16 @@ export interface PluginTask {
     exec: () => Promise<any>
 }
 
-export interface PluginConfigVMReponse {
+export interface PluginConfigLazyVMResponse {
+    vmPromise: Promise<VM>
+    methods: {
+        processEvent: (event: PluginEvent) => Promise<PluginEvent>
+        processEventBatch: (batch: PluginEvent[]) => Promise<PluginEvent[]>
+    }
+    tasks: Record<string, PluginTask>
+}
+
+export interface PluginConfigVMResponse {
     vm: VM
     methods: {
         processEvent: (event: PluginEvent) => Promise<PluginEvent>
diff --git a/src/vm/vm.ts b/src/vm/vm.ts
index c51283b5..4dcb4102 100644
--- a/src/vm/vm.ts
+++ b/src/vm/vm.ts
@@ -2,7 +2,7 @@ import { randomBytes } from 'crypto'
 import fetch from 'node-fetch'
 import { VM } from 'vm2'
 
-import { PluginConfig, PluginConfigVMReponse, PluginsServer } from '../types'
+import { PluginConfig, PluginConfigLazyVMResponse, PluginConfigVMResponse, PluginsServer } from '../types'
 import { createCache } from './extensions/cache'
 import { createConsole } from './extensions/console'
 import { createGoogle } from './extensions/google'
@@ -10,11 +10,28 @@ import { createPosthog } from './extensions/posthog'
 import { createStorage } from './extensions/storage'
 import { secureCode } from './transforms'
 
-export async function createPluginConfigVM(
+export function createPluginConfigVM(
     server: PluginsServer,
     pluginConfig: PluginConfig, // NB! might have team_id = 0
     indexJs: string
-): Promise<PluginConfigVMReponse> {
+): PluginConfigLazyVMResponse {
+    const createVmPromise = createActualVM(server, pluginConfig, indexJs)
+
+    return {
+        vmPromise: createVmPromise.then((vm) => vm.vm),
+        methods: {
+            processEvent: (event) => createVmPromise.then((vm) => vm.methods.processEvent(event)),
+            processEventBatch: (batch) => createVmPromise.then((vm) => vm.methods.processEventBatch(batch)),
+        },
+        tasks: {}, // new Proxy(),
+    }
+}
+
+export async function createActualVM(
+    server: PluginsServer,
+    pluginConfig: PluginConfig, // NB! might have team_id = 0
+    indexJs: string
+): Promise<PluginConfigVMResponse> {
     const securedCode = secureCode(indexJs, server)
 
     // Create virtual machine

From 02d9c25f095a4c6c5e0aa598907c02415248cd99 Mon Sep 17 00:00:00 2001
From: Marius Andra <marius.andra@gmail.com>
Date: Thu, 25 Feb 2021 23:27:33 +0100
Subject: [PATCH 07/10] clean up tests, fix empty plugins test

---
 src/vm/vm.ts              |  4 +--
 tests/postgres/vm.test.ts | 60 +++++++++++++++++++--------------------
 2 files changed, 32 insertions(+), 32 deletions(-)

diff --git a/src/vm/vm.ts b/src/vm/vm.ts
index 4dcb4102..95afe2ee 100644
--- a/src/vm/vm.ts
+++ b/src/vm/vm.ts
@@ -146,8 +146,8 @@ export async function createActualVM(
 
             // export various functions
             const __methods = {
-                processEvent: __asyncFunctionGuard(__bindMeta('processEvent')),
-                processEventBatch: __asyncFunctionGuard(__bindMeta('processEventBatch'))
+                processEvent: __asyncFunctionGuard(__bindMeta('processEvent')) || ((event) => event),
+                processEventBatch: __asyncFunctionGuard(__bindMeta('processEventBatch')) || ((batch) => batch)
             };
 
             // gather the runEveryX commands and export in __tasks
diff --git a/tests/postgres/vm.test.ts b/tests/postgres/vm.test.ts
index 96d8c98d..1c88d7be 100644
--- a/tests/postgres/vm.test.ts
+++ b/tests/postgres/vm.test.ts
@@ -35,12 +35,12 @@ afterEach(async () => {
 
 test('empty plugins', async () => {
     const indexJs = ''
-    const vm = await createPluginConfigVM(mockServer, pluginConfig39, indexJs)
+    const vm = createPluginConfigVM(mockServer, pluginConfig39, indexJs)
 
-    expect(Object.keys(vm).sort()).toEqual(['methods', 'tasks', 'vm'])
+    expect(Object.keys(vm).sort()).toEqual(['methods', 'tasks', 'vmPromise'])
     expect(Object.keys(vm.methods).sort()).toEqual(['processEvent', 'processEventBatch'])
-    expect(vm.methods.processEvent).toEqual(undefined)
-    expect(vm.methods.processEventBatch).toEqual(undefined)
+    expect(await vm.methods.processEvent(defaultEvent)).toEqual(defaultEvent)
+    expect(await vm.methods.processEventBatch([defaultEvent])).toEqual([defaultEvent])
 })
 
 test('setupPlugin sync', async () => {
@@ -54,7 +54,7 @@ test('setupPlugin sync', async () => {
         }
     `
     await resetTestDatabase(indexJs)
-    const vm = await createPluginConfigVM(mockServer, pluginConfig39, indexJs)
+    const vm = createPluginConfigVM(mockServer, pluginConfig39, indexJs)
     const newEvent = await vm.methods.processEvent({ ...defaultEvent })
     expect(newEvent.event).toEqual('haha')
 })
@@ -71,7 +71,7 @@ test('setupPlugin async', async () => {
         }
     `
     await resetTestDatabase(indexJs)
-    const vm = await createPluginConfigVM(mockServer, pluginConfig39, indexJs)
+    const vm = createPluginConfigVM(mockServer, pluginConfig39, indexJs)
     const newEvent = await vm.methods.processEvent({ ...defaultEvent })
     expect(newEvent.event).toEqual('haha')
 })
@@ -84,7 +84,7 @@ test('processEvent', async () => {
         }
     `
     await resetTestDatabase(indexJs)
-    const vm = await createPluginConfigVM(mockServer, pluginConfig39, indexJs)
+    const vm = createPluginConfigVM(mockServer, pluginConfig39, indexJs)
     expect(vm.methods.processEvent).not.toEqual(undefined)
     expect(vm.methods.processEventBatch).not.toEqual(undefined)
 
@@ -117,7 +117,7 @@ test('async processEvent', async () => {
         }
     `
     await resetTestDatabase(indexJs)
-    const vm = await createPluginConfigVM(mockServer, pluginConfig39, indexJs)
+    const vm = createPluginConfigVM(mockServer, pluginConfig39, indexJs)
     expect(vm.methods.processEvent).not.toEqual(undefined)
     expect(vm.methods.processEventBatch).not.toEqual(undefined)
 
@@ -152,7 +152,7 @@ test('processEventBatch', async () => {
         }
     `
     await resetTestDatabase(indexJs)
-    const vm = await createPluginConfigVM(mockServer, pluginConfig39, indexJs)
+    const vm = createPluginConfigVM(mockServer, pluginConfig39, indexJs)
     expect(vm.methods.processEvent).not.toEqual(undefined)
     expect(vm.methods.processEventBatch).not.toEqual(undefined)
 
@@ -187,7 +187,7 @@ test('async processEventBatch', async () => {
         }
     `
     await resetTestDatabase(indexJs)
-    const vm = await createPluginConfigVM(mockServer, pluginConfig39, indexJs)
+    const vm = createPluginConfigVM(mockServer, pluginConfig39, indexJs)
     expect(vm.methods.processEvent).not.toEqual(undefined)
     expect(vm.methods.processEventBatch).not.toEqual(undefined)
 
@@ -226,7 +226,7 @@ test('processEvent && processEventBatch', async () => {
         }
     `
     await resetTestDatabase(indexJs)
-    const vm = await createPluginConfigVM(mockServer, pluginConfig39, indexJs)
+    const vm = createPluginConfigVM(mockServer, pluginConfig39, indexJs)
     expect(vm.methods.processEvent).not.toEqual(undefined)
     expect(vm.methods.processEventBatch).not.toEqual(undefined)
 
@@ -258,7 +258,7 @@ test('processEvent without returning', async () => {
         }
     `
     await resetTestDatabase(indexJs)
-    const vm = await createPluginConfigVM(mockServer, pluginConfig39, indexJs)
+    const vm = createPluginConfigVM(mockServer, pluginConfig39, indexJs)
     expect(vm.methods.processEvent).not.toEqual(undefined)
 
     const event: PluginEvent = {
@@ -283,7 +283,7 @@ test('async processEvent', async () => {
         }
     `
     await resetTestDatabase(indexJs)
-    const vm = await createPluginConfigVM(mockServer, pluginConfig39, indexJs)
+    const vm = createPluginConfigVM(mockServer, pluginConfig39, indexJs)
 
     const event: PluginEvent = {
         ...defaultEvent,
@@ -303,7 +303,7 @@ test('module.exports override', async () => {
         module.exports = { processEvent: myProcessEventFunction }
     `
     await resetTestDatabase(indexJs)
-    const vm = await createPluginConfigVM(mockServer, pluginConfig39, indexJs)
+    const vm = createPluginConfigVM(mockServer, pluginConfig39, indexJs)
 
     const event: PluginEvent = {
         ...defaultEvent,
@@ -323,7 +323,7 @@ test('module.exports set', async () => {
         module.exports.processEvent = myProcessEventFunction
     `
     await resetTestDatabase(indexJs)
-    const vm = await createPluginConfigVM(mockServer, pluginConfig39, indexJs)
+    const vm = createPluginConfigVM(mockServer, pluginConfig39, indexJs)
 
     const event: PluginEvent = {
         ...defaultEvent,
@@ -343,7 +343,7 @@ test('exports override', async () => {
         exports = { processEvent: myProcessEventFunction }
     `
     await resetTestDatabase(indexJs)
-    const vm = await createPluginConfigVM(mockServer, pluginConfig39, indexJs)
+    const vm = createPluginConfigVM(mockServer, pluginConfig39, indexJs)
     const event: PluginEvent = {
         ...defaultEvent,
         event: 'original event',
@@ -362,7 +362,7 @@ test('exports set', async () => {
         exports.processEvent = myProcessEventFunction
     `
     await resetTestDatabase(indexJs)
-    const vm = await createPluginConfigVM(mockServer, pluginConfig39, indexJs)
+    const vm = createPluginConfigVM(mockServer, pluginConfig39, indexJs)
     const event: PluginEvent = {
         ...defaultEvent,
         event: 'original event',
@@ -380,7 +380,7 @@ test('meta.config', async () => {
         }
     `
     await resetTestDatabase(indexJs)
-    const vm = await createPluginConfigVM(mockServer, pluginConfig39, indexJs)
+    const vm = createPluginConfigVM(mockServer, pluginConfig39, indexJs)
     const event: PluginEvent = {
         ...defaultEvent,
         event: 'original event',
@@ -404,7 +404,7 @@ test('meta.cache set/get', async () => {
         }
     `
     await resetTestDatabase(indexJs)
-    const vm = await createPluginConfigVM(mockServer, pluginConfig39, indexJs)
+    const vm = createPluginConfigVM(mockServer, pluginConfig39, indexJs)
     const event: PluginEvent = {
         ...defaultEvent,
         event: 'original event',
@@ -442,7 +442,7 @@ test('meta.storage set/get', async () => {
         }
     `
     await resetTestDatabase(indexJs)
-    const vm = await createPluginConfigVM(mockServer, pluginConfig39, indexJs)
+    const vm = createPluginConfigVM(mockServer, pluginConfig39, indexJs)
     const event: PluginEvent = {
         ...defaultEvent,
         event: 'original event',
@@ -473,7 +473,7 @@ test('meta.cache expire', async () => {
         }
     `
     await resetTestDatabase(indexJs)
-    const vm = await createPluginConfigVM(mockServer, pluginConfig39, indexJs)
+    const vm = createPluginConfigVM(mockServer, pluginConfig39, indexJs)
     const event: PluginEvent = {
         ...defaultEvent,
         event: 'original event',
@@ -505,7 +505,7 @@ test('meta.cache set ttl', async () => {
         }
     `
     await resetTestDatabase(indexJs)
-    const vm = await createPluginConfigVM(mockServer, pluginConfig39, indexJs)
+    const vm = createPluginConfigVM(mockServer, pluginConfig39, indexJs)
     const event: PluginEvent = {
         ...defaultEvent,
         event: 'original event',
@@ -536,7 +536,7 @@ test('meta.cache incr', async () => {
         }
     `
     await resetTestDatabase(indexJs)
-    const vm = await createPluginConfigVM(mockServer, pluginConfig39, indexJs)
+    const vm = createPluginConfigVM(mockServer, pluginConfig39, indexJs)
     const event: PluginEvent = {
         ...defaultEvent,
         event: 'original event',
@@ -570,7 +570,7 @@ test('console.log', async () => {
         }
     `
     await resetTestDatabase(indexJs)
-    const vm = await createPluginConfigVM(mockServer, pluginConfig39, indexJs)
+    const vm = createPluginConfigVM(mockServer, pluginConfig39, indexJs)
     const event: PluginEvent = {
         ...defaultEvent,
         event: 'logged event',
@@ -593,7 +593,7 @@ test('fetch', async () => {
         }
     `
     await resetTestDatabase(indexJs)
-    const vm = await createPluginConfigVM(mockServer, pluginConfig39, indexJs)
+    const vm = createPluginConfigVM(mockServer, pluginConfig39, indexJs)
     const event: PluginEvent = {
         ...defaultEvent,
         event: 'fetched',
@@ -619,7 +619,7 @@ test('attachments', async () => {
             contents: Buffer.from('{"name": "plugin"}'),
         },
     }
-    const vm = await createPluginConfigVM(
+    const vm = createPluginConfigVM(
         mockServer,
         {
             ...pluginConfig39,
@@ -650,7 +650,7 @@ test('runEvery', async () => {
         }
     `
     await resetTestDatabase(indexJs)
-    const vm = await createPluginConfigVM(mockServer, pluginConfig39, indexJs)
+    const vm = createPluginConfigVM(mockServer, pluginConfig39, indexJs)
 
     expect(Object.keys(vm.tasks)).toEqual(['runEveryMinute', 'runEveryHour', 'runEveryDay'])
     expect(Object.values(vm.tasks).map((v) => v?.name)).toEqual(['runEveryMinute', 'runEveryHour', 'runEveryDay'])
@@ -667,7 +667,7 @@ test('runEvery must be a function', async () => {
         const runEveryDay = { some: 'object' }
     `
     await resetTestDatabase(indexJs)
-    const vm = await createPluginConfigVM(mockServer, pluginConfig39, indexJs)
+    const vm = createPluginConfigVM(mockServer, pluginConfig39, indexJs)
 
     expect(Object.keys(vm.tasks)).toEqual(['runEveryMinute'])
     expect(Object.values(vm.tasks).map((v) => v?.name)).toEqual(['runEveryMinute'])
@@ -683,7 +683,7 @@ test('posthog in runEvery', async () => {
         }
     `
     await resetTestDatabase(indexJs)
-    const vm = await createPluginConfigVM(mockServer, pluginConfig39, indexJs)
+    const vm = createPluginConfigVM(mockServer, pluginConfig39, indexJs)
 
     expect(Client).not.toHaveBeenCalled
 
@@ -722,7 +722,7 @@ test('posthog in runEvery with timestamp', async () => {
         }
     `
     await resetTestDatabase(indexJs)
-    const vm = await createPluginConfigVM(mockServer, pluginConfig39, indexJs)
+    const vm = createPluginConfigVM(mockServer, pluginConfig39, indexJs)
 
     expect(Client).not.toHaveBeenCalled
 

From e994c7e8c3c5e9e62730a48455e1e0b8581c5ce7 Mon Sep 17 00:00:00 2001
From: Marius Andra <marius.andra@gmail.com>
Date: Thu, 25 Feb 2021 23:29:12 +0100
Subject: [PATCH 08/10] improve memory benchmark vm code

---
 benchmarks/vm/memory.benchmark.ts | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/benchmarks/vm/memory.benchmark.ts b/benchmarks/vm/memory.benchmark.ts
index 40e7263e..a2b32f19 100644
--- a/benchmarks/vm/memory.benchmark.ts
+++ b/benchmarks/vm/memory.benchmark.ts
@@ -1,7 +1,7 @@
 import { PluginEvent } from '@posthog/plugin-scaffold/src/types'
 
 import { createServer } from '../../src/server'
-import { Plugin, PluginConfig, PluginConfigVMReponse } from '../../src/types'
+import { Plugin, PluginConfig, PluginConfigLazyVMResponse, PluginConfigVMResponse } from '../../src/types'
 import { createPluginConfigVM } from '../../src/vm/vm'
 import { commonOrganizationId } from '../../tests/helpers/plugins'
 
@@ -71,10 +71,10 @@ test('test vm memory usage', async () => {
     const usedAtStart = getUsed()
 
     let used = usedAtStart
-    const vms: PluginConfigVMReponse[] = []
+    const vms: PluginConfigLazyVMResponse[] = []
 
     for (let i = 0; i < numVMs; i++) {
-        const vm = await createPluginConfigVM(server, mockConfig, indexJs)
+        const vm = createPluginConfigVM(server, mockConfig, indexJs)
         vms.push(vm)
 
         if (debug || i === numVMs - 1) {

From 12f18da6f531f9fe6838d96c6e732e3091b66ace Mon Sep 17 00:00:00 2001
From: Marius Andra <marius.andra@gmail.com>
Date: Thu, 25 Feb 2021 23:34:02 +0100
Subject: [PATCH 09/10] tests pass if vm has been awaited

---
 src/vm/vm.ts              | 8 +++++++-
 tests/postgres/vm.test.ts | 8 ++++++++
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/src/vm/vm.ts b/src/vm/vm.ts
index 95afe2ee..c9faf31a 100644
--- a/src/vm/vm.ts
+++ b/src/vm/vm.ts
@@ -17,7 +17,7 @@ export function createPluginConfigVM(
 ): PluginConfigLazyVMResponse {
     const createVmPromise = createActualVM(server, pluginConfig, indexJs)
 
-    return {
+    const lazyResponse: PluginConfigLazyVMResponse = {
         vmPromise: createVmPromise.then((vm) => vm.vm),
         methods: {
             processEvent: (event) => createVmPromise.then((vm) => vm.methods.processEvent(event)),
@@ -25,6 +25,12 @@ export function createPluginConfigVM(
         },
         tasks: {}, // new Proxy(),
     }
+
+    void createVmPromise.then(({ tasks }) => {
+        lazyResponse.tasks = tasks
+    })
+
+    return lazyResponse
 }
 
 export async function createActualVM(
diff --git a/tests/postgres/vm.test.ts b/tests/postgres/vm.test.ts
index 1c88d7be..b7b56de0 100644
--- a/tests/postgres/vm.test.ts
+++ b/tests/postgres/vm.test.ts
@@ -652,6 +652,9 @@ test('runEvery', async () => {
     await resetTestDatabase(indexJs)
     const vm = createPluginConfigVM(mockServer, pluginConfig39, indexJs)
 
+    expect(Object.keys(vm.tasks)).toEqual([])
+    await vm.vmPromise
+
     expect(Object.keys(vm.tasks)).toEqual(['runEveryMinute', 'runEveryHour', 'runEveryDay'])
     expect(Object.values(vm.tasks).map((v) => v?.name)).toEqual(['runEveryMinute', 'runEveryHour', 'runEveryDay'])
     expect(Object.values(vm.tasks).map((v) => v?.type)).toEqual(['runEvery', 'runEvery', 'runEvery'])
@@ -669,6 +672,9 @@ test('runEvery must be a function', async () => {
     await resetTestDatabase(indexJs)
     const vm = createPluginConfigVM(mockServer, pluginConfig39, indexJs)
 
+    expect(Object.keys(vm.tasks)).toEqual([])
+    await vm.vmPromise
+
     expect(Object.keys(vm.tasks)).toEqual(['runEveryMinute'])
     expect(Object.values(vm.tasks).map((v) => v?.name)).toEqual(['runEveryMinute'])
     expect(Object.values(vm.tasks).map((v) => v?.type)).toEqual(['runEvery'])
@@ -684,6 +690,7 @@ test('posthog in runEvery', async () => {
     `
     await resetTestDatabase(indexJs)
     const vm = createPluginConfigVM(mockServer, pluginConfig39, indexJs)
+    await vm.vmPromise
 
     expect(Client).not.toHaveBeenCalled
 
@@ -723,6 +730,7 @@ test('posthog in runEvery with timestamp', async () => {
     `
     await resetTestDatabase(indexJs)
     const vm = createPluginConfigVM(mockServer, pluginConfig39, indexJs)
+    await vm.vmPromise
 
     expect(Client).not.toHaveBeenCalled
 

From db21e6543170deabc6d3a7850bd9cbfea290f9d4 Mon Sep 17 00:00:00 2001
From: Marius Andra <marius.andra@gmail.com>
Date: Thu, 25 Feb 2021 23:38:32 +0100
Subject: [PATCH 10/10] wait until plugin has loaded before starting

---
 src/plugins.ts | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/plugins.ts b/src/plugins.ts
index af71af9b..cd24b74d 100644
--- a/src/plugins.ts
+++ b/src/plugins.ts
@@ -131,6 +131,7 @@ async function loadPlugin(server: PluginsServer, pluginConfig: PluginConfig): Pr
 
             try {
                 pluginConfig.vm = createPluginConfigVM(server, pluginConfig, indexJs)
+                await pluginConfig.vm.vmPromise //
                 status.info('🔌', `Loaded local plugin "${plugin.name}" from "${pluginPath}"!`)
                 await clearError(server, pluginConfig)
                 return true
@@ -155,6 +156,7 @@ async function loadPlugin(server: PluginsServer, pluginConfig: PluginConfig): Pr
             if (indexJs) {
                 try {
                     pluginConfig.vm = createPluginConfigVM(server, pluginConfig, indexJs)
+                    await pluginConfig.vm.vmPromise
                     status.info('🔌', `Loaded plugin "${plugin.name}"!`)
                     await clearError(server, pluginConfig)
                     return true
@@ -167,6 +169,7 @@ async function loadPlugin(server: PluginsServer, pluginConfig: PluginConfig): Pr
         } else if (plugin.plugin_type === 'source' && plugin.source) {
             try {
                 pluginConfig.vm = createPluginConfigVM(server, pluginConfig, plugin.source)
+                await pluginConfig.vm.vmPromise
                 status.info('🔌', `Loaded plugin "${plugin.name}"!`)
                 await clearError(server, pluginConfig)
                 return true