From 7b2df4b5f1b822e943c6a9d9a36b67eff1ece641 Mon Sep 17 00:00:00 2001 From: Cristiano Belloni Date: Wed, 1 Dec 2021 16:02:53 +0000 Subject: [PATCH] Web Worker support for esbuild (#1103) * Add worker plugin - this works with build * Use outdir to decide where workers are built * web worker plugin stuff * emit as a file * clean up and correct regex * Use parent esbuild conf, fix build * Pin the file extension * use loader URL * Lint * Fix tests * inherit minify and use relative paths to not break snapshots * Add allow-list plugin * split extension allow list plugin * use onLoad for allow list * update test desc and use fs directly * Use postfix to be able to not use loader string * Pass relative path, reconstruct absolute path * Make allow list message generic * Delete entry from cache once contents generated * Use onResolve and onLoad for extension allow plugin + specify a reason * Use errors interface * Fix error msg * Add docs * Create fifty-goats-shave.md Co-authored-by: Cristiano Belloni Co-authored-by: Luke Sheard --- .changeset/fifty-goats-shave.md | 5 + docs/building-apps/index.md | 7 ++ docs/building-apps/web workers.md | 51 ++++++++ packages/modular-scripts/react-app-env.d.ts | 8 ++ .../worker-plugin/alive.worker.ts | 3 + .../__fixtures__/worker-plugin/index.ts | 7 ++ .../__snapshots__/workerPlugin.test.ts.snap | 31 +++++ .../esbuild-scripts/workerPlugin.test.ts | 64 ++++++++++ .../config/createEsbuildConfig.ts | 3 + .../plugins/extensionAllowList.ts | 45 +++++++ .../plugins/workerFactoryPlugin.ts | 117 ++++++++++++++++++ packages/modular-scripts/tsconfig.json | 2 +- 12 files changed, 342 insertions(+), 1 deletion(-) create mode 100644 .changeset/fifty-goats-shave.md create mode 100644 docs/building-apps/index.md create mode 100644 docs/building-apps/web workers.md create mode 100644 packages/modular-scripts/src/__tests__/esbuild-scripts/__fixtures__/worker-plugin/alive.worker.ts create mode 100644 packages/modular-scripts/src/__tests__/esbuild-scripts/__fixtures__/worker-plugin/index.ts create mode 100644 packages/modular-scripts/src/__tests__/esbuild-scripts/__snapshots__/workerPlugin.test.ts.snap create mode 100644 packages/modular-scripts/src/__tests__/esbuild-scripts/workerPlugin.test.ts create mode 100644 packages/modular-scripts/src/esbuild-scripts/plugins/extensionAllowList.ts create mode 100644 packages/modular-scripts/src/esbuild-scripts/plugins/workerFactoryPlugin.ts diff --git a/.changeset/fifty-goats-shave.md b/.changeset/fifty-goats-shave.md new file mode 100644 index 000000000..05873e5f8 --- /dev/null +++ b/.changeset/fifty-goats-shave.md @@ -0,0 +1,5 @@ +--- +"modular-scripts": minor +--- + +Web Worker support and docs for esbuild. diff --git a/docs/building-apps/index.md b/docs/building-apps/index.md new file mode 100644 index 000000000..1170418ff --- /dev/null +++ b/docs/building-apps/index.md @@ -0,0 +1,7 @@ +--- +has_children: true +title: Building your Apps +nav_order: 500 +--- + +# Building your Apps diff --git a/docs/building-apps/web workers.md b/docs/building-apps/web workers.md new file mode 100644 index 000000000..1a6831b12 --- /dev/null +++ b/docs/building-apps/web workers.md @@ -0,0 +1,51 @@ +--- +parent: Building your Apps +title: Adding web workers +--- + +esbuild {: .label .label-yellow } + +It is possible to add +[web workers](https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API/Using_web_workers) +to your application just by writing them as normal typescript modules. There are +some rules to follow to write a worker: + +- Your worker module must follow the `.worker.[ts|js|jsx|tsx]` name + pattern for Modular to build it as a worker. +- Worker extension must be explicitly included in the import statement for the + typechecker to correctly type it. `import Worker from './my.worker.ts'` is ok, + `import Worker from './my.worker'` is not. +- A worker can only `import` other modules. Trying to `import` files that have a + different extension than `[ts|js|jsx|tsx]` will trigger a build error. +- If a worker doesn't `import` any other module, it should `export {}` or + `export default {}` to avoid being marked as global module by the type + checker. + +Importing a worker will return a `Class` that, when instantiated, returns a +worker instance. For example: + +```ts +// ./index.ts +import DateFormatterCls from './worker/dateFormatter.worker.ts'; + +// Instantiate the worker +const worker = new DateFormatterCls(); + +worker.current.onmessage = (message) => + console.log('Received a message from worker', message.data); +worker.postMessage(new Date.now()); +``` + +```ts +// ./worker/dateFormatter.worker.ts +import { wait, format } from '../utils/date-utils'; +// These imports are allowed because they refer to other modules + +globalThis.self.onmessage = async (message: { data: number }) => { + postMessage(`Hello there. Processing date...`); + // Simulate work + await wait(500); + // Send back the formatter date + postMessage(`Date is: ${format(message.data)}`); +}; +``` diff --git a/packages/modular-scripts/react-app-env.d.ts b/packages/modular-scripts/react-app-env.d.ts index da297d96c..1f7485696 100644 --- a/packages/modular-scripts/react-app-env.d.ts +++ b/packages/modular-scripts/react-app-env.d.ts @@ -11,6 +11,14 @@ declare namespace NodeJS { } } +declare module '*.worker.ts' { + class WebWorkerClass extends Worker { + constructor(); + } + + export default WebWorkerClass; +} + declare module '*.avif' { const src: string; export default src; diff --git a/packages/modular-scripts/src/__tests__/esbuild-scripts/__fixtures__/worker-plugin/alive.worker.ts b/packages/modular-scripts/src/__tests__/esbuild-scripts/__fixtures__/worker-plugin/alive.worker.ts new file mode 100644 index 000000000..bc890319a --- /dev/null +++ b/packages/modular-scripts/src/__tests__/esbuild-scripts/__fixtures__/worker-plugin/alive.worker.ts @@ -0,0 +1,3 @@ +globalThis.self.postMessage("I'm alive!"); + +export {}; diff --git a/packages/modular-scripts/src/__tests__/esbuild-scripts/__fixtures__/worker-plugin/index.ts b/packages/modular-scripts/src/__tests__/esbuild-scripts/__fixtures__/worker-plugin/index.ts new file mode 100644 index 000000000..688000294 --- /dev/null +++ b/packages/modular-scripts/src/__tests__/esbuild-scripts/__fixtures__/worker-plugin/index.ts @@ -0,0 +1,7 @@ +import WorkerCls from './alive.worker.ts'; + +const worker = new WorkerCls(); + +worker.addEventListener('message', (event: MessageEvent) => { + console.log(`Received message from worker: ${event.data}`); +}); diff --git a/packages/modular-scripts/src/__tests__/esbuild-scripts/__snapshots__/workerPlugin.test.ts.snap b/packages/modular-scripts/src/__tests__/esbuild-scripts/__snapshots__/workerPlugin.test.ts.snap new file mode 100644 index 000000000..609b4df2d --- /dev/null +++ b/packages/modular-scripts/src/__tests__/esbuild-scripts/__snapshots__/workerPlugin.test.ts.snap @@ -0,0 +1,31 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`WHEN running esbuild with the workerFactoryPlugin WHEN there's a url import SHOULD ouput the correct alive.worker-[hash].ts file 1`] = ` +"// packages/modular-scripts/src/__tests__/esbuild-scripts/__fixtures__/worker-plugin/alive.worker.ts +globalThis.self.postMessage(\\"I'm alive!\\"); +" +`; + +exports[`WHEN running esbuild with the workerFactoryPlugin WHEN there's a url import SHOULD ouput the correct index.js 1`] = ` +"// worker-url:packages/modular-scripts/src/__tests__/esbuild-scripts/__fixtures__/worker-plugin/alive.worker.js +var alive_worker_default = \\"./alive.worker-T4TLN6IN.js\\"; + +// web-worker:packages/modular-scripts/src/__tests__/esbuild-scripts/__fixtures__/worker-plugin/alive.worker.js +var workerPath = new URL(alive_worker_default, import.meta.url); +var importSrc = 'import \\"' + workerPath + '\\";'; +var blob = new Blob([importSrc], { + type: \\"text/javascript\\" +}); +var alive_worker_default2 = class { + constructor() { + return new Worker(URL.createObjectURL(blob), { type: \\"module\\" }); + } +}; + +// packages/modular-scripts/src/__tests__/esbuild-scripts/__fixtures__/worker-plugin/index.ts +var worker = new alive_worker_default2(); +worker.addEventListener(\\"message\\", (event) => { + console.log(\`Received message from worker: \${event.data}\`); +}); +" +`; diff --git a/packages/modular-scripts/src/__tests__/esbuild-scripts/workerPlugin.test.ts b/packages/modular-scripts/src/__tests__/esbuild-scripts/workerPlugin.test.ts new file mode 100644 index 000000000..ef306f11f --- /dev/null +++ b/packages/modular-scripts/src/__tests__/esbuild-scripts/workerPlugin.test.ts @@ -0,0 +1,64 @@ +import * as esbuild from 'esbuild'; +import * as path from 'path'; +import * as fs from 'fs-extra'; +import * as tmp from 'tmp'; +import tree from 'tree-view-for-tests'; +import webworkerPlugin from '../../esbuild-scripts/plugins/workerFactoryPlugin'; +import getModularRoot from '../../utils/getModularRoot'; + +describe('WHEN running esbuild with the workerFactoryPlugin', () => { + describe("WHEN there's a url import", () => { + let tmpDir: tmp.DirResult; + let result: esbuild.BuildResult; + let outdir: string; + + beforeAll(async () => { + tmpDir = tmp.dirSync(); + outdir = path.join(tmpDir.name, 'output'); + result = await esbuild.build({ + entryPoints: [ + path.join(__dirname, '__fixtures__', 'worker-plugin', 'index.ts'), + ], + plugins: [webworkerPlugin()], + outdir, + sourceRoot: getModularRoot(), + bundle: true, + splitting: true, + format: 'esm', + target: 'es2021', + }); + }); + + afterAll(async () => { + await fs.emptyDir(tmpDir.name); + tmpDir.removeCallback(); + }); + + it('SHOULD be successful', () => { + expect(result.errors).toEqual([]); + expect(result.warnings).toEqual([]); + }); + + it('SHOULD have the correct output structure', () => { + expect(tree(outdir)).toMatchInlineSnapshot(` + "output + ├─ alive.worker-T4TLN6IN.js #y0mybi + └─ index.js #1kx9oa0" + `); + }); + + it('SHOULD ouput the correct index.js', () => { + let content = String(fs.readFileSync(path.join(outdir, 'index.js'))); + content = content.replaceAll(getModularRoot(), ''); + expect(content).toMatchSnapshot(); + }); + + it('SHOULD ouput the correct alive.worker-[hash].ts file', () => { + let content = String( + fs.readFileSync(path.join(outdir, 'alive.worker-T4TLN6IN.js')), + ); + content = content.replaceAll(getModularRoot(), ''); + expect(content).toMatchSnapshot(); + }); + }); +}); diff --git a/packages/modular-scripts/src/esbuild-scripts/config/createEsbuildConfig.ts b/packages/modular-scripts/src/esbuild-scripts/config/createEsbuildConfig.ts index 3b1a54ec0..e7afc5aca 100644 --- a/packages/modular-scripts/src/esbuild-scripts/config/createEsbuildConfig.ts +++ b/packages/modular-scripts/src/esbuild-scripts/config/createEsbuildConfig.ts @@ -8,15 +8,18 @@ import * as logger from '../../utils/logger'; import moduleScopePlugin from '../plugins/moduleScopePlugin'; import svgrPlugin from '../plugins/svgr'; +import workerFactoryPlugin from '../plugins/workerFactoryPlugin'; export default function createEsbuildConfig( paths: Paths, config: Partial = {}, ): esbuild.BuildOptions { const { plugins: configPlugins, ...partialConfig } = config; + const plugins: esbuild.Plugin[] = [ moduleScopePlugin(paths), svgrPlugin(), + workerFactoryPlugin(), ].concat(configPlugins || []); const define = Object.assign( diff --git a/packages/modular-scripts/src/esbuild-scripts/plugins/extensionAllowList.ts b/packages/modular-scripts/src/esbuild-scripts/plugins/extensionAllowList.ts new file mode 100644 index 000000000..63ecb9638 --- /dev/null +++ b/packages/modular-scripts/src/esbuild-scripts/plugins/extensionAllowList.ts @@ -0,0 +1,45 @@ +import * as esbuild from 'esbuild'; +import * as path from 'path'; + +// This plugin resolves all the files in the project and excepts if one of the extensions is not in the allow list +// Please note that implicit (empty) extensions in the importer are always valid. + +interface ExtensionPluginConf { + allowedExtensions?: string[]; + reason?: string; +} + +function createExtensionAllowlistPlugin({ + reason, + allowedExtensions = ['.js', '.jsx', '.ts', '.tsx'], +}: ExtensionPluginConf): esbuild.Plugin { + return { + name: 'extension-allow-list-plugin', + setup(build) { + // No lookbehind in Go regexp; need to look at all the files and do the check manually. + build.onResolve({ filter: /.*/ }, (args) => { + // Extract the extension; if not in the allow list, return an error. + const extension = path.extname(args.path); + if (extension && !allowedExtensions.includes(extension)) { + const errorReason = reason ? ` Reason: ${reason}` : ''; + return { + errors: [ + { + pluginName: 'extension-allow-list-plugin', + text: `Extension not allowed`, + detail: `Extension for file "${args.path}", imported by "${ + args.importer + }", is not allowed. Permitted extensions are: ${JSON.stringify( + allowedExtensions, + )}.${errorReason}`, + }, + ], + }; + } + return undefined; + }); + }, + }; +} + +export default createExtensionAllowlistPlugin; diff --git a/packages/modular-scripts/src/esbuild-scripts/plugins/workerFactoryPlugin.ts b/packages/modular-scripts/src/esbuild-scripts/plugins/workerFactoryPlugin.ts new file mode 100644 index 000000000..9e77c8817 --- /dev/null +++ b/packages/modular-scripts/src/esbuild-scripts/plugins/workerFactoryPlugin.ts @@ -0,0 +1,117 @@ +import * as esbuild from 'esbuild'; +import * as path from 'path'; +import getModularRoot from '../../utils/getModularRoot'; +import createExtensionAllowlistPlugin from './extensionAllowList'; + +// This plugin builds Web Workers on the fly and exports them to use like worker-loader for Webpack 4: https://v4.webpack.js.org/loaders/worker-loader/ +// The workers are not inlined, a new file is generated in the bundle. Only files *imported* with the *.worker.[jt]sx pattern are matched. +// The workers are trampolined to avoid CORS errors. +// This will be deprecated in the future when esbuild supports the Worker signature: see https://github.com/evanw/esbuild/issues/312 +// And will probably end up being compatible with Webpack 5 support https://webpack.js.org/guides/web-workers + +function createPlugin(): esbuild.Plugin { + const plugin: esbuild.Plugin = { + name: 'worker-factory-plugin', + setup(build) { + // This stores built workers for later use + const workerBuildCache: Map = new Map(); + + build.onResolve({ filter: /.*\.worker.[jt]sx?$/ }, (args) => { + const importPath = args.path; + const importAbsolutePath = path.join(args.resolveDir, importPath); + + // Pin the file extension to .js + const workerAbsolutePath = path.join( + path.dirname(importAbsolutePath), + path.basename(importAbsolutePath).replace(/\.[jt]sx?$/, '.js'), + ); + + const relativePath = path.relative( + getModularRoot(), + workerAbsolutePath, + ); + + return { + path: relativePath, + namespace: 'web-worker', + }; + }); + + build.onLoad({ filter: /.*/, namespace: 'web-worker' }, async (args) => { + // Build the worker file with the same format, target and definitions of the bundle + try { + const result = await esbuild.build({ + format: build.initialOptions.format, + target: build.initialOptions.target, + define: build.initialOptions.define, + minify: build.initialOptions.minify, + entryPoints: [path.join(getModularRoot(), args.path)], + plugins: [ + createExtensionAllowlistPlugin({ + reason: 'Web workers can only import other modules.', + }), + ], + bundle: true, + write: false, + }); + + // Store the file in the build cache for later use, since we need to emit a file and trampoline it transparently to the user + workerBuildCache.set(args.path, result); + + // Trampoline the worker within the bundle, to avoid CORS errors + return { + contents: ` + // Web worker bundled by worker-factory-plugin, mimicking the Worker constructor + import workerUrl from '${args.path}:__worker-url'; + + const workerPath = new URL(workerUrl, import.meta.url); + const importSrc = 'import "' + workerPath + '";'; + + const blob = new Blob([importSrc], { + type: "text/javascript", + }); + + export default class { + constructor() { + return new Worker(URL.createObjectURL(blob), { type: "module" }); + } + } + `, + }; + } catch (e) { + console.error('Error building worker script:', e); + } + }); + + build.onResolve({ filter: /.*:__worker-url/ }, (args) => { + return { + path: args.path.split(':__worker-url')[0], + namespace: 'worker-url', + }; + }); + + build.onLoad({ filter: /.*/, namespace: 'worker-url' }, (args) => { + const result = workerBuildCache.get(args.path); + workerBuildCache.delete(args.path); + if (result) { + const { outputFiles } = result; + if (outputFiles?.length === 1) { + const outputFile = outputFiles[0]; + return { + contents: outputFile.contents, + loader: 'file', + }; + } else { + throw new Error(`Could not read output files`); + } + } else { + throw new Error(`Could not find result for ${args.path}`); + } + }); + }, + }; + + return plugin; +} + +export default createPlugin; diff --git a/packages/modular-scripts/tsconfig.json b/packages/modular-scripts/tsconfig.json index 9954e47a9..ac69ea998 100644 --- a/packages/modular-scripts/tsconfig.json +++ b/packages/modular-scripts/tsconfig.json @@ -2,7 +2,7 @@ "compilerOptions": { "target": "es5", "downlevelIteration": true, - "lib": ["dom", "dom.iterable", "esnext"], + "lib": ["dom", "dom.iterable", "esnext", "WebWorker"], "allowJs": true, "skipLibCheck": true, "esModuleInterop": true,