Skip to content

Commit

Permalink
Web Worker support for esbuild (#1103)
Browse files Browse the repository at this point in the history
* Add worker plugin - this works with build

* Use outdir to decide where workers are built

* web worker plugin stuff

* emit as a file

* clean up and correct regex

* Use parent esbuild conf, fix build

* Pin the file extension

* use loader URL

* Lint

* Fix tests

* inherit minify and use relative paths to not break snapshots

* Add allow-list plugin

* split extension allow list plugin

* use onLoad for allow list

* update test desc and use fs directly

* Use postfix to be able to not use loader string

* Pass relative path, reconstruct absolute path

* Make allow list message generic

* Delete entry from cache once contents generated

* Use onResolve and onLoad for extension allow plugin + specify a reason

* Use errors interface

* Fix error msg

* Add docs

* Create fifty-goats-shave.md

Co-authored-by: Cristiano Belloni <cristiano.belloni@jpmorgan.com>
Co-authored-by: Luke Sheard <luke.sheard@jpmorgan.com>
  • Loading branch information
3 people authored Dec 1, 2021
1 parent 1a26b8f commit 7b2df4b
Show file tree
Hide file tree
Showing 12 changed files with 342 additions and 1 deletion.
5 changes: 5 additions & 0 deletions .changeset/fifty-goats-shave.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
"modular-scripts": minor
---

Web Worker support and docs for esbuild.
7 changes: 7 additions & 0 deletions docs/building-apps/index.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
---
has_children: true
title: Building your Apps
nav_order: 500
---

# Building your Apps
51 changes: 51 additions & 0 deletions docs/building-apps/web workers.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
---
parent: Building your Apps
title: Adding web workers
---

esbuild {: .label .label-yellow }

It is possible to add
[web workers](https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API/Using_web_workers)
to your application just by writing them as normal typescript modules. There are
some rules to follow to write a worker:

- Your worker module must follow the `<filename>.worker.[ts|js|jsx|tsx]` name
pattern for Modular to build it as a worker.
- Worker extension must be explicitly included in the import statement for the
typechecker to correctly type it. `import Worker from './my.worker.ts'` is ok,
`import Worker from './my.worker'` is not.
- A worker can only `import` other modules. Trying to `import` files that have a
different extension than `[ts|js|jsx|tsx]` will trigger a build error.
- If a worker doesn't `import` any other module, it should `export {}` or
`export default {}` to avoid being marked as global module by the type
checker.

Importing a worker will return a `Class` that, when instantiated, returns a
worker instance. For example:

```ts
// ./index.ts
import DateFormatterCls from './worker/dateFormatter.worker.ts';

// Instantiate the worker
const worker = new DateFormatterCls();

worker.current.onmessage = (message) =>
console.log('Received a message from worker', message.data);
worker.postMessage(new Date.now());
```

```ts
// ./worker/dateFormatter.worker.ts
import { wait, format } from '../utils/date-utils';
// These imports are allowed because they refer to other modules

globalThis.self.onmessage = async (message: { data: number }) => {
postMessage(`Hello there. Processing date...`);
// Simulate work
await wait(500);
// Send back the formatter date
postMessage(`Date is: ${format(message.data)}`);
};
```
8 changes: 8 additions & 0 deletions packages/modular-scripts/react-app-env.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,14 @@ declare namespace NodeJS {
}
}

declare module '*.worker.ts' {
class WebWorkerClass extends Worker {
constructor();
}

export default WebWorkerClass;
}

declare module '*.avif' {
const src: string;
export default src;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
globalThis.self.postMessage("I'm alive!");

export {};
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
import WorkerCls from './alive.worker.ts';

const worker = new WorkerCls();

worker.addEventListener('message', (event: MessageEvent<string>) => {
console.log(`Received message from worker: ${event.data}`);
});
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP

exports[`WHEN running esbuild with the workerFactoryPlugin WHEN there's a url import SHOULD ouput the correct alive.worker-[hash].ts file 1`] = `
"// packages/modular-scripts/src/__tests__/esbuild-scripts/__fixtures__/worker-plugin/alive.worker.ts
globalThis.self.postMessage(\\"I'm alive!\\");
"
`;

exports[`WHEN running esbuild with the workerFactoryPlugin WHEN there's a url import SHOULD ouput the correct index.js 1`] = `
"// worker-url:packages/modular-scripts/src/__tests__/esbuild-scripts/__fixtures__/worker-plugin/alive.worker.js
var alive_worker_default = \\"./alive.worker-T4TLN6IN.js\\";
// web-worker:packages/modular-scripts/src/__tests__/esbuild-scripts/__fixtures__/worker-plugin/alive.worker.js
var workerPath = new URL(alive_worker_default, import.meta.url);
var importSrc = 'import \\"' + workerPath + '\\";';
var blob = new Blob([importSrc], {
type: \\"text/javascript\\"
});
var alive_worker_default2 = class {
constructor() {
return new Worker(URL.createObjectURL(blob), { type: \\"module\\" });
}
};
// packages/modular-scripts/src/__tests__/esbuild-scripts/__fixtures__/worker-plugin/index.ts
var worker = new alive_worker_default2();
worker.addEventListener(\\"message\\", (event) => {
console.log(\`Received message from worker: \${event.data}\`);
});
"
`;
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
import * as esbuild from 'esbuild';
import * as path from 'path';
import * as fs from 'fs-extra';
import * as tmp from 'tmp';
import tree from 'tree-view-for-tests';
import webworkerPlugin from '../../esbuild-scripts/plugins/workerFactoryPlugin';
import getModularRoot from '../../utils/getModularRoot';

describe('WHEN running esbuild with the workerFactoryPlugin', () => {
describe("WHEN there's a url import", () => {
let tmpDir: tmp.DirResult;
let result: esbuild.BuildResult;
let outdir: string;

beforeAll(async () => {
tmpDir = tmp.dirSync();
outdir = path.join(tmpDir.name, 'output');
result = await esbuild.build({
entryPoints: [
path.join(__dirname, '__fixtures__', 'worker-plugin', 'index.ts'),
],
plugins: [webworkerPlugin()],
outdir,
sourceRoot: getModularRoot(),
bundle: true,
splitting: true,
format: 'esm',
target: 'es2021',
});
});

afterAll(async () => {
await fs.emptyDir(tmpDir.name);
tmpDir.removeCallback();
});

it('SHOULD be successful', () => {
expect(result.errors).toEqual([]);
expect(result.warnings).toEqual([]);
});

it('SHOULD have the correct output structure', () => {
expect(tree(outdir)).toMatchInlineSnapshot(`
"output
├─ alive.worker-T4TLN6IN.js #y0mybi
└─ index.js #1kx9oa0"
`);
});

it('SHOULD ouput the correct index.js', () => {
let content = String(fs.readFileSync(path.join(outdir, 'index.js')));
content = content.replaceAll(getModularRoot(), '');
expect(content).toMatchSnapshot();
});

it('SHOULD ouput the correct alive.worker-[hash].ts file', () => {
let content = String(
fs.readFileSync(path.join(outdir, 'alive.worker-T4TLN6IN.js')),
);
content = content.replaceAll(getModularRoot(), '');
expect(content).toMatchSnapshot();
});
});
});
Original file line number Diff line number Diff line change
Expand Up @@ -8,15 +8,18 @@ import * as logger from '../../utils/logger';

import moduleScopePlugin from '../plugins/moduleScopePlugin';
import svgrPlugin from '../plugins/svgr';
import workerFactoryPlugin from '../plugins/workerFactoryPlugin';

export default function createEsbuildConfig(
paths: Paths,
config: Partial<esbuild.BuildOptions> = {},
): esbuild.BuildOptions {
const { plugins: configPlugins, ...partialConfig } = config;

const plugins: esbuild.Plugin[] = [
moduleScopePlugin(paths),
svgrPlugin(),
workerFactoryPlugin(),
].concat(configPlugins || []);

const define = Object.assign(
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
import * as esbuild from 'esbuild';
import * as path from 'path';

// This plugin resolves all the files in the project and excepts if one of the extensions is not in the allow list
// Please note that implicit (empty) extensions in the importer are always valid.

interface ExtensionPluginConf {
allowedExtensions?: string[];
reason?: string;
}

function createExtensionAllowlistPlugin({
reason,
allowedExtensions = ['.js', '.jsx', '.ts', '.tsx'],
}: ExtensionPluginConf): esbuild.Plugin {
return {
name: 'extension-allow-list-plugin',
setup(build) {
// No lookbehind in Go regexp; need to look at all the files and do the check manually.
build.onResolve({ filter: /.*/ }, (args) => {
// Extract the extension; if not in the allow list, return an error.
const extension = path.extname(args.path);
if (extension && !allowedExtensions.includes(extension)) {
const errorReason = reason ? ` Reason: ${reason}` : '';
return {
errors: [
{
pluginName: 'extension-allow-list-plugin',
text: `Extension not allowed`,
detail: `Extension for file "${args.path}", imported by "${
args.importer
}", is not allowed. Permitted extensions are: ${JSON.stringify(
allowedExtensions,
)}.${errorReason}`,
},
],
};
}
return undefined;
});
},
};
}

export default createExtensionAllowlistPlugin;
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
import * as esbuild from 'esbuild';
import * as path from 'path';
import getModularRoot from '../../utils/getModularRoot';
import createExtensionAllowlistPlugin from './extensionAllowList';

// This plugin builds Web Workers on the fly and exports them to use like worker-loader for Webpack 4: https://v4.webpack.js.org/loaders/worker-loader/
// The workers are not inlined, a new file is generated in the bundle. Only files *imported* with the *.worker.[jt]sx pattern are matched.
// The workers are trampolined to avoid CORS errors.
// This will be deprecated in the future when esbuild supports the Worker signature: see https://github.com/evanw/esbuild/issues/312
// And will probably end up being compatible with Webpack 5 support https://webpack.js.org/guides/web-workers

function createPlugin(): esbuild.Plugin {
const plugin: esbuild.Plugin = {
name: 'worker-factory-plugin',
setup(build) {
// This stores built workers for later use
const workerBuildCache: Map<string, esbuild.BuildResult> = new Map();

build.onResolve({ filter: /.*\.worker.[jt]sx?$/ }, (args) => {
const importPath = args.path;
const importAbsolutePath = path.join(args.resolveDir, importPath);

// Pin the file extension to .js
const workerAbsolutePath = path.join(
path.dirname(importAbsolutePath),
path.basename(importAbsolutePath).replace(/\.[jt]sx?$/, '.js'),
);

const relativePath = path.relative(
getModularRoot(),
workerAbsolutePath,
);

return {
path: relativePath,
namespace: 'web-worker',
};
});

build.onLoad({ filter: /.*/, namespace: 'web-worker' }, async (args) => {
// Build the worker file with the same format, target and definitions of the bundle
try {
const result = await esbuild.build({
format: build.initialOptions.format,
target: build.initialOptions.target,
define: build.initialOptions.define,
minify: build.initialOptions.minify,
entryPoints: [path.join(getModularRoot(), args.path)],
plugins: [
createExtensionAllowlistPlugin({
reason: 'Web workers can only import other modules.',
}),
],
bundle: true,
write: false,
});

// Store the file in the build cache for later use, since we need to emit a file and trampoline it transparently to the user
workerBuildCache.set(args.path, result);

// Trampoline the worker within the bundle, to avoid CORS errors
return {
contents: `
// Web worker bundled by worker-factory-plugin, mimicking the Worker constructor
import workerUrl from '${args.path}:__worker-url';
const workerPath = new URL(workerUrl, import.meta.url);
const importSrc = 'import "' + workerPath + '";';
const blob = new Blob([importSrc], {
type: "text/javascript",
});
export default class {
constructor() {
return new Worker(URL.createObjectURL(blob), { type: "module" });
}
}
`,
};
} catch (e) {
console.error('Error building worker script:', e);
}
});

build.onResolve({ filter: /.*:__worker-url/ }, (args) => {
return {
path: args.path.split(':__worker-url')[0],
namespace: 'worker-url',
};
});

build.onLoad({ filter: /.*/, namespace: 'worker-url' }, (args) => {
const result = workerBuildCache.get(args.path);
workerBuildCache.delete(args.path);
if (result) {
const { outputFiles } = result;
if (outputFiles?.length === 1) {
const outputFile = outputFiles[0];
return {
contents: outputFile.contents,
loader: 'file',
};
} else {
throw new Error(`Could not read output files`);
}
} else {
throw new Error(`Could not find result for ${args.path}`);
}
});
},
};

return plugin;
}

export default createPlugin;
2 changes: 1 addition & 1 deletion packages/modular-scripts/tsconfig.json
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
"compilerOptions": {
"target": "es5",
"downlevelIteration": true,
"lib": ["dom", "dom.iterable", "esnext"],
"lib": ["dom", "dom.iterable", "esnext", "WebWorker"],
"allowJs": true,
"skipLibCheck": true,
"esModuleInterop": true,
Expand Down

0 comments on commit 7b2df4b

Please sign in to comment.