Skip to content

Commit

Permalink
feat: bundle package deps as local packages
Browse files Browse the repository at this point in the history
  • Loading branch information
@vigneshshanmugam
vigneshshanmugam committed Dec 7, 2022
1 parent 8e0f4e3 commit b514b73
Show file tree
Hide file tree
Showing 2 changed files with 136 additions and 113 deletions.
91 changes: 72 additions & 19 deletions src/push/bundler.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,26 +24,33 @@
*/

import path from 'path';
import { stat, unlink, readFile } from 'fs/promises';
import { stat, unlink, readFile, writeFile } from 'fs/promises';
import { createWriteStream } from 'fs';
import * as esbuild from 'esbuild';
import NodeResolve from '@esbuild-plugins/node-resolve';
import archiver from 'archiver';
import { commonOptions, MultiAssetPlugin, PluginData } from './plugin';
import {
commonOptions,
isBare,
SyntheticsBundlePlugin,
PluginData,
} from './plugin';
import { builtinModules } from 'module';

const SIZE_LIMIT_KB = 800;
const BUNDLES_PATH = 'bundles';
const EXTERNAL_MODULES = ['@elastic/synthetics'];

function relativeToCwd(entry: string) {
return path.relative(process.cwd(), entry);
}

export class Bundler {
moduleMap = new Map<string, string>();
constructor() {}
private moduleMap = new Map<string, string>();

async prepare(absPath: string) {
const addToMap = (data: PluginData) => {
this.moduleMap.set(data.path, data.contents);
const bundlePath = this.getModulesPath(data.path);
this.moduleMap.set(bundlePath, data.contents);
};

const options: esbuild.BuildOptions = {
Expand All @@ -52,19 +59,65 @@ export class Bundler {
entryPoints: {
[absPath]: absPath,
},
plugins: [
MultiAssetPlugin(addToMap),
NodeResolve({
extensions: ['.ts', '.js'],
}),
],
plugins: [SyntheticsBundlePlugin(addToMap, EXTERNAL_MODULES)],
},
};
const result = await esbuild.build(options);
if (result.errors.length > 0) {
throw result.errors;
}
this.moduleMap.set(absPath, result.outputFiles[0].text);
}

resolvePath(bundlePath: string) {
for (const mod of this.moduleMap.keys()) {
if (mod.startsWith(bundlePath)) {
return mod.substring(0, mod.lastIndexOf(path.extname(mod)));
}
}
return null;
}

/**
* Rewrite the imports/requires to local node modules dependency
* to relative paths that references the bundles directory
*/
rewriteImports(
contents: string,
bundlePath: string,
external: string[] = EXTERNAL_MODULES
) {
const packageRegex =
/s*(from|require\()\s*(['"`][^'"`]+['"`])(?=;?)(?=([^"'`]*["'`][^"'`]*["'`])*[^"'`]*$)/gi;

return contents.replace(packageRegex, (raw, _, dep) => {
dep = dep.replace(/['"`]/g, '');
// Ignore rewriting for built-in modules, ignored modules and bare modules
if (
builtinModules.includes(dep) ||
external.includes(dep) ||
isBare(dep)
) {
return raw;
}
// If the module is not in node_modules, we need to go up the directory
// tree till we reach the bundles directory
let deep = bundlePath.split(path.sep).length;
let resolvedpath = this.resolvePath(BUNDLES_PATH + '/' + dep);
// If its already part of the bundles directory, we don't need to go up
if (bundlePath.startsWith(BUNDLES_PATH)) {
deep -= 1;
resolvedpath = resolvedpath.replace(BUNDLES_PATH + '/', '');
}
return raw.replace(dep, '.'.repeat(deep) + '/' + resolvedpath);
});
}

getModulesPath(path: string) {
const relativePath = relativeToCwd(path);
if (relativePath.startsWith('node_modules')) {
return relativePath.replace('node_modules', BUNDLES_PATH);
}
return relativePath;
}

async zip(outputPath: string) {
Expand All @@ -76,12 +129,12 @@ export class Bundler {
archive.on('error', reject);
output.on('close', fulfill);
archive.pipe(output);
for (const [path, content] of this.moduleMap.entries()) {
const relativePath = relativeToCwd(path);
// Date is fixed to Unix epoch so the file metadata is
// not modified everytime when files are bundled
archive.append(content, {
name: relativePath,
for (const [path, contents] of this.moduleMap.entries()) {
// Rewrite the imports to relative paths
archive.append(this.rewriteImports(contents, path), {
name: path,
// Date is fixed to Unix epoch so the file metadata is
// not modified everytime when files are bundled
date: new Date('1970-01-01'),
});
}
Expand Down
158 changes: 64 additions & 94 deletions src/push/plugin.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,14 +23,16 @@
*
*/

import { isAbsolute, dirname, extname, join } from 'path';
import fs from 'fs/promises';
import { join } from 'path';
import { readFile } from 'fs/promises';
import * as esbuild from 'esbuild';
import NodeResolvePlugin from '@esbuild-plugins/node-resolve';

// ROOT directory of the Package - /
const ROOT_DIR = join(__dirname, '..', '..');
// Source of the package - /src, /dist, etc.
const SOURCE_DIR = join(__dirname, '..');
const SOURCE_DIR = join(ROOT_DIR, 'src');
const DIST_DIR = join(ROOT_DIR, 'dist');
// Node modules directory of the package - /node_modules
const SOURCE_NODE_MODULES = join(ROOT_DIR, 'node_modules');

Expand All @@ -44,9 +46,8 @@ export function commonOptions(): esbuild.BuildOptions {
minifyWhitespace: false,
treeShaking: false,
keepNames: false,
logLevel: 'error',
platform: 'node',
logLevel: 'silent',
format: 'esm',
write: false,
outExtension: {
'.js': '.js',
Expand All @@ -60,109 +61,78 @@ export type PluginData = {
};
export type PluginCallback = (data: PluginData) => void;

export function MultiAssetPlugin(callback: PluginCallback): esbuild.Plugin {
// Check that the path isn't in an external package by making sure it's at a standard
// local filesystem location
const isBare = (str: string) => {
// Note that we use `isAbsolute` to handle UNC/windows style paths like C:\path\to\thing
// This is not necessary for relative directories since `.\file` is not supported as an import
// nor is `~/path/to/file`.
if (isAbsolute(str) || str.startsWith('./') || str.startsWith('../')) {
return true;
}
return false;
};

// If we're importing the @elastic/synthetics package
// directly from source instead of using the fully
// qualified name, we must skip it too. That's just
// so it doesn't get bundled on tests or when we locally
// refer to the package itself.
const isLocalSynthetics = (entryPath: string) => {
return entryPath.startsWith(SOURCE_DIR);
};

// When importing the local synthetics module directly
// it may import its own local dependencies, so we must
// make sure those will be resolved using Node's resolution
// algorithm, as they're still "node_modules" that we must bundle
const isLocalSyntheticsModule = (str: string) => {
return str.startsWith(SOURCE_NODE_MODULES);
};

return {
name: 'esbuild-multiasset-plugin',
setup(build) {
build.onResolve({ filter: /.*?/ }, async args => {
// External and other packages need be marked external to
// be removed from the bundle
if (build.initialOptions.external?.includes(args.path)) {
return {
external: true,
};
}
// Check that the path isn't in an external package by making sure it's at a standard
// local filesystem location
export const isBare = (str: string) => {
if (str.startsWith('./') || str.startsWith('../')) {
return true;
}
return false;
};

if (
!isBare(args.path) ||
args.importer.includes('/node_modules/') ||
isLocalSyntheticsModule(args.importer)
) {
return;
}
// Avoid importing @elastic/synthetics package from source
const isLocalSynthetics = (entryPath: string) => {
return entryPath.startsWith(SOURCE_DIR) || entryPath.startsWith(DIST_DIR);
};

if (args.kind === 'entry-point') {
return {
path: args.path,
namespace: 'journey',
};
}
// Avoid importing the local dependenceis of the @elastic/synthetics module
// from source
const isLocalSyntheticsModule = (str: string) => {
return str.startsWith(SOURCE_NODE_MODULES);
};

// If the modules are resolved locally, then
// use the imported path to get full path
const entryPath =
join(dirname(args.importer), args.path) + extname(args.importer);
export function SyntheticsBundlePlugin(
callback: PluginCallback,
external: string[]
): esbuild.Plugin {
const visited = new Set<string>();

return NodeResolvePlugin({
name: 'SyntheticsBundlePlugin',
extensions: ['.ts', '.js', '.mjs'],
onNonResolved: (_, __, error) => {
throw error;
},
onResolved: async resolved => {
if (
external.includes(resolved) ||
isLocalSynthetics(resolved) ||
isLocalSyntheticsModule(resolved)
) {
return {
external: true,
};
}

if (isLocalSynthetics(entryPath)) {
return { external: true };
}
if (visited.has(resolved)) {
return;
}

// Spin off another build to copy over the imported modules without bundling
// Spin off another build to copy over the imported modules without bundling
if (resolved.includes('/node_modules/')) {
const result = await esbuild.build({
...commonOptions(),
entryPoints: {
[entryPath]: entryPath,
},
entryPoints: [resolved],
bundle: false,
external: [],
});

callback({
path: entryPath,
path: resolved,
contents: result.outputFiles[0].text,
});

return {
errors: result.errors,
external: true,
};
});

build.onLoad({ filter: /.*?/, namespace: 'journey' }, async args => {
const contents = await fs.readFile(args.path, 'utf-8');
callback({ path: args.path, contents });

// Use correct loader for the journey entry path
let loader: esbuild.Loader = 'default';
const ext = extname(args.path).slice(1);
if (ext === 'cjs' || ext === 'mjs') {
loader = 'js';
} else if (ext === 'cts' || ext === 'mts') {
loader = 'ts';
} else {
loader = ext as esbuild.Loader;
} else {
// If it's a local file, read it and return the contents
// to preserve the source without modifications
try {
const contents = await readFile(resolved, 'utf-8');
callback({ path: resolved, contents });
} catch (e) {
throw new Error(`Could not read file ${resolved}`);
}
return { contents, loader };
});
}
visited.add(resolved);
return;
},
};
});
}

0 comments on commit b514b73

Please sign in to comment.