From 66f15a23f3e5f71cc8dcb3cdfb36aada654ef29e Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Thu, 18 May 2023 12:00:09 -0400 Subject: [PATCH 01/72] temporarily disabling new addon index resolving It was unused because the code to emit the index doesn't exist, but I'm about to add it, and I'd like to focus on one thihng at a time. --- packages/core/src/module-resolver.ts | 30 +++++++++++++-------------- tests/scenarios/core-resolver-test.ts | 24 ++++++++++----------- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/packages/core/src/module-resolver.ts b/packages/core/src/module-resolver.ts index 08e0fa645..83f56e416 100644 --- a/packages/core/src/module-resolver.ts +++ b/packages/core/src/module-resolver.ts @@ -18,8 +18,8 @@ import { } from './virtual-content'; import { Memoize } from 'typescript-memoize'; import { describeExports } from './describe-exports'; -import { existsSync, readFileSync } from 'fs'; -import { readJSONSync } from 'fs-extra'; +import { /* existsSync, */ readFileSync } from 'fs'; +// import { readJSONSync } from 'fs-extra'; const debug = makeDebug('embroider:resolver'); function logTransition(reason: string, before: R, after: R = before): R { @@ -646,19 +646,19 @@ export class Resolver { @Memoize() private get legacyAddonsIndex(): { v1ToV2: Map; v2toV1: Map } { - let addonsDir = resolve(this.options.appRoot, 'node_modules', '.embroider', 'addons'); - let indexFile = resolve(addonsDir, 'v1-addon-index.json'); - if (existsSync(indexFile)) { - let { v1Addons } = readJSONSync(indexFile) as { v1Addons: Record }; - return { - v1ToV2: new Map( - Object.entries(v1Addons).map(([oldRoot, relativeNewRoot]) => [oldRoot, resolve(addonsDir, relativeNewRoot)]) - ), - v2toV1: new Map( - Object.entries(v1Addons).map(([oldRoot, relativeNewRoot]) => [resolve(addonsDir, relativeNewRoot), oldRoot]) - ), - }; - } + // let addonsDir = resolve(this.options.appRoot, 'node_modules', '.embroider', 'rewritten-packages'); + // let indexFile = resolve(addonsDir, 'index.json'); + // if (existsSync(indexFile)) { + // let { packages } = readJSONSync(indexFile) as { packages: Record }; + // return { + // v1ToV2: new Map( + // Object.entries(packages).map(([oldRoot, relativeNewRoot]) => [oldRoot, resolve(addonsDir, relativeNewRoot)]) + // ), + // v2toV1: new Map( + // Object.entries(packages).map(([oldRoot, relativeNewRoot]) => [resolve(addonsDir, relativeNewRoot), oldRoot]) + // ), + // }; + // } return { v1ToV2: new Map(), v2toV1: new Map() }; } diff --git a/tests/scenarios/core-resolver-test.ts b/tests/scenarios/core-resolver-test.ts index d2da534fc..0377005a6 100644 --- a/tests/scenarios/core-resolver-test.ts +++ b/tests/scenarios/core-resolver-test.ts @@ -601,15 +601,15 @@ Scenarios.fromProject(() => new Project()) }); Qmodule('legacy-addons', function () { - test('app can resolve file in rewritten addon', async function () { + QUnit.skip('app can resolve file in rewritten addon', async function () { givenFiles({ - 'node_modules/.embroider/addons/v1-addon-index.json': JSON.stringify({ - v1Addons: { + 'node_modules/.embroider/rewritten-packages/index.json': JSON.stringify({ + packages: { [resolve(app.dir, 'node_modules/my-addon')]: 'my-addon.1234', }, }), - 'node_modules/.embroider/addons/my-addon.1234/hello-world.js': ``, - 'node_modules/.embroider/addons/my-addon.1234/package.json': addonPackageJSON(), + 'node_modules/.embroider/rewritten-packages/my-addon.1234/hello-world.js': ``, + 'node_modules/.embroider/rewritten-packages/my-addon.1234/package.json': addonPackageJSON(), 'app.js': `import "my-addon/hello-world"`, }); @@ -618,26 +618,26 @@ Scenarios.fromProject(() => new Project()) expectAudit .module('./app.js') .resolves('my-addon/hello-world') - .to('./node_modules/.embroider/addons/my-addon.1234/hello-world.js'); + .to('./node_modules/.embroider/rewritten-packages/my-addon.1234/hello-world.js'); }); - test('moved addon resolves dependencies from its original location', async function () { + QUnit.skip('moved addon resolves dependencies from its original location', async function () { givenFiles({ 'node_modules/my-addon/node_modules/inner-dep/index.js': '', - 'node_modules/.embroider/addons/v1-addon-index.json': JSON.stringify({ - v1Addons: { + 'node_modules/.embroider/rewritten-packages/index.json': JSON.stringify({ + packages: { [resolve(app.dir, 'node_modules/my-addon')]: 'my-addon.1234', }, }), - 'node_modules/.embroider/addons/my-addon.1234/hello-world.js': `import "inner-dep"`, - 'node_modules/.embroider/addons/my-addon.1234/package.json': addonPackageJSON(), + 'node_modules/.embroider/rewritten-packages/my-addon.1234/hello-world.js': `import "inner-dep"`, + 'node_modules/.embroider/rewritten-packages/my-addon.1234/package.json': addonPackageJSON(), 'app.js': `import "my-addon/hello-world"`, }); await configure({}); expectAudit - .module('./node_modules/.embroider/addons/my-addon.1234/hello-world.js') + .module('./node_modules/.embroider/rewritten-packages/my-addon.1234/hello-world.js') .resolves('inner-dep') .to('./node_modules/my-addon/node_modules/inner-dep/index.js'); }); From 21fb86516ee4fbd7426c88bd097d590983877ba5 Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Thu, 18 May 2023 12:13:05 -0400 Subject: [PATCH 02/72] emit addon index --- packages/compat/src/moved-package-cache.ts | 18 +++++++++++++++--- packages/core/src/index.ts | 1 + packages/core/src/module-resolver.ts | 14 ++++++++++++-- 3 files changed, 28 insertions(+), 5 deletions(-) diff --git a/packages/compat/src/moved-package-cache.ts b/packages/compat/src/moved-package-cache.ts index 997ae870d..d0ad3e9f4 100644 --- a/packages/compat/src/moved-package-cache.ts +++ b/packages/compat/src/moved-package-cache.ts @@ -1,7 +1,7 @@ -import { join, sep, isAbsolute } from 'path'; -import { ensureSymlinkSync, readdirSync, realpathSync, lstatSync } from 'fs-extra'; +import { join, sep, isAbsolute, resolve } from 'path'; +import { ensureSymlinkSync, readdirSync, realpathSync, lstatSync, outputJSONSync } from 'fs-extra'; import { Memoize } from 'typescript-memoize'; -import { PackageCache, Package, getOrCreate } from '@embroider/core'; +import { PackageCache, Package, getOrCreate, RewrittenPackageIndex } from '@embroider/core'; import { MacrosConfig } from '@embroider/macros/src/node'; import os from 'os'; @@ -87,6 +87,18 @@ export class MovedPackageCache extends PackageCache { this.rootCache = rootCache; this.resolutionCache = resolutionCache; this.unmovedAddons = movedSet.unmovedAddons; + this.writeAddonIndex(); + } + + private writeAddonIndex() { + let indexFile = resolve(this.origApp.root, 'node_modules', '.embroider', 'rewritten-packages', 'index.json'); + let content: RewrittenPackageIndex = { + packages: {}, + }; + for (let [oldPkg, newPkg] of this.moved) { + content.packages[oldPkg.root] = newPkg.root; + } + outputJSONSync(indexFile, content, { spaces: 2 }); } private movedPackage(originalPkg: Package): Package { diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 349028890..94de4f280 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -24,6 +24,7 @@ export { Resolution, ResolverFunction, SyncResolverFunction, + RewrittenPackageIndex, } from './module-resolver'; export { virtualContent } from './virtual-content'; export type { Engine } from './app-files'; diff --git a/packages/core/src/module-resolver.ts b/packages/core/src/module-resolver.ts index 83f56e416..43da3e3a0 100644 --- a/packages/core/src/module-resolver.ts +++ b/packages/core/src/module-resolver.ts @@ -21,6 +21,16 @@ import { describeExports } from './describe-exports'; import { /* existsSync, */ readFileSync } from 'fs'; // import { readJSONSync } from 'fs-extra'; +export interface RewrittenPackageIndex { + // keys are paths to original package root directories. + // + // values are paths to rewritten directories. + // + // all paths are interpreted relative to the rewritten package index file + // itself. + packages: Record; +} + const debug = makeDebug('embroider:resolver'); function logTransition(reason: string, before: R, after: R = before): R { if (after.isVirtual) { @@ -652,10 +662,10 @@ export class Resolver { // let { packages } = readJSONSync(indexFile) as { packages: Record }; // return { // v1ToV2: new Map( - // Object.entries(packages).map(([oldRoot, relativeNewRoot]) => [oldRoot, resolve(addonsDir, relativeNewRoot)]) + // Object.entries(packages).map(([oldRoot, newRoot]) => [resolve(addonsDir, oldRoot), resolve(addonsDir, newRoot)]) // ), // v2toV1: new Map( - // Object.entries(packages).map(([oldRoot, relativeNewRoot]) => [resolve(addonsDir, relativeNewRoot), oldRoot]) + // Object.entries(packages).map(([oldRoot, newRoot]) => [resolve(addonsDir, newRoot), resolve(addonsDir, oldRoot)]) // ), // }; // } From 0ce586ef0734b4c5dce6e75b72fed05d1a2065dc Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Thu, 18 May 2023 15:52:17 -0400 Subject: [PATCH 03/72] creating RewrittenPackageCache that consumes RewrittenPackageIndex --- packages/compat/src/moved-package-cache.ts | 5 + packages/core/src/index.ts | 3 +- packages/core/src/module-resolver.ts | 10 - packages/core/src/rewritten-package-cache.ts | 267 +++++++++++++++++++ packages/shared-internals/src/package.ts | 7 +- 5 files changed, 279 insertions(+), 13 deletions(-) create mode 100644 packages/core/src/rewritten-package-cache.ts diff --git a/packages/compat/src/moved-package-cache.ts b/packages/compat/src/moved-package-cache.ts index d0ad3e9f4..b13dd57dd 100644 --- a/packages/compat/src/moved-package-cache.ts +++ b/packages/compat/src/moved-package-cache.ts @@ -94,9 +94,14 @@ export class MovedPackageCache extends PackageCache { let indexFile = resolve(this.origApp.root, 'node_modules', '.embroider', 'rewritten-packages', 'index.json'); let content: RewrittenPackageIndex = { packages: {}, + extraResolutions: {}, }; for (let [oldPkg, newPkg] of this.moved) { content.packages[oldPkg.root] = newPkg.root; + let nonResolvableDeps = oldPkg.nonResolvableDeps; + if (nonResolvableDeps) { + content.extraResolutions[newPkg.root] = [...nonResolvableDeps.values()].map(v => v.root); + } } outputJSONSync(indexFile, content, { spaces: 2 }); } diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 94de4f280..e8906ac43 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -24,10 +24,11 @@ export { Resolution, ResolverFunction, SyncResolverFunction, - RewrittenPackageIndex, } from './module-resolver'; export { virtualContent } from './virtual-content'; export type { Engine } from './app-files'; +export type { RewrittenPackageIndex } from './rewritten-package-cache'; +export { RewrittenPackageCache } from './rewritten-package-cache'; // this is reexported because we already make users manage a peerDep from some // other packages (like embroider/webpack and @embroider/compat diff --git a/packages/core/src/module-resolver.ts b/packages/core/src/module-resolver.ts index 43da3e3a0..434747ab9 100644 --- a/packages/core/src/module-resolver.ts +++ b/packages/core/src/module-resolver.ts @@ -21,16 +21,6 @@ import { describeExports } from './describe-exports'; import { /* existsSync, */ readFileSync } from 'fs'; // import { readJSONSync } from 'fs-extra'; -export interface RewrittenPackageIndex { - // keys are paths to original package root directories. - // - // values are paths to rewritten directories. - // - // all paths are interpreted relative to the rewritten package index file - // itself. - packages: Record; -} - const debug = makeDebug('embroider:resolver'); function logTransition(reason: string, before: R, after: R = before): R { if (after.isVirtual) { diff --git a/packages/core/src/rewritten-package-cache.ts b/packages/core/src/rewritten-package-cache.ts new file mode 100644 index 000000000..aa44b6641 --- /dev/null +++ b/packages/core/src/rewritten-package-cache.ts @@ -0,0 +1,267 @@ +import { PackageCache, Package } from '@embroider/shared-internals'; +import { existsSync, readJSONSync } from 'fs-extra'; +import { resolve } from 'path'; +import { Memoize } from 'typescript-memoize'; + +export interface RewrittenPackageIndex { + // keys are paths to original package root directories. + // + // values are paths to rewritten directories. + // + // all paths are interpreted relative to the rewritten package index file + // itself. + packages: Record; + + // key is path to the rewritten package that needs to resolve an extra + // dependency + // + // value is list of paths to packages that it should be able to resolve. + // + // while the key is always one of our rewritten packages, the values can be + // rewritten ones or not. + extraResolutions: Record; +} + +// without this, using a class as an interface forces you to have the same +// private and protected methods too (since people trying to extend from you +// could see all of those) +type PublicAPI = { [K in keyof T]: T[K] }; + +// TODO: as our refactor lands we should be able to remove these things from +// PackageCache itself. +type PackageCacheTheGoodParts = Omit, 'basedir' | 'seed' | 'shareAs'>; + +export class RewrittenPackageCache implements PackageCacheTheGoodParts { + constructor(private plainCache: PackageCache) {} + + get appRoot(): string { + return this.plainCache.appRoot; + } + + resolve(packageName: string, fromPackage: Package): Package { + let oldRoot = this.index.newToOld.get(fromPackage.root); + if (!oldRoot) { + // the fromPackage has not been moved, so we're just providing the plain + // behavior. + return this.plainCache.resolve(packageName, fromPackage); + } + + // check for any extraResolutions + let extraResolutions = this.index.extraResolutions.get(fromPackage.root); + if (extraResolutions) { + for (let depRoot of extraResolutions) { + let depPkg = this.plainCache.get(depRoot); + if (depPkg.name === packageName) { + return this.maybeMoved(depPkg); + } + } + } + + // do the real resolving from the old location + let oldSrc = this.plainCache.get(oldRoot); + let oldDest = this.plainCache.resolve(packageName, oldSrc); + + // and if the package we found was itself moved return the moved one. + return this.maybeMoved(oldDest); + } + + // ensure we have the moved version of the package + private maybeMoved(pkg: Package): Package { + let newRoot = this.index.oldToNew.get(pkg.root); + if (newRoot) { + return this.get(newRoot); + } else { + return pkg; + } + } + + get(packageRoot: string): Package { + return this.maybeWrap(this.plainCache.get(packageRoot)); + } + + ownerOfFile(filename: string): Package | undefined { + let owner = this.plainCache.ownerOfFile(filename); + if (owner) { + return this.maybeWrap(owner); + } + } + + @Memoize() + private get index(): { + oldToNew: Map; + newToOld: Map; + extraResolutions: Map; + } { + let addonsDir = resolve(this.appRoot, 'node_modules', '.embroider', 'rewritten-packages'); + let indexFile = resolve(addonsDir, 'index.json'); + if (existsSync(indexFile)) { + let { packages, extraResolutions } = readJSONSync(indexFile) as RewrittenPackageIndex; + return { + oldToNew: new Map( + Object.entries(packages).map(([oldRoot, newRoot]) => [ + resolve(addonsDir, oldRoot), + resolve(addonsDir, newRoot), + ]) + ), + newToOld: new Map( + Object.entries(packages).map(([oldRoot, newRoot]) => [ + resolve(addonsDir, newRoot), + resolve(addonsDir, oldRoot), + ]) + ), + extraResolutions: new Map( + Object.entries(extraResolutions).map(([fromRoot, toRoots]) => [ + resolve(addonsDir, fromRoot), + toRoots.map(r => resolve(addonsDir, r)), + ]) + ), + }; + } + return { oldToNew: new Map(), newToOld: new Map(), extraResolutions: new Map() }; + } + + // put a WrappedPackage around Packages that do in fact represent ones that we + // have moved, leaving other Packages alone. + private maybeWrap(pkg: Package) { + let oldRoot = this.index.newToOld.get(pkg.root); + if (oldRoot) { + let found = wrapped.get(pkg); + if (!found) { + found = new MovedPackage(this, pkg); + wrapped.set(pkg, found); + } + return castToPackage(found); + } else { + return pkg; + } + } +} + +const wrapped = new WeakMap(); + +// TODO: as our refactor lands we should be able to remove this from Package +// itself. +type PackageTheGoodParts = Omit, 'nonResolvableDeps'>; + +// TODO: this goes with the above TODO and can get deleted when it does. +function castToPackage(m: MovedPackage): Package { + return m as unknown as Package; +} + +class MovedPackage implements PackageTheGoodParts { + // plainPkg is not the Package in the original un-moved location, it's the + // plain representation of the moved location. That is, when you grab + // plainPkg.root it will show the new location, and plainPkg.packageJSON shows + // the rewritten package.json contained there. + // + // The point of MovedPackage is to finesse the parts of plainPkg that wouldn't + // be correct if you used them directly. For example, plainPkg.dependencies + // won't necessarily even work, because if you try to resolve them from the + // moved location they might not be there. + constructor(private packageCache: RewrittenPackageCache, private plainPkg: Package) {} + + get root() { + return this.plainPkg.root; + } + + get name() { + return this.plainPkg.name; + } + + get version() { + return this.plainPkg.version; + } + + get packageJSON() { + return this.plainPkg.packageJSON; + } + + get meta() { + return this.plainPkg.meta; + } + + get isEmberPackage() { + return this.plainPkg.isEmberPackage; + } + + get isEngine() { + return this.plainPkg.isEngine; + } + + get isLazyEngine() { + return this.plainPkg.isLazyEngine; + } + + get isV2Ember() { + return this.plainPkg.isV2Ember; + } + + get isV2App() { + return this.plainPkg.isV2App; + } + + get isV2Addon() { + return this.plainPkg.isV2Addon; + } + + // it's important that we're calling this.dependencies here at this level, not + // plainPkg.dependencies, which wouldn't be correct + findDescendants(filter?: (pkg: Package) => boolean): Package[] { + let pkgs = new Set(); + let queue: Package[] = [castToPackage(this)]; + while (true) { + let pkg = queue.shift(); + if (!pkg) { + break; + } + if (!pkgs.has(pkg)) { + pkgs.add(pkg); + let nextLevel; + if (filter) { + nextLevel = pkg.dependencies.filter(filter); + } else { + nextLevel = pkg.dependencies; + } + nextLevel.forEach(d => queue.push(d)); + } + } + pkgs.delete(castToPackage(this)); + return [...pkgs.values()]; + } + + get mayRebuild() { + return this.plainPkg.mayRebuild; + } + + get dependencyNames() { + return this.plainPkg.dependencyNames; + } + + get dependencies() { + return this.plainPkg.dependencyNames + .map(name => { + try { + return this.packageCache.resolve(name, castToPackage(this)); + } catch (error) { + // if the package was not found do not error out here. this is relevant + // for the case where a package might be an optional peerDependency and we dont + // want to error if it was not found. Additionally, erroring here is "far" away + // from the actual logical failure point and so not failing here will provide a better + // error message down the line + if (error.code === 'MODULE_NOT_FOUND') { + return false; + } + + throw error; + } + }) + .filter(Boolean) as Package[]; + } + + hasDependency(name: string): boolean { + // this is *not* extended because it's understood that the rewritten package + // should explictly list the things that need extraResolutions in its own + // package.json.ß + return this.plainPkg.hasDependency(name); + } +} diff --git a/packages/shared-internals/src/package.ts b/packages/shared-internals/src/package.ts index 0113fc2a9..e1e8ebfba 100644 --- a/packages/shared-internals/src/package.ts +++ b/packages/shared-internals/src/package.ts @@ -158,10 +158,13 @@ export default class Package { } } + get dependencyNames(): string[] { + return flatMap(this.dependencyKeys, key => Object.keys(this.packageJSON[key] || {})); + } + @Memoize() get dependencies(): Package[] { - let names = flatMap(this.dependencyKeys, key => Object.keys(this.packageJSON[key] || {})); - return names + return this.dependencyNames .map(name => { if (this.nonResolvableDeps) { let dep = this.nonResolvableDeps.get(name); From 8bccbd77d9531261fb5e12d2a7896ce87aac8a9a Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Fri, 19 May 2023 10:11:11 -0400 Subject: [PATCH 04/72] integrate RewrittenPackageCache with module-resolver --- packages/core/src/module-resolver.ts | 104 +++++++++---------- packages/core/src/rewritten-package-cache.ts | 22 +++- 2 files changed, 71 insertions(+), 55 deletions(-) diff --git a/packages/core/src/module-resolver.ts b/packages/core/src/module-resolver.ts index 434747ab9..5d87f5d0f 100644 --- a/packages/core/src/module-resolver.ts +++ b/packages/core/src/module-resolver.ts @@ -5,7 +5,7 @@ import { packageName as getPackageName, } from '@embroider/shared-internals'; import { dirname, resolve } from 'path'; -import { PackageCache, Package, V2Package, explicitRelative } from '@embroider/shared-internals'; +import { Package, V2Package, explicitRelative } from '@embroider/shared-internals'; import makeDebug from 'debug'; import assertNever from 'assert-never'; import resolveModule from 'resolve'; @@ -18,8 +18,8 @@ import { } from './virtual-content'; import { Memoize } from 'typescript-memoize'; import { describeExports } from './describe-exports'; -import { /* existsSync, */ readFileSync } from 'fs'; -// import { readJSONSync } from 'fs-extra'; +import { readFileSync } from 'fs'; +import { RewrittenPackageCache } from './rewritten-package-cache'; const debug = makeDebug('embroider:resolver'); function logTransition(reason: string, before: R, after: R = before): R { @@ -156,7 +156,7 @@ export class Resolver { request = this.handleFastbootCompat(request); request = this.handleGlobalsCompat(request); - request = this.handleLegacyAddons(request); + request = this.handleRewrittenPackages(request); request = this.handleRenaming(request); return this.preHandleExternal(request); } @@ -272,13 +272,15 @@ export class Resolver { } owningPackage(fromFile: string): Package | undefined { - return PackageCache.shared('embroider-stage3', this.options.appRoot).ownerOfFile(fromFile); + return RewrittenPackageCache.shared('embroider-stage3', this.options.appRoot).ownerOfFile(fromFile); } private logicalPackage(owningPackage: V2Package, file: string): V2Package { let logicalLocation = this.reverseSearchAppTree(owningPackage, file); if (logicalLocation) { - let pkg = PackageCache.shared('embroider-stage3', this.options.appRoot).get(logicalLocation.owningEngine.root); + let pkg = RewrittenPackageCache.shared('embroider-stage3', this.options.appRoot).get( + logicalLocation.owningEngine.root + ); if (!pkg.isV2Ember()) { throw new Error(`bug: all engines should be v2 addons by the time we see them here`); } @@ -481,7 +483,7 @@ export class Resolver { // out. @Memoize() private get mergeMap(): MergeMap { - let packageCache = PackageCache.shared('embroider-stage3', this.options.appRoot); + let packageCache = RewrittenPackageCache.shared('embroider-stage3', this.options.appRoot); let result: MergeMap = new Map(); for (let engine of this.options.engines) { let engineModules: Map = new Map(); @@ -600,66 +602,55 @@ export class Resolver { return owningEngine; } - private handleLegacyAddons(request: R): R { - let packageCache = PackageCache.shared('embroider-stage3', this.options.appRoot); + private handleRewrittenPackages(request: R): R { + let packageCache = RewrittenPackageCache.shared('embroider-stage3', this.options.appRoot); - // first we handle output requests from moved packages - let pkg = this.owningPackage(request.fromFile); - if (!pkg) { + let requestingPkg = this.owningPackage(request.fromFile); + if (!requestingPkg) { return request; } - let originalRoot = this.legacyAddonsIndex.v2toV1.get(pkg.root); - if (originalRoot) { - request = logTransition( - 'outbound from moved v1 addon', - request, - request.rehome(resolve(originalRoot, 'package.json')) - ); - pkg = packageCache.get(originalRoot)!; - } - // then we handle inbound requests to moved packages + let targetPkg: Package | undefined; let packageName = getPackageName(request.specifier); - if (packageName && packageName !== pkg.name) { + if (packageName && packageName !== requestingPkg.name) { // non-relative, non-self request, so check if it aims at a rewritten addon try { - let target = PackageCache.shared('embroider-stage3', this.options.appRoot).resolve(packageName, pkg); - if (target) { - let movedRoot = this.legacyAddonsIndex.v1ToV2.get(target.root); - if (movedRoot) { - request = logTransition( - 'inbound to moved v1 addon', - request, - this.resolveWithinPackage(request, packageCache.get(movedRoot)) - ); - } - } + // up above we already ensured that the source `pkg` here is always referring to the *original* copy if it had been rewritten. + targetPkg = RewrittenPackageCache.shared('embroider-stage3', this.options.appRoot).resolve( + packageName, + requestingPkg + ); } catch (err) { + // this is not the place to report resolution failures. If the thing + // doesn't resolve, we're just not interested in redirecting it for + // backward-compat, that's all. The rest of the system will take care of + // reporting a failure to resolve (or handling it a different way) if (err.code !== 'MODULE_NOT_FOUND') { throw err; } } } - return request; - } + let originalRequestingPkg = packageCache.original(requestingPkg); + let originalTargetPkg = targetPkg ? packageCache.original(targetPkg) : undefined; + + if (targetPkg && originalTargetPkg) { + // in this case it doesn't matter whether or not the requesting package + // was moved. RewrittenPackageCache.resolve already took care of finding + // the right target, and we redirect the request so it will look inside + // that target. + return logTransition('request targets a moved package', request, this.resolveWithinPackage(request, targetPkg)); + } else if (originalRequestingPkg) { + // in this case, the requesting package is moved but its destination is + // not, so we need to rehome the request back to the original location. + return logTransition( + 'outbound request from moved package', + request, + request.rehome(resolve(originalRequestingPkg.root, 'package.json')) + ); + } - @Memoize() - private get legacyAddonsIndex(): { v1ToV2: Map; v2toV1: Map } { - // let addonsDir = resolve(this.options.appRoot, 'node_modules', '.embroider', 'rewritten-packages'); - // let indexFile = resolve(addonsDir, 'index.json'); - // if (existsSync(indexFile)) { - // let { packages } = readJSONSync(indexFile) as { packages: Record }; - // return { - // v1ToV2: new Map( - // Object.entries(packages).map(([oldRoot, newRoot]) => [resolve(addonsDir, oldRoot), resolve(addonsDir, newRoot)]) - // ), - // v2toV1: new Map( - // Object.entries(packages).map(([oldRoot, newRoot]) => [resolve(addonsDir, newRoot), resolve(addonsDir, oldRoot)]) - // ), - // }; - // } - return { v1ToV2: new Map(), v2toV1: new Map() }; + return request; } private handleRenaming(request: R): R { @@ -793,7 +784,10 @@ export class Resolver { if (logicalPackage.meta['auto-upgraded'] && !logicalPackage.hasDependency('ember-auto-import')) { try { - let dep = PackageCache.shared('embroider-stage3', this.options.appRoot).resolve(packageName, logicalPackage); + let dep = RewrittenPackageCache.shared('embroider-stage3', this.options.appRoot).resolve( + packageName, + logicalPackage + ); if (!dep.isEmberPackage()) { // classic ember addons can only import non-ember dependencies if they // have ember-auto-import. @@ -880,7 +874,9 @@ export class Resolver { request, this.resolveWithinPackage( request, - PackageCache.shared('embroider-stage3', this.options.appRoot).get(this.options.activeAddons[packageName]) + RewrittenPackageCache.shared('embroider-stage3', this.options.appRoot).get( + this.options.activeAddons[packageName] + ) ) ); } diff --git a/packages/core/src/rewritten-package-cache.ts b/packages/core/src/rewritten-package-cache.ts index aa44b6641..55c0e0356 100644 --- a/packages/core/src/rewritten-package-cache.ts +++ b/packages/core/src/rewritten-package-cache.ts @@ -1,4 +1,4 @@ -import { PackageCache, Package } from '@embroider/shared-internals'; +import { PackageCache, Package, getOrCreate } from '@embroider/shared-internals'; import { existsSync, readJSONSync } from 'fs-extra'; import { resolve } from 'path'; import { Memoize } from 'typescript-memoize'; @@ -79,6 +79,13 @@ export class RewrittenPackageCache implements PackageCacheTheGoodParts { return this.maybeWrap(this.plainCache.get(packageRoot)); } + original(pkg: Package): Package | undefined { + let oldRoot = this.index.newToOld.get(pkg.root); + if (oldRoot) { + return this.plainCache.get(oldRoot); + } + } + ownerOfFile(filename: string): Package | undefined { let owner = this.plainCache.ownerOfFile(filename); if (owner) { @@ -95,6 +102,7 @@ export class RewrittenPackageCache implements PackageCacheTheGoodParts { let addonsDir = resolve(this.appRoot, 'node_modules', '.embroider', 'rewritten-packages'); let indexFile = resolve(addonsDir, 'index.json'); if (existsSync(indexFile)) { + // I should probably make the else case throw here soon. let { packages, extraResolutions } = readJSONSync(indexFile) as RewrittenPackageIndex; return { oldToNew: new Map( @@ -135,8 +143,20 @@ export class RewrittenPackageCache implements PackageCacheTheGoodParts { return pkg; } } + static shared(identifier: string, appRoot: string) { + let pk = getOrCreate( + shared, + identifier + appRoot, + () => new RewrittenPackageCache(PackageCache.shared(identifier, appRoot)) + ); + if (pk.appRoot !== appRoot) { + throw new Error(`bug: PackageCache appRoot disagreement ${appRoot}!=${pk.appRoot}`); + } + return pk; + } } +const shared: Map = new Map(); const wrapped = new WeakMap(); // TODO: as our refactor lands we should be able to remove this from Package From 9425b0690627a270c0c6c2d654decea929bfdf0d Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Mon, 22 May 2023 12:09:42 -0400 Subject: [PATCH 05/72] Eliminate BuildStage It existed primarily to power our node modules rewriting. --- packages/compat/src/compat-app.ts | 72 +++++++++++++++++++++++++++-- packages/core/src/build-stage.ts | 76 ------------------------------- packages/core/src/index.ts | 1 - 3 files changed, 68 insertions(+), 81 deletions(-) delete mode 100644 packages/core/src/build-stage.ts diff --git a/packages/compat/src/compat-app.ts b/packages/compat/src/compat-app.ts index bffc12e00..f9f6ec1c8 100644 --- a/packages/compat/src/compat-app.ts +++ b/packages/compat/src/compat-app.ts @@ -4,7 +4,6 @@ import { Stage, PackageCache, OutputPaths, - BuildStage, Asset, EmberAsset, AppAdapter, @@ -13,6 +12,7 @@ import { Package, AddonPackage, Engine, + WaitForTrees, } from '@embroider/core'; import V1InstanceCache from './v1-instance-cache'; import V1App from './v1-app'; @@ -399,10 +399,74 @@ class CompatAppAdapter implements AppAdapter { } } -export default class CompatApp extends BuildStage { - constructor(legacyEmberAppInstance: object, addons: Stage, options?: Options) { +interface BuilderInstance { + build(inputPaths: OutputPaths): Promise; +} + +interface ExtraTree { + __prevStageTree: BroccoliNode; +} + +export default class CompatApp { + private inTrees: TreeNames; + private annotation = '@embroider/compat/app'; + private instantiate: ( + root: string, + appSrcDir: string, + packageCache: PackageCache + ) => Promise>; + + private active: BuilderInstance | undefined; + private outputPath: string | undefined; + private packageCache: PackageCache | undefined; + + constructor(legacyEmberAppInstance: object, private prevStage: Stage, options?: Options) { let { inTrees, instantiate } = setup(legacyEmberAppInstance, optionsWithDefaults(options)); - super(addons, inTrees, '@embroider/compat/app', instantiate); + this.inTrees = inTrees; + this.instantiate = instantiate; + } + + @Memoize() + get tree(): BroccoliNode { + return new WaitForTrees(this.augment(this.inTrees), this.annotation, async treePaths => { + if (!this.active) { + let { outputPath, packageCache } = await this.prevStage.ready(); + this.outputPath = outputPath; + this.packageCache = packageCache; + this.active = await this.instantiate(outputPath, this.prevStage.inputPath, packageCache); + } + delete (treePaths as any).__prevStageTree; + await this.active.build(this.deAugment(treePaths)); + this.deferReady.resolve(); + }); + } + + get inputPath(): string { + return this.prevStage.inputPath; + } + + async ready(): Promise<{ outputPath: string; packageCache: PackageCache }> { + await this.deferReady.promise; + return { + outputPath: this.outputPath!, + packageCache: this.packageCache!, + }; + } + + @Memoize() + private get deferReady() { + let resolve: Function; + let promise: Promise = new Promise(r => (resolve = r)); + return { resolve: resolve!, promise }; + } + + private augment(inTrees: TreeNames): TreeNames & ExtraTree { + return Object.assign({ __prevStageTree: this.prevStage.tree }, inTrees); + } + + private deAugment(treePaths: OutputPaths): OutputPaths { + delete (treePaths as any).__prevStageTree; + return treePaths; } } diff --git a/packages/core/src/build-stage.ts b/packages/core/src/build-stage.ts deleted file mode 100644 index 35ac64f05..000000000 --- a/packages/core/src/build-stage.ts +++ /dev/null @@ -1,76 +0,0 @@ -import WaitForTrees, { OutputPaths } from './wait-for-trees'; -import { PackageCache } from '@embroider/shared-internals'; -import Stage from './stage'; -import { Node } from 'broccoli-node-api'; -import { Memoize } from 'typescript-memoize'; - -// This is a utility class for defining new Stages. It aids in handling the -// boilerplate required to split your functionality between the -// broccoli-pipeline-construction phase and the actual building phase. -export default class BuildStage implements Stage { - private active: BuilderInstance | undefined; - private outputPath: string | undefined; - private packageCache: PackageCache | undefined; - - constructor( - private prevStage: Stage, - private inTrees: NamedTrees, - private annotation: string, - private instantiate: ( - root: string, - appSrcDir: string, - packageCache: PackageCache - ) => Promise> - ) {} - - @Memoize() - get tree(): Node { - return new WaitForTrees(this.augment(this.inTrees), this.annotation, async treePaths => { - if (!this.active) { - let { outputPath, packageCache } = await this.prevStage.ready(); - this.outputPath = outputPath; - this.packageCache = packageCache; - this.active = await this.instantiate(outputPath, this.prevStage.inputPath, packageCache); - } - delete (treePaths as any).__prevStageTree; - await this.active.build(this.deAugment(treePaths)); - this.deferReady.resolve(); - }); - } - - get inputPath(): string { - return this.prevStage.inputPath; - } - - async ready(): Promise<{ outputPath: string; packageCache: PackageCache }> { - await this.deferReady.promise; - return { - outputPath: this.outputPath!, - packageCache: this.packageCache!, - }; - } - - @Memoize() - private get deferReady() { - let resolve: Function; - let promise: Promise = new Promise(r => (resolve = r)); - return { resolve: resolve!, promise }; - } - - private augment(inTrees: NamedTrees): NamedTrees & ExtraTree { - return Object.assign({ __prevStageTree: this.prevStage.tree }, inTrees); - } - - private deAugment(treePaths: OutputPaths): OutputPaths { - delete (treePaths as any).__prevStageTree; - return treePaths; - } -} - -interface BuilderInstance { - build(inputPaths: OutputPaths): Promise; -} - -interface ExtraTree { - __prevStageTree: Node; -} diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index e8906ac43..5ef0a821b 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -12,7 +12,6 @@ export { Asset, EmberAsset, ImplicitAssetPaths } from './asset'; export { default as Options, optionsWithDefaults } from './options'; export { default as toBroccoliPlugin } from './to-broccoli-plugin'; export { default as WaitForTrees, OutputPaths } from './wait-for-trees'; -export { default as BuildStage } from './build-stage'; export { compile as jsHandlebarsCompile } from './js-handlebars'; export { AppAdapter, AppBuilder, EmberENV } from './app'; export { todo, unsupported, warn, debug, expectWarning, throwOnWarnings } from './messages'; From b21c81d5d882dd04b07383844e667b7085272091 Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Mon, 22 May 2023 13:01:03 -0400 Subject: [PATCH 06/72] Simplifying compat app adapter pattern The original intent here was a guess about what part we'd want to swap out for a v2 app. But it assumed incorrectly that there would still be broccoli driving the v2 app. --- packages/compat/package.json | 2 + packages/compat/src/compat-app.ts | 1436 +++++++++++++++++++++++++++- packages/compat/src/v1-config.ts | 3 +- packages/core/src/app.ts | 1474 ----------------------------- packages/core/src/index.ts | 1 - packages/core/tests/app.test.ts | 55 -- pnpm-lock.yaml | 10 +- 7 files changed, 1400 insertions(+), 1581 deletions(-) delete mode 100644 packages/core/src/app.ts delete mode 100644 packages/core/tests/app.test.ts diff --git a/packages/compat/package.json b/packages/compat/package.json index d759bf408..fa411972b 100644 --- a/packages/compat/package.json +++ b/packages/compat/package.json @@ -47,6 +47,8 @@ "broccoli-source": "^3.0.1", "chalk": "^4.1.1", "debug": "^4.3.2", + "escape-string-regexp": "^4.0.0", + "fast-sourcemap-concat": "^1.4.0", "fs-extra": "^9.1.0", "fs-tree-diff": "^2.0.1", "jsdom": "^16.6.0", diff --git a/packages/compat/src/compat-app.ts b/packages/compat/src/compat-app.ts index f9f6ec1c8..854d7e57a 100644 --- a/packages/compat/src/compat-app.ts +++ b/packages/compat/src/compat-app.ts @@ -6,34 +6,65 @@ import { OutputPaths, Asset, EmberAsset, - AppAdapter, - AppBuilder, - EmberENV, Package, AddonPackage, Engine, WaitForTrees, + AddonMeta, + AppMeta, + explicitRelative, + extensionsPattern, + PackageInfo, + TemplateColocationPluginOptions, + debug, + warn, + jsHandlebarsCompile, } from '@embroider/core'; import V1InstanceCache from './v1-instance-cache'; import V1App from './v1-app'; import walkSync from 'walk-sync'; -import { join } from 'path'; +import { dirname, join, resolve as resolvePath, sep, posix } from 'path'; import { JSDOM } from 'jsdom'; +import resolve from 'resolve'; import { V1Config } from './v1-config'; -import { statSync, readdirSync } from 'fs'; import Options, { optionsWithDefaults } from './options'; import { CompatResolverOptions } from './resolver-transform'; import { activePackageRules, PackageRules } from './dependency-rules'; import flatMap from 'lodash/flatMap'; +import sortBy from 'lodash/sortBy'; +import flatten from 'lodash/flatten'; +import partition from 'lodash/partition'; +import mergeWith from 'lodash/mergeWith'; +import cloneDeep from 'lodash/cloneDeep'; import { Memoize } from 'typescript-memoize'; import { sync as resolveSync } from 'resolve'; import { MacrosConfig } from '@embroider/macros/src/node'; import bind from 'bind-decorator'; -import { pathExistsSync } from 'fs-extra'; -import type { Transform } from 'babel-plugin-ember-template-compilation'; +import { + pathExistsSync, + copySync, + ensureDirSync, + outputJSONSync, + readJSONSync, + statSync, + readdirSync, + unlinkSync, + writeFileSync, +} from 'fs-extra'; +import type { Options as EtcOptions, Transform } from 'babel-plugin-ember-template-compilation'; import type { Options as ResolverTransformOptions } from './resolver-transform'; import type { Options as AdjustImportsOptions } from './babel-plugin-adjust-imports'; -import type { PluginItem } from '@babel/core'; +import type { PluginItem, TransformOptions } from '@babel/core'; +import { PreparedEmberHTML } from '@embroider/core/src/ember-html'; +import { InMemoryAsset, OnDiskAsset, ImplicitAssetPaths } from '@embroider/core/src/asset'; +import { makePortable } from '@embroider/core/src/portable-babel-config'; +import { AppFiles, EngineSummary, RouteFiles } from '@embroider/core/src/app-files'; +import { mangledEngineRoot } from '@embroider/core/src/engine-mangler'; +import { PortableHint, maybeNodeModuleVersion } from '@embroider/core/src/portable'; +import AppDiffer from '@embroider/core/src/app-differ'; +import SourceMapConcat from 'fast-sourcemap-concat'; +import assertNever from 'assert-never'; +import escapeRegExp from 'escape-string-regexp'; interface TreeNames { appJS: BroccoliNode; @@ -42,6 +73,52 @@ interface TreeNames { configTree: BroccoliNode; } +class ParsedEmberAsset { + kind: 'parsed-ember' = 'parsed-ember'; + relativePath: string; + fileAsset: EmberAsset; + html: PreparedEmberHTML; + + constructor(asset: EmberAsset) { + this.fileAsset = asset; + this.html = new PreparedEmberHTML(asset); + this.relativePath = asset.relativePath; + } + + validFor(other: EmberAsset) { + return this.fileAsset.mtime === other.mtime && this.fileAsset.size === other.size; + } +} + +type EmberENV = unknown; + +class BuiltEmberAsset { + kind: 'built-ember' = 'built-ember'; + relativePath: string; + parsedAsset: ParsedEmberAsset; + source: string; + + constructor(asset: ParsedEmberAsset) { + this.parsedAsset = asset; + this.source = asset.html.dom.serialize(); + this.relativePath = asset.relativePath; + } +} + +class ConcatenatedAsset { + kind: 'concatenated-asset' = 'concatenated-asset'; + constructor( + public relativePath: string, + public sources: (OnDiskAsset | InMemoryAsset)[], + private resolvableExtensions: RegExp + ) {} + get sourcemapPath() { + return this.relativePath.replace(this.resolvableExtensions, '') + '.map'; + } +} + +type InternalAsset = OnDiskAsset | InMemoryAsset | BuiltEmberAsset | ConcatenatedAsset; + // This runs at broccoli-pipeline-construction time, whereas our actual // CompatAppAdapter instance only becomes available during tree-building // time. @@ -68,29 +145,27 @@ function setup(legacyEmberAppInstance: object, options: Required) { let instantiate = async (root: string, appSrcDir: string, packageCache: PackageCache) => { let appPackage = packageCache.get(appSrcDir); - let adapter = new CompatAppAdapter( + let macrosConfig = MacrosConfig.for(legacyEmberAppInstance, appSrcDir); + + return new CompatAppAdapter( root, appPackage, options, oldPackage, configTree, packageCache.get(join(root, 'node_modules', '@embroider', 'synthesized-vendor')), - packageCache.get(join(root, 'node_modules', '@embroider', 'synthesized-styles')) - ); - - return new AppBuilder( - root, - appPackage, - adapter, - options, - MacrosConfig.for(legacyEmberAppInstance, appSrcDir) + packageCache.get(join(root, 'node_modules', '@embroider', 'synthesized-styles')), + macrosConfig ); }; return { inTrees, instantiate }; } -class CompatAppAdapter implements AppAdapter { +class CompatAppAdapter { + // for each relativePath, an Asset we have already emitted + private assets: Map = new Map(); + constructor( private root: string, private appPackage: Package, @@ -98,26 +173,37 @@ class CompatAppAdapter implements AppAdapter { private oldPackage: V1App, private configTree: V1Config, private synthVendor: Package, - private synthStyles: Package - ) {} + private synthStyles: Package, + private macrosConfig: MacrosConfig + ) { + // this uses globalConfig because it's a way for packages to ask "is + // Embroider doing this build?". So it's necessarily global, not scoped to + // any subgraph of dependencies. + macrosConfig.setGlobalConfig(__filename, `@embroider/core`, { + // this is hard-coded to true because it literally means "embroider is + // building this Ember app". You can see non-true when using the Embroider + // macros in a classic build. + active: true, + }); + } - appJSSrcDir(treePaths: OutputPaths) { + private appJSSrcDir(treePaths: OutputPaths) { return treePaths.appJS; } @Memoize() - fastbootJSSrcDir(_treePaths: OutputPaths) { + private fastbootJSSrcDir(_treePaths: OutputPaths) { let target = join(this.oldPackage.root, 'fastboot'); if (pathExistsSync(target)) { return target; } } - get env() { + private get env() { return this.oldPackage.env; } - assets(treePaths: OutputPaths): Asset[] { + extractAssets(treePaths: OutputPaths): Asset[] { let assets: Asset[] = []; // Everything in our traditional public tree is an on-disk asset @@ -171,15 +257,14 @@ class CompatAppAdapter implements AppAdapter { } } - developingAddons(): string[] { + private developingAddons(): string[] { if (this.oldPackage.owningAddon) { return [this.oldPackage.owningAddon.root]; } return []; } - @Memoize() - activeAddonChildren(pkg: Package = this.appPackage): AddonPackage[] { + private activeAddonChildren(pkg: Package = this.appPackage): AddonPackage[] { let result = (pkg.dependencies.filter(this.isActiveAddon) as AddonPackage[]).filter( // When looking for child addons, we want to ignore 'peerDependencies' of // a given package, to align with how ember-cli resolves addons. So here @@ -195,7 +280,7 @@ class CompatAppAdapter implements AppAdapter { } @Memoize() - get allActiveAddons(): AddonPackage[] { + private get allActiveAddons(): AddonPackage[] { let result = this.appPackage.findDescendants(this.isActiveAddon) as AddonPackage[]; let extras = [this.synthVendor, this.synthStyles].filter(this.isActiveAddon) as AddonPackage[]; let extraDescendants = flatMap(extras, dep => dep.findDescendants(this.isActiveAddon)) as AddonPackage[]; @@ -224,8 +309,7 @@ class CompatAppAdapter implements AppAdapter { return depAIdx - depBIdx; } - @Memoize() - resolvableExtensions(): string[] { + private resolvableExtensions(): string[] { // webpack's default is ['.wasm', '.mjs', '.js', '.json']. Keeping that // subset in that order is sensible, since many third-party libraries will // expect it to work that way. @@ -274,43 +358,43 @@ class CompatAppAdapter implements AppAdapter { } } - autoRun(): boolean { + private autoRun(): boolean { return this.oldPackage.autoRun; } - appBoot(): string | undefined { + private appBoot(): string | undefined { return this.oldPackage.appBoot.readAppBoot(); } - mainModule(): string { + private mainModule(): string { return 'app'; } - mainModuleConfig(): unknown { + private mainModuleConfig(): unknown { return this.configTree.readConfig().APP; } - emberENV(): EmberENV { + private emberENV(): EmberENV { return this.configTree.readConfig().EmberENV; } - modulePrefix(): string { + private modulePrefix(): string { return this.configTree.readConfig().modulePrefix; } - podModulePrefix(): string | undefined { + private podModulePrefix(): string | undefined { return this.configTree.readConfig().podModulePrefix; } - rootURL(): string { + private rootURL(): string { return this.configTree.readConfig().rootURL; } - templateCompilerPath(): string { + private templateCompilerPath(): string { return 'ember-source/vendor/ember/ember-template-compiler'; } - strictV2Format() { + private strictV2Format() { return false; } @@ -322,7 +406,7 @@ class CompatAppAdapter implements AppAdapter { ]); } - hbsTransforms(resolverConfig: CompatResolverOptions): Transform[] { + private hbsTransforms(resolverConfig: CompatResolverOptions): Transform[] { if ( this.options.staticComponents || this.options.staticHelpers || @@ -338,14 +422,14 @@ class CompatAppAdapter implements AppAdapter { } } - jsPlugins(resolverConfig: CompatResolverOptions): PluginItem[] { + private jsPlugins(resolverConfig: CompatResolverOptions): PluginItem[] { let pluginConfig: AdjustImportsOptions = { appRoot: resolverConfig.appRoot, }; return [[require.resolve('./babel-plugin-adjust-imports'), pluginConfig]]; } - resolverConfig(engines: Engine[]): CompatResolverOptions { + private resolverConfig(engines: Engine[]): CompatResolverOptions { let renamePackages = Object.assign({}, ...this.allActiveAddons.map(dep => dep.meta['renamed-packages'])); let renameModules = Object.assign({}, ...this.allActiveAddons.map(dep => dep.meta['renamed-modules'])); @@ -386,17 +470,1094 @@ class CompatAppAdapter implements AppAdapter { return config; } - htmlbarsPlugins(): Transform[] { + private htmlbarsPlugins(): Transform[] { return this.oldPackage.htmlbarsPlugins; } - babelMajorVersion() { + private babelMajorVersion() { return this.oldPackage.babelMajorVersion(); } - babelConfig() { + private scriptPriority(pkg: Package) { + switch (pkg.name) { + case 'loader.js': + return 0; + case 'ember-source': + return 10; + default: + return 1000; + } + } + + @Memoize() + private get resolvableExtensionsPattern(): RegExp { + return extensionsPattern(this.resolvableExtensions()); + } + + private impliedAssets( + type: keyof ImplicitAssetPaths, + engine: Engine, + emberENV?: EmberENV + ): (OnDiskAsset | InMemoryAsset)[] { + let result: (OnDiskAsset | InMemoryAsset)[] = this.impliedAddonAssets(type, engine).map( + (sourcePath: string): OnDiskAsset => { + let stats = statSync(sourcePath); + return { + kind: 'on-disk', + relativePath: explicitRelative(this.root, sourcePath), + sourcePath, + mtime: stats.mtimeMs, + size: stats.size, + }; + } + ); + + if (type === 'implicit-scripts') { + result.unshift({ + kind: 'in-memory', + relativePath: '_testing_prefix_.js', + source: `var runningTests=false;`, + }); + + result.unshift({ + kind: 'in-memory', + relativePath: '_ember_env_.js', + source: `window.EmberENV={ ...(window.EmberENV || {}), ...${JSON.stringify(emberENV, null, 2)} };`, + }); + + result.push({ + kind: 'in-memory', + relativePath: '_loader_.js', + source: `loader.makeDefaultExport=false;`, + }); + } + + if (type === 'implicit-test-scripts') { + // this is the traditional test-support-suffix.js + result.push({ + kind: 'in-memory', + relativePath: '_testing_suffix_.js', + source: ` + var runningTests=true; + if (typeof Testem !== 'undefined' && (typeof QUnit !== 'undefined' || typeof Mocha !== 'undefined')) { + Testem.hookIntoTestFramework(); + }`, + }); + + // whether or not anybody was actually using @embroider/macros + // explicitly as an addon, we ensure its test-support file is always + // present. + if (!result.find(s => s.kind === 'on-disk' && s.sourcePath.endsWith('embroider-macros-test-support.js'))) { + result.unshift({ + kind: 'on-disk', + sourcePath: require.resolve('@embroider/macros/src/vendor/embroider-macros-test-support'), + mtime: 0, + size: 0, + relativePath: 'embroider-macros-test-support.js', + }); + } + } + + return result; + } + + private impliedAddonAssets(type: keyof ImplicitAssetPaths, engine: Engine): string[] { + let result: Array = []; + for (let addon of sortBy(Array.from(engine.addons), this.scriptPriority.bind(this))) { + let implicitScripts = addon.meta[type]; + if (implicitScripts) { + let styles = []; + let options = { basedir: addon.root }; + for (let mod of implicitScripts) { + if (type === 'implicit-styles') { + // exclude engines because they will handle their own css importation + if (!addon.isLazyEngine()) { + styles.push(resolve.sync(mod, options)); + } + } else { + result.push(resolve.sync(mod, options)); + } + } + if (styles.length) { + result = [...styles, ...result]; + } + } + } + return result; + } + + private originalBabelConfig() { return this.oldPackage.babelConfig(); } + + // unlike our full config, this one just needs to know how to parse all the + // syntax our app can contain. + @Memoize() + private babelParserConfig(): TransformOptions { + let babel = cloneDeep(this.originalBabelConfig()); + + if (!babel.plugins) { + babel.plugins = []; + } + + // Our stage3 code is always allowed to use dynamic import. We may emit it + // ourself when splitting routes. + babel.plugins.push(require.resolve('@babel/plugin-syntax-dynamic-import')); + return babel; + } + + @Memoize() + private babelConfig(resolverConfig: CompatResolverOptions) { + let babel = cloneDeep(this.originalBabelConfig()); + + if (!babel.plugins) { + babel.plugins = []; + } + + // Our stage3 code is always allowed to use dynamic import. We may emit it + // ourself when splitting routes. + babel.plugins.push(require.resolve('@babel/plugin-syntax-dynamic-import')); + + // https://github.com/webpack/webpack/issues/12154 + babel.plugins.push(require.resolve('./rename-require-plugin')); + + babel.plugins.push([require.resolve('babel-plugin-ember-template-compilation'), this.etcOptions(resolverConfig)]); + + // this is @embroider/macros configured for full stage3 resolution + babel.plugins.push(...this.macrosConfig.babelPluginConfig()); + + let colocationOptions: TemplateColocationPluginOptions = { + appRoot: this.root, + + // This extra weirdness is a compromise in favor of build performance. + // + // 1. When auto-upgrading an addon from v1 to v2, we definitely want to + // run any custom AST transforms in stage1. + // + // 2. In general case, AST transforms are allowed to manipulate Javascript + // scope. This means that running transforms -- even when we're doing + // source-to-source compilation that emits handlebars and not wire + // format -- implies changing .hbs files into .js files. + // + // 3. So stage1 may need to rewrite .hbs to .hbs.js (to avoid colliding + // with an existing co-located .js file). + // + // 4. But stage1 doesn't necessarily want to run babel over the + // corresponding JS file. Most of the time, that's just an + // unnecessarily expensive second parse. (We only run it in stage1 to + // eliminate an addon's custom babel plugins, and many addons don't + // have any.) + // + // 5. Therefore, the work of template-colocation gets defered until here, + // and it may see co-located templates named `.hbs.js` instead of the + // usual `.hbs. + templateExtensions: ['.hbs', '.hbs.js'], + + // All of the above only applies to auto-upgraded packages that were + // authored in v1. V2 packages don't get any of this complexity, they're + // supposed to take care of colocating their own templates explicitly. + packageGuard: true, + }; + babel.plugins.push([ + require.resolve('@embroider/shared-internals/src/template-colocation-plugin'), + colocationOptions, + ]); + + for (let p of this.jsPlugins(resolverConfig)) { + babel.plugins.push(p); + } + + // we can use globally shared babel runtime by default + babel.plugins.push([ + require.resolve('@babel/plugin-transform-runtime'), + { absoluteRuntime: __dirname, useESModules: true, regenerator: false }, + ]); + + const portable = makePortable(babel, { basedir: this.root }, this.portableHints); + addCachablePlugin(portable.config); + return portable; + } + + private insertEmberApp( + asset: ParsedEmberAsset, + appFiles: Engine[], + prepared: Map, + emberENV: EmberENV + ) { + let html = asset.html; + + if (this.fastbootConfig) { + // ignore scripts like ember-cli-livereload.js which are not really associated with + // "the app". + let ignoreScripts = html.dom.window.document.querySelectorAll('script'); + ignoreScripts.forEach(script => { + script.setAttribute('data-fastboot-ignore', ''); + }); + } + + // our tests entrypoint already includes a correct module dependency on the + // app, so we only insert the app when we're not inserting tests + if (!asset.fileAsset.includeTests) { + let appJS = this.topAppJSAsset(appFiles, prepared); + html.insertScriptTag(html.javascript, appJS.relativePath, { type: 'module' }); + } + + if (this.fastbootConfig) { + // any extra fastboot app files get inserted into our html.javascript + // section, after the app has been inserted. + for (let script of this.fastbootConfig.extraAppFiles) { + html.insertScriptTag(html.javascript, script, { tag: 'fastboot-script' }); + } + } + + html.insertStyleLink(html.styles, `assets/${this.appPackage.name}.css`); + + const parentEngine = appFiles.find(e => !e.parent) as Engine; + let vendorJS = this.implicitScriptsAsset(prepared, parentEngine, emberENV); + if (vendorJS) { + html.insertScriptTag(html.implicitScripts, vendorJS.relativePath); + } + + if (this.fastbootConfig) { + // any extra fastboot vendor files get inserted into our + // html.implicitScripts section, after the regular implicit script + // (vendor.js) have been inserted. + for (let script of this.fastbootConfig.extraVendorFiles) { + html.insertScriptTag(html.implicitScripts, script, { tag: 'fastboot-script' }); + } + } + + let implicitStyles = this.implicitStylesAsset(prepared, parentEngine); + if (implicitStyles) { + html.insertStyleLink(html.implicitStyles, implicitStyles.relativePath); + } + + if (!asset.fileAsset.includeTests) { + return; + } + + // Test-related assets happen below this point + + let testJS = this.testJSEntrypoint(appFiles, prepared); + html.insertScriptTag(html.testJavascript, testJS.relativePath, { type: 'module' }); + + let implicitTestScriptsAsset = this.implicitTestScriptsAsset(prepared, parentEngine); + if (implicitTestScriptsAsset) { + html.insertScriptTag(html.implicitTestScripts, implicitTestScriptsAsset.relativePath); + } + + let implicitTestStylesAsset = this.implicitTestStylesAsset(prepared, parentEngine); + if (implicitTestStylesAsset) { + html.insertStyleLink(html.implicitTestStyles, implicitTestStylesAsset.relativePath); + } + } + + private implicitScriptsAsset( + prepared: Map, + application: Engine, + emberENV: EmberENV + ): InternalAsset | undefined { + let asset = prepared.get('assets/vendor.js'); + if (!asset) { + let implicitScripts = this.impliedAssets('implicit-scripts', application, emberENV); + if (implicitScripts.length > 0) { + asset = new ConcatenatedAsset('assets/vendor.js', implicitScripts, this.resolvableExtensionsPattern); + prepared.set(asset.relativePath, asset); + } + } + return asset; + } + + private implicitStylesAsset(prepared: Map, application: Engine): InternalAsset | undefined { + let asset = prepared.get('assets/vendor.css'); + if (!asset) { + let implicitStyles = this.impliedAssets('implicit-styles', application); + if (implicitStyles.length > 0) { + // we reverse because we want the synthetic vendor style at the top + asset = new ConcatenatedAsset('assets/vendor.css', implicitStyles.reverse(), this.resolvableExtensionsPattern); + prepared.set(asset.relativePath, asset); + } + } + return asset; + } + + private implicitTestScriptsAsset( + prepared: Map, + application: Engine + ): InternalAsset | undefined { + let testSupportJS = prepared.get('assets/test-support.js'); + if (!testSupportJS) { + let implicitTestScripts = this.impliedAssets('implicit-test-scripts', application); + if (implicitTestScripts.length > 0) { + testSupportJS = new ConcatenatedAsset( + 'assets/test-support.js', + implicitTestScripts, + this.resolvableExtensionsPattern + ); + prepared.set(testSupportJS.relativePath, testSupportJS); + } + } + return testSupportJS; + } + + private implicitTestStylesAsset( + prepared: Map, + application: Engine + ): InternalAsset | undefined { + let asset = prepared.get('assets/test-support.css'); + if (!asset) { + let implicitTestStyles = this.impliedAssets('implicit-test-styles', application); + if (implicitTestStyles.length > 0) { + asset = new ConcatenatedAsset('assets/test-support.css', implicitTestStyles, this.resolvableExtensionsPattern); + prepared.set(asset.relativePath, asset); + } + } + return asset; + } + + // recurse to find all active addons that don't cross an engine boundary. + // Inner engines themselves will be returned, but not those engines' children. + // The output set's insertion order is the proper ember-cli compatible + // ordering of the addons. + private findActiveAddons(pkg: Package, engine: EngineSummary, isChild = false): void { + for (let child of this.activeAddonChildren(pkg)) { + if (!child.isEngine()) { + this.findActiveAddons(child, engine, true); + } + engine.addons.add(child); + } + // ensure addons are applied in the correct order, if set (via @embroider/compat/v1-addon) + if (!isChild) { + engine.addons = new Set( + [...engine.addons].sort((a, b) => { + return (a.meta['order-index'] || 0) - (b.meta['order-index'] || 0); + }) + ); + } + } + + private partitionEngines(appJSPath: string): EngineSummary[] { + let queue: EngineSummary[] = [ + { + package: this.appPackage, + addons: new Set(), + parent: undefined, + sourcePath: appJSPath, + destPath: this.root, + modulePrefix: this.modulePrefix(), + appRelativePath: '.', + }, + ]; + let done: EngineSummary[] = []; + let seenEngines: Set = new Set(); + while (true) { + let current = queue.shift(); + if (!current) { + break; + } + this.findActiveAddons(current.package, current); + for (let addon of current.addons) { + if (addon.isEngine() && !seenEngines.has(addon)) { + seenEngines.add(addon); + queue.push({ + package: addon, + addons: new Set(), + parent: current, + sourcePath: mangledEngineRoot(addon), + destPath: addon.root, + modulePrefix: addon.name, + appRelativePath: explicitRelative(this.root, addon.root), + }); + } + } + done.push(current); + } + return done; + } + + @Memoize() + private get activeFastboot() { + return this.activeAddonChildren(this.appPackage).find(a => a.name === 'ember-cli-fastboot'); + } + + @Memoize() + private get fastbootConfig(): + | { packageJSON: PackageInfo; extraAppFiles: string[]; extraVendorFiles: string[] } + | undefined { + if (this.activeFastboot) { + // this is relying on work done in stage1 by @embroider/compat/src/compat-adapters/ember-cli-fastboot.ts + let packageJSON = readJSONSync(join(this.activeFastboot.root, '_fastboot_', 'package.json')); + let { extraAppFiles, extraVendorFiles } = packageJSON['embroider-fastboot']; + delete packageJSON['embroider-fastboot']; + extraVendorFiles.push('assets/embroider_macros_fastboot_init.js'); + return { packageJSON, extraAppFiles, extraVendorFiles }; + } + } + + private appDiffers: { differ: AppDiffer; engine: EngineSummary }[] | undefined; + + private updateAppJS(inputPaths: OutputPaths): Engine[] { + let appJSPath = this.appJSSrcDir(inputPaths); + if (!this.appDiffers) { + let engines = this.partitionEngines(appJSPath); + this.appDiffers = engines.map(engine => { + let differ: AppDiffer; + if (this.activeFastboot) { + differ = new AppDiffer( + engine.destPath, + engine.sourcePath, + [...engine.addons], + true, + this.fastbootJSSrcDir(inputPaths), + this.babelParserConfig() + ); + } else { + differ = new AppDiffer(engine.destPath, engine.sourcePath, [...engine.addons]); + } + return { + differ, + engine, + }; + }); + } + // this is in reverse order because we need deeper engines to update before + // their parents, because they aren't really valid packages until they + // update, and their parents will go looking for their own `app-js` content. + this.appDiffers + .slice() + .reverse() + .forEach(a => a.differ.update()); + return this.appDiffers.map(a => { + return { + ...a.engine, + appFiles: new AppFiles(a.differ, this.resolvableExtensionsPattern, this.podModulePrefix()), + }; + }); + } + + private prepareAsset(asset: Asset, appFiles: Engine[], prepared: Map, emberENV: EmberENV) { + if (asset.kind === 'ember') { + let prior = this.assets.get(asset.relativePath); + let parsed: ParsedEmberAsset; + if (prior && prior.kind === 'built-ember' && prior.parsedAsset.validFor(asset)) { + // we can reuse the parsed html + parsed = prior.parsedAsset; + parsed.html.clear(); + } else { + parsed = new ParsedEmberAsset(asset); + } + this.insertEmberApp(parsed, appFiles, prepared, emberENV); + prepared.set(asset.relativePath, new BuiltEmberAsset(parsed)); + } else { + prepared.set(asset.relativePath, asset); + } + } + + private prepareAssets(requestedAssets: Asset[], appFiles: Engine[], emberENV: EmberENV): Map { + let prepared: Map = new Map(); + for (let asset of requestedAssets) { + this.prepareAsset(asset, appFiles, prepared, emberENV); + } + return prepared; + } + + private assetIsValid(asset: InternalAsset, prior: InternalAsset | undefined): boolean { + if (!prior) { + return false; + } + switch (asset.kind) { + case 'on-disk': + return prior.kind === 'on-disk' && prior.size === asset.size && prior.mtime === asset.mtime; + case 'in-memory': + return prior.kind === 'in-memory' && stringOrBufferEqual(prior.source, asset.source); + case 'built-ember': + return prior.kind === 'built-ember' && prior.source === asset.source; + case 'concatenated-asset': + return ( + prior.kind === 'concatenated-asset' && + prior.sources.length === asset.sources.length && + prior.sources.every((priorFile, index) => { + let newFile = asset.sources[index]; + return this.assetIsValid(newFile, priorFile); + }) + ); + } + } + + private updateOnDiskAsset(asset: OnDiskAsset) { + let destination = join(this.root, asset.relativePath); + ensureDirSync(dirname(destination)); + copySync(asset.sourcePath, destination, { dereference: true }); + } + + private updateInMemoryAsset(asset: InMemoryAsset) { + let destination = join(this.root, asset.relativePath); + ensureDirSync(dirname(destination)); + writeFileSync(destination, asset.source, 'utf8'); + } + + private updateBuiltEmberAsset(asset: BuiltEmberAsset) { + let destination = join(this.root, asset.relativePath); + ensureDirSync(dirname(destination)); + writeFileSync(destination, asset.source, 'utf8'); + } + + private async updateConcatenatedAsset(asset: ConcatenatedAsset) { + let concat = new SourceMapConcat({ + outputFile: join(this.root, asset.relativePath), + mapCommentType: asset.relativePath.endsWith('.js') ? 'line' : 'block', + baseDir: this.root, + }); + if (process.env.EMBROIDER_CONCAT_STATS) { + let MeasureConcat = (await import('@embroider/core/src/measure-concat')).default; + concat = new MeasureConcat(asset.relativePath, concat, this.root); + } + for (let source of asset.sources) { + switch (source.kind) { + case 'on-disk': + concat.addFile(explicitRelative(this.root, source.sourcePath)); + break; + case 'in-memory': + if (typeof source.source !== 'string') { + throw new Error(`attempted to concatenated a Buffer-backed in-memory asset`); + } + concat.addSpace(source.source); + break; + default: + assertNever(source); + } + } + await concat.end(); + } + + private async updateAssets(requestedAssets: Asset[], appFiles: Engine[], emberENV: EmberENV) { + let assets = this.prepareAssets(requestedAssets, appFiles, emberENV); + for (let asset of assets.values()) { + if (this.assetIsValid(asset, this.assets.get(asset.relativePath))) { + continue; + } + debug('rebuilding %s', asset.relativePath); + switch (asset.kind) { + case 'on-disk': + this.updateOnDiskAsset(asset); + break; + case 'in-memory': + this.updateInMemoryAsset(asset); + break; + case 'built-ember': + this.updateBuiltEmberAsset(asset); + break; + case 'concatenated-asset': + await this.updateConcatenatedAsset(asset); + break; + default: + assertNever(asset); + } + } + for (let oldAsset of this.assets.values()) { + if (!assets.has(oldAsset.relativePath)) { + unlinkSync(join(this.root, oldAsset.relativePath)); + } + } + this.assets = assets; + return [...assets.values()]; + } + + private gatherAssets(inputPaths: OutputPaths): Asset[] { + // first gather all the assets out of addons + let assets: Asset[] = []; + for (let pkg of this.allActiveAddons) { + if (pkg.meta['public-assets']) { + for (let [filename, appRelativeURL] of Object.entries(pkg.meta['public-assets'] || {})) { + let sourcePath = resolvePath(pkg.root, filename); + let stats = statSync(sourcePath); + assets.push({ + kind: 'on-disk', + sourcePath, + relativePath: appRelativeURL, + mtime: stats.mtimeMs, + size: stats.size, + }); + } + } + } + + if (this.activeFastboot) { + const source = ` + (function(){ + var key = '_embroider_macros_runtime_config'; + if (!window[key]){ window[key] = [];} + window[key].push(function(m) { + m.setGlobalConfig('fastboot', Object.assign({}, m.getGlobalConfig().fastboot, { isRunning: true })); + }); + }())`; + assets.push({ + kind: 'in-memory', + source, + relativePath: 'assets/embroider_macros_fastboot_init.js', + }); + } + + // and finally tack on the ones from our app itself + return assets.concat(this.extractAssets(inputPaths)); + } + + async build(inputPaths: OutputPaths) { + if (this.env !== 'production') { + this.macrosConfig.enablePackageDevelopment(this.root); + this.macrosConfig.enableRuntimeMode(); + } + for (let pkgRoot of this.developingAddons()) { + this.macrosConfig.enablePackageDevelopment(pkgRoot); + } + + // on the first build, we lock down the macros config. on subsequent builds, + // this doesn't do anything anyway because it's idempotent. + this.macrosConfig.finalize(); + + let appFiles = this.updateAppJS(inputPaths); + let emberENV = this.emberENV(); + let assets = this.gatherAssets(inputPaths); + + let finalAssets = await this.updateAssets(assets, appFiles, emberENV); + + let assetPaths = assets.map(asset => asset.relativePath); + + if (this.activeFastboot) { + // when using fastboot, our own package.json needs to be in the output so fastboot can read it. + assetPaths.push('package.json'); + } + + for (let asset of finalAssets) { + // our concatenated assets all have map files that ride along. Here we're + // telling the final stage packager to be sure and serve the map files + // too. + if (asset.kind === 'concatenated-asset') { + assetPaths.push(asset.sourcemapPath); + } + } + + let meta: AppMeta = { + type: 'app', + version: 2, + assets: assetPaths, + babel: { + filename: '_babel_config_.js', + isParallelSafe: true, // TODO + majorVersion: this.babelMajorVersion(), + fileFilter: '_babel_filter_.js', + }, + 'root-url': this.rootURL(), + }; + + if (!this.strictV2Format()) { + meta['auto-upgraded'] = true; + } + + let pkg = this.combinePackageJSON(meta); + writeFileSync(join(this.root, 'package.json'), JSON.stringify(pkg, null, 2), 'utf8'); + + let resolverConfig = this.resolverConfig(appFiles); + this.addResolverConfig(resolverConfig); + let babelConfig = this.babelConfig(resolverConfig); + this.addBabelConfig(babelConfig); + } + + private combinePackageJSON(meta: AppMeta): object { + let pkgLayers: any[] = [this.appPackage.packageJSON]; + let fastbootConfig = this.fastbootConfig; + if (fastbootConfig) { + // fastboot-specific package.json output is allowed to add to our original package.json + pkgLayers.push(fastbootConfig.packageJSON); + } + // but our own new v2 app metadata takes precedence over both + pkgLayers.push({ keywords: ['ember-addon'], 'ember-addon': meta }); + return combinePackageJSON(...pkgLayers); + } + + private etcOptions(resolverConfig: CompatResolverOptions): EtcOptions { + let transforms = this.htmlbarsPlugins(); + + let { plugins: macroPlugins, setConfig } = MacrosConfig.transforms(); + setConfig(this.macrosConfig); + for (let macroPlugin of macroPlugins) { + transforms.push(macroPlugin as any); + } + + for (let t of this.hbsTransforms(resolverConfig)) { + transforms.push(t); + } + + return { + transforms, + compilerPath: resolve.sync(this.templateCompilerPath(), { basedir: this.root }), + enableLegacyModules: ['ember-cli-htmlbars', 'ember-cli-htmlbars-inline-precompile', 'htmlbars-inline-precompile'], + }; + } + + @Memoize() + private get portableHints(): PortableHint[] { + return this.options.pluginHints.map(hint => { + let cursor = join(this.appPackage.root, 'package.json'); + for (let i = 0; i < hint.resolve.length; i++) { + let target = hint.resolve[i]; + if (i < hint.resolve.length - 1) { + target = join(target, 'package.json'); + } + cursor = resolve.sync(target, { basedir: dirname(cursor) }); + } + + return { + requireFile: cursor, + useMethod: hint.useMethod, + packageVersion: maybeNodeModuleVersion(cursor), + }; + }); + } + + private addBabelConfig(pconfig: { config: TransformOptions; isParallelSafe: boolean }) { + if (!pconfig.isParallelSafe) { + warn('Your build is slower because some babel plugins are non-serializable'); + } + writeFileSync( + join(this.root, '_babel_config_.js'), + `module.exports = ${JSON.stringify(pconfig.config, null, 2)}`, + 'utf8' + ); + writeFileSync( + join(this.root, '_babel_filter_.js'), + babelFilterTemplate({ skipBabel: this.options.skipBabel, appRoot: this.root }), + 'utf8' + ); + } + + private addResolverConfig(config: CompatResolverOptions) { + outputJSONSync(join(this.root, '.embroider', 'resolver.json'), config); + } + + private shouldSplitRoute(routeName: string) { + return ( + !this.options.splitAtRoutes || + this.options.splitAtRoutes.find(pattern => { + if (typeof pattern === 'string') { + return pattern === routeName; + } else { + return pattern.test(routeName); + } + }) + ); + } + + private splitRoute( + routeName: string, + files: RouteFiles, + addToParent: (routeName: string, filename: string) => void, + addLazyBundle: (routeNames: string[], files: string[]) => void + ) { + let shouldSplit = routeName && this.shouldSplitRoute(routeName); + let ownFiles = []; + let ownNames = new Set() as Set; + + if (files.template) { + if (shouldSplit) { + ownFiles.push(files.template); + ownNames.add(routeName); + } else { + addToParent(routeName, files.template); + } + } + + if (files.controller) { + if (shouldSplit) { + ownFiles.push(files.controller); + ownNames.add(routeName); + } else { + addToParent(routeName, files.controller); + } + } + + if (files.route) { + if (shouldSplit) { + ownFiles.push(files.route); + ownNames.add(routeName); + } else { + addToParent(routeName, files.route); + } + } + + for (let [childName, childFiles] of files.children) { + this.splitRoute( + `${routeName}.${childName}`, + childFiles, + + (childRouteName: string, childFile: string) => { + // this is our child calling "addToParent" + if (shouldSplit) { + ownFiles.push(childFile); + ownNames.add(childRouteName); + } else { + addToParent(childRouteName, childFile); + } + }, + (routeNames: string[], files: string[]) => { + addLazyBundle(routeNames, files); + } + ); + } + + if (ownFiles.length > 0) { + addLazyBundle([...ownNames], ownFiles); + } + } + + private topAppJSAsset(engines: Engine[], prepared: Map): InternalAsset { + let [app, ...childEngines] = engines; + let relativePath = `assets/${this.appPackage.name}.js`; + return this.appJSAsset(relativePath, app, childEngines, prepared, { + autoRun: this.autoRun(), + appBoot: !this.autoRun() ? this.appBoot() : '', + mainModule: explicitRelative(dirname(relativePath), this.mainModule()), + appConfig: this.mainModuleConfig(), + }); + } + + @Memoize() + private get staticAppPathsPattern(): RegExp | undefined { + if (this.options.staticAppPaths.length > 0) { + return new RegExp( + '^(?:' + this.options.staticAppPaths.map(staticAppPath => escapeRegExp(staticAppPath)).join('|') + ')(?:$|/)' + ); + } + } + + private requiredOtherFiles(appFiles: AppFiles): readonly string[] { + let pattern = this.staticAppPathsPattern; + if (pattern) { + return appFiles.otherAppFiles.filter(f => { + return !pattern!.test(f); + }); + } else { + return appFiles.otherAppFiles; + } + } + + private appJSAsset( + relativePath: string, + engine: Engine, + childEngines: Engine[], + prepared: Map, + entryParams?: Partial[0]> + ): InternalAsset { + let { appFiles } = engine; + let cached = prepared.get(relativePath); + if (cached) { + return cached; + } + + let eagerModules = []; + + let requiredAppFiles = [this.requiredOtherFiles(appFiles)]; + if (!this.options.staticComponents) { + requiredAppFiles.push(appFiles.components); + } + if (!this.options.staticHelpers) { + requiredAppFiles.push(appFiles.helpers); + } + if (!this.options.staticModifiers) { + requiredAppFiles.push(appFiles.modifiers); + } + + let styles = []; + // only import styles from engines with a parent (this excludeds the parent application) as their styles + // will be inserted via a direct tag. + if (engine.parent && engine.package.isLazyEngine()) { + let implicitStyles = this.impliedAssets('implicit-styles', engine); + for (let style of implicitStyles) { + styles.push({ + path: explicitRelative('assets/_engine_', style.relativePath), + }); + } + + let engineMeta = engine.package.meta as AddonMeta; + if (engineMeta && engineMeta['implicit-styles']) { + for (let style of engineMeta['implicit-styles']) { + styles.push({ + path: explicitRelative(dirname(relativePath), join(engine.appRelativePath, style)), + }); + } + } + } + + let lazyEngines: { names: string[]; path: string }[] = []; + for (let childEngine of childEngines) { + let asset = this.appJSAsset( + `assets/_engine_/${encodeURIComponent(childEngine.package.name)}.js`, + childEngine, + [], + prepared + ); + if (childEngine.package.isLazyEngine()) { + lazyEngines.push({ + names: [childEngine.package.name], + path: explicitRelative(dirname(relativePath), asset.relativePath), + }); + } else { + eagerModules.push(explicitRelative(dirname(relativePath), asset.relativePath)); + } + } + let lazyRoutes: { names: string[]; path: string }[] = []; + for (let [routeName, routeFiles] of appFiles.routeFiles.children) { + this.splitRoute( + routeName, + routeFiles, + (_: string, filename: string) => { + requiredAppFiles.push([filename]); + }, + (routeNames: string[], files: string[]) => { + let routeEntrypoint = `assets/_route_/${encodeURIComponent(routeNames[0])}.js`; + if (!prepared.has(routeEntrypoint)) { + prepared.set(routeEntrypoint, this.routeEntrypoint(engine, routeEntrypoint, files)); + } + lazyRoutes.push({ + names: routeNames, + path: this.importPaths(engine, routeEntrypoint).buildtime, + }); + } + ); + } + + let [fastboot, nonFastboot] = partition(excludeDotFiles(flatten(requiredAppFiles)), file => + appFiles.isFastbootOnly.get(file) + ); + let amdModules = nonFastboot.map(file => this.importPaths(engine, file)); + let fastbootOnlyAmdModules = fastboot.map(file => this.importPaths(engine, file)); + + // this is a backward-compatibility feature: addons can force inclusion of + // modules. + this.gatherImplicitModules('implicit-modules', engine, amdModules); + + let params = { amdModules, fastbootOnlyAmdModules, lazyRoutes, lazyEngines, eagerModules, styles }; + if (entryParams) { + Object.assign(params, entryParams); + } + + let source = entryTemplate(params); + + let asset: InternalAsset = { + kind: 'in-memory', + source, + relativePath, + }; + prepared.set(relativePath, asset); + return asset; + } + + private importPaths(engine: Engine, engineRelativePath: string) { + let noHBS = engineRelativePath.replace(this.resolvableExtensionsPattern, '').replace(/\.hbs$/, ''); + return { + runtime: `${engine.modulePrefix}/${noHBS}`, + buildtime: posix.join(engine.package.name, engineRelativePath), + }; + } + + private routeEntrypoint(engine: Engine, relativePath: string, files: string[]) { + let [fastboot, nonFastboot] = partition(files, file => engine.appFiles.isFastbootOnly.get(file)); + + let asset: InternalAsset = { + kind: 'in-memory', + source: routeEntryTemplate({ + files: nonFastboot.map(f => this.importPaths(engine, f)), + fastbootOnlyFiles: fastboot.map(f => this.importPaths(engine, f)), + }), + relativePath, + }; + return asset; + } + + private testJSEntrypoint(engines: Engine[], prepared: Map): InternalAsset { + let asset = prepared.get(`assets/test.js`); + if (asset) { + return asset; + } + + // We're only building tests from the first engine (the app). This is the + // normal thing to do -- tests from engines don't automatically roll up into + // the app. + let engine = engines[0]; + + const myName = 'assets/test.js'; + + // tests necessarily also include the app. This is where we account for + // that. The classic solution was to always include the app's separate + // script tag in the tests HTML, but that isn't as easy for final stage + // packagers to understand. It's better to express it here as a direct + // module dependency. + let eagerModules: string[] = [ + explicitRelative(dirname(myName), this.topAppJSAsset(engines, prepared).relativePath), + ]; + + let amdModules: { runtime: string; buildtime: string }[] = []; + // this is a backward-compatibility feature: addons can force inclusion of + // test support modules. + this.gatherImplicitModules('implicit-test-modules', engine, amdModules); + + let { appFiles } = engine; + for (let relativePath of appFiles.tests) { + amdModules.push(this.importPaths(engine, relativePath)); + } + + let source = entryTemplate({ + amdModules, + eagerModules, + testSuffix: true, + }); + + asset = { + kind: 'in-memory', + source, + relativePath: myName, + }; + prepared.set(asset.relativePath, asset); + return asset; + } + + private gatherImplicitModules( + section: 'implicit-modules' | 'implicit-test-modules', + engine: Engine, + lazyModules: { runtime: string; buildtime: string }[] + ) { + for (let addon of engine.addons) { + let implicitModules = addon.meta[section]; + if (implicitModules) { + let renamedModules = inverseRenamedModules(addon.meta, this.resolvableExtensionsPattern); + for (let name of implicitModules) { + let packageName = addon.name; + + if (addon.isV2Addon()) { + let renamedMeta = addon.meta['renamed-packages']; + if (renamedMeta) { + Object.entries(renamedMeta).forEach(([key, value]) => { + if (value === addon!.name) { + packageName = key; + } + }); + } + } + + let runtime = join(packageName, name).replace(this.resolvableExtensionsPattern, ''); + let runtimeRenameLookup = runtime.split('\\').join('/'); + if (renamedModules && renamedModules[runtimeRenameLookup]) { + runtime = renamedModules[runtimeRenameLookup]; + } + runtime = runtime.split(sep).join('/'); + lazyModules.push({ + runtime, + buildtime: posix.join(packageName, name), + }); + } + } + } + } } interface BuilderInstance { @@ -493,3 +1654,184 @@ function defaultAddonPackageRules(): PackageRules[] { .filter(Boolean) .reduce((a, b) => a.concat(b), []); } + +const entryTemplate = jsHandlebarsCompile(` +import { importSync as i, macroCondition, getGlobalConfig } from '@embroider/macros'; +let w = window; +let d = w.define; + +{{#if styles}} + if (macroCondition(!getGlobalConfig().fastboot?.isRunning)) { + {{#each styles as |stylePath| ~}} + i("{{js-string-escape stylePath.path}}"); + {{/each}} + } +{{/if}} + +{{#each amdModules as |amdModule| ~}} + d("{{js-string-escape amdModule.runtime}}", function(){ return i("{{js-string-escape amdModule.buildtime}}");}); +{{/each}} + +{{#if fastbootOnlyAmdModules}} + if (macroCondition(getGlobalConfig().fastboot?.isRunning)) { + {{#each fastbootOnlyAmdModules as |amdModule| ~}} + d("{{js-string-escape amdModule.runtime}}", function(){ return i("{{js-string-escape amdModule.buildtime}}");}); + {{/each}} + } +{{/if}} + +{{#each eagerModules as |eagerModule| ~}} + i("{{js-string-escape eagerModule}}"); +{{/each}} + +{{#if lazyRoutes}} +w._embroiderRouteBundles_ = [ + {{#each lazyRoutes as |route|}} + { + names: {{{json-stringify route.names}}}, + load: function() { + return import("{{js-string-escape route.path}}"); + } + }, + {{/each}} +] +{{/if}} + +{{#if lazyEngines}} +w._embroiderEngineBundles_ = [ + {{#each lazyEngines as |engine|}} + { + names: {{{json-stringify engine.names}}}, + load: function() { + return import("{{js-string-escape engine.path}}"); + } + }, + {{/each}} +] +{{/if}} + +{{#if autoRun ~}} +if (!runningTests) { + i("{{js-string-escape mainModule}}").default.create({{{json-stringify appConfig}}}); +} +{{else if appBoot ~}} + {{{ appBoot }}} +{{/if}} + +{{#if testSuffix ~}} + {{!- TODO: both of these suffixes should get dynamically generated so they incorporate + any content-for added by addons. -}} + + + {{!- this is the traditional tests-suffix.js -}} + i('../tests/test-helper'); + EmberENV.TESTS_FILE_LOADED = true; +{{/if}} +`) as (params: { + amdModules: { runtime: string; buildtime: string }[]; + fastbootOnlyAmdModules?: { runtime: string; buildtime: string }[]; + eagerModules?: string[]; + autoRun?: boolean; + appBoot?: string; + mainModule?: string; + appConfig?: unknown; + testSuffix?: boolean; + lazyRoutes?: { names: string[]; path: string }[]; + lazyEngines?: { names: string[]; path: string }[]; + styles?: { path: string }[]; +}) => string; + +const routeEntryTemplate = jsHandlebarsCompile(` +import { importSync as i } from '@embroider/macros'; +let d = window.define; +{{#each files as |amdModule| ~}} +d("{{js-string-escape amdModule.runtime}}", function(){ return i("{{js-string-escape amdModule.buildtime}}");}); +{{/each}} +{{#if fastbootOnlyFiles}} + import { macroCondition, getGlobalConfig } from '@embroider/macros'; + if (macroCondition(getGlobalConfig().fastboot?.isRunning)) { + {{#each fastbootOnlyFiles as |amdModule| ~}} + d("{{js-string-escape amdModule.runtime}}", function(){ return i("{{js-string-escape amdModule.buildtime}}");}); + {{/each}} + } +{{/if}} +`) as (params: { + files: { runtime: string; buildtime: string }[]; + fastbootOnlyFiles: { runtime: string; buildtime: string }[]; +}) => string; + +function stringOrBufferEqual(a: string | Buffer, b: string | Buffer): boolean { + if (typeof a === 'string' && typeof b === 'string') { + return a === b; + } + if (a instanceof Buffer && b instanceof Buffer) { + return Buffer.compare(a, b) === 0; + } + return false; +} + +const babelFilterTemplate = jsHandlebarsCompile(` +const { babelFilter } = require(${JSON.stringify(require.resolve('./index.js'))}); +module.exports = babelFilter({{{json-stringify skipBabel}}}, "{{{js-string-escape appRoot}}}"); +`) as (params: { skipBabel: Options['skipBabel']; appRoot: string }) => string; + +// meta['renamed-modules'] has mapping from classic filename to real filename. +// This takes that and converts it to the inverst mapping from real import path +// to classic import path. +function inverseRenamedModules(meta: AddonPackage['meta'], extensions: RegExp) { + let renamed = meta['renamed-modules']; + if (renamed) { + let inverted = {} as { [name: string]: string }; + for (let [classic, real] of Object.entries(renamed)) { + inverted[real.replace(extensions, '')] = classic.replace(extensions, ''); + } + return inverted; + } +} + +function combinePackageJSON(...layers: object[]) { + function custom(objValue: any, srcValue: any, key: string, _object: any, _source: any, stack: { size: number }) { + if (key === 'keywords' && stack.size === 0) { + if (Array.isArray(objValue)) { + return objValue.concat(srcValue); + } + } + } + return mergeWith({}, ...layers, custom); +} + +const CACHE_BUSTING_PLUGIN = { + path: require.resolve('@embroider/shared-internals/src/babel-plugin-cache-busting.js'), + version: readJSONSync(`${__dirname}/../package.json`).version, +}; + +function addCachablePlugin(babelConfig: TransformOptions) { + if (Array.isArray(babelConfig.plugins) && babelConfig.plugins.length > 0) { + const plugins = Object.create(null); + plugins[CACHE_BUSTING_PLUGIN.path] = CACHE_BUSTING_PLUGIN.version; + + for (const plugin of babelConfig.plugins) { + let absolutePathToPlugin: string; + if (Array.isArray(plugin) && typeof plugin[0] === 'string') { + absolutePathToPlugin = plugin[0] as string; + } else if (typeof plugin === 'string') { + absolutePathToPlugin = plugin; + } else { + throw new Error(`[Embroider] a babel plugin without an absolute path was from: ${plugin}`); + } + + plugins[absolutePathToPlugin] = maybeNodeModuleVersion(absolutePathToPlugin); + } + + babelConfig.plugins.push([ + CACHE_BUSTING_PLUGIN.path, + { + plugins, + }, + ]); + } +} + +function excludeDotFiles(files: string[]) { + return files.filter(file => !file.startsWith('.') && !file.includes('/.')); +} diff --git a/packages/compat/src/v1-config.ts b/packages/compat/src/v1-config.ts index fe20f27eb..091a13dd3 100644 --- a/packages/compat/src/v1-config.ts +++ b/packages/compat/src/v1-config.ts @@ -2,12 +2,11 @@ import Plugin from 'broccoli-plugin'; import { Node } from 'broccoli-node-api'; import { join } from 'path'; import { readFileSync, outputFileSync } from 'fs-extra'; -import { EmberENV } from '@embroider/core'; export interface ConfigContents { modulePrefix: string; podModulePrefix?: string; - EmberENV: EmberENV; + EmberENV: unknown; APP: unknown; rootURL: string; } diff --git a/packages/core/src/app.ts b/packages/core/src/app.ts deleted file mode 100644 index f6acd12df..000000000 --- a/packages/core/src/app.ts +++ /dev/null @@ -1,1474 +0,0 @@ -import { - AddonMeta, - AppMeta, - Package, - AddonPackage, - explicitRelative, - extensionsPattern, - PackageInfo, -} from '@embroider/shared-internals'; -import { OutputPaths } from './wait-for-trees'; -import { compile } from './js-handlebars'; -import resolve from 'resolve'; -import { Memoize } from 'typescript-memoize'; -import { copySync, ensureDirSync, outputJSONSync, readJSONSync, statSync, unlinkSync, writeFileSync } from 'fs-extra'; -import { dirname, join, resolve as resolvePath, sep, posix } from 'path'; -import { debug, warn } from './messages'; -import sortBy from 'lodash/sortBy'; -import flatten from 'lodash/flatten'; -import AppDiffer from './app-differ'; -import { PreparedEmberHTML } from './ember-html'; -import { Asset, EmberAsset, ImplicitAssetPaths, InMemoryAsset, OnDiskAsset } from './asset'; -import assertNever from 'assert-never'; -import SourceMapConcat from 'fast-sourcemap-concat'; -import Options from './options'; -import { MacrosConfig } from '@embroider/macros/src/node'; -import { PluginItem, TransformOptions } from '@babel/core'; -import { makePortable } from './portable-babel-config'; -import { Options as ResolverConfig } from './module-resolver'; -import { mangledEngineRoot } from './engine-mangler'; -import { AppFiles, Engine, EngineSummary, RouteFiles } from './app-files'; -import partition from 'lodash/partition'; -import mergeWith from 'lodash/mergeWith'; -import cloneDeep from 'lodash/cloneDeep'; -import { PortableHint, maybeNodeModuleVersion } from './portable'; -import escapeRegExp from 'escape-string-regexp'; -import type { Options as EtcOptions, Transform } from 'babel-plugin-ember-template-compilation'; -import type { Options as ColocationOptions } from '@embroider/shared-internals/src/template-colocation-plugin'; - -export type EmberENV = unknown; - -/* - This interface is the boundary between the general-purpose build system in - AppBuilder and the messy specifics of apps. - - - CompatAppAdapter in `@embroider/compat` implements this interface for - building based of a legacy ember-cli EmberApp instance - - We will want to make a different class that implements this interface for - building apps that don't need an EmberApp instance at all (presumably - because they opt into new authoring standards. -*/ -export interface AppAdapter { - // the set of all addon packages that are active (recursive) - readonly allActiveAddons: AddonPackage[]; - - // the direct active addon dependencies of a given package - activeAddonChildren(pkg: Package): AddonPackage[]; - - // path to the directory where the app's own Javascript lives. Doesn't include - // any files copied out of addons, we take care of that generically in - // AppBuilder. - appJSSrcDir(treePaths: OutputPaths): string; - - // path to the directory where the app's own Fastboot-only Javascript lives. - // Doesn't include any files copied out of addons, we take care of that - // generically in AppBuilder. - fastbootJSSrcDir(treePaths: OutputPaths): string | undefined; - - // this is where you declare what assets must be in the final output - // (especially index.html, tests/index.html, and anything from your classic - // public tree). - assets(treePaths: OutputPaths): Asset[]; - - // whether the ember app should boot itself automatically - autoRun(): boolean; - - // custom app-boot logic when the autoRun is set to false - appBoot(): string | undefined; - - // the ember app's main module - mainModule(): string; - - // the configuration that will get passed into the ember app's main module. - // This traditionally comes from the `APP` property returned by - // config/environment.js. - mainModuleConfig(): unknown; - - // The namespace for the app's own modules at runtime. - // - // (For apps, we _do_ still allow this to be arbitrary. This is in contrast - // with _addons_, which absolutley must use their real NPM package name as - // their modulePrefix.) - modulePrefix(): string; - - // The module prefix when pods file layout is used - podModulePrefix(): string | undefined; - - // The public URL at which your app will be served. - rootURL(): string; - - // The path to ember's template compiler source - templateCompilerPath(): string; - - // extra handlebars transforms needed by the compat layer - hbsTransforms(resolverConfig: SpecificResolverConfig): Transform[]; - - // extra babel plugins needed by the compat layer. Distinct from - // `babelConfg()` because that's supposed to be the underlying app's original - // babel config. - jsPlugins(resolverConfig: SpecificResolverConfig): PluginItem[]; - - // describes the special module naming rules that we need to achieve - // compatibility - resolverConfig(engines: Engine[]): SpecificResolverConfig; - - resolvableExtensions(): string[]; - - // The template preprocessor plugins that are configured in the app. - htmlbarsPlugins(): Transform[]; - - // the app's preferred babel config. No need to worry about making it portable - // yet, we will do that for you. - babelConfig(): TransformOptions; - - // the babel version that works with your babelConfig. - babelMajorVersion(): 7; - - // The environment settings used to control Ember itself. In a classic app, - // this comes from the EmberENV property returned by config/environment.js. - emberENV(): EmberENV; - - // when true, the app's own code is understood to already follow v2 standards. - // For example, all imports of templates have an explicit `hbs` extension, and - // all imports of your own package use relative imports instead of you rown - // name. When false, your code is treated more leniently and you get the - // auto-upgraded behaviors that v1 addons also get. - strictV2Format(): boolean; - - // list of directories that point to the roots of addon packages that are - // under active development - developingAddons(): string[]; - - // development, test, or production - env: string; -} - -export function excludeDotFiles(files: string[]) { - return files.filter(file => !file.startsWith('.') && !file.includes('/.')); -} - -export const CACHE_BUSTING_PLUGIN = { - path: require.resolve('@embroider/shared-internals/src/babel-plugin-cache-busting.js'), - version: readJSONSync(`${__dirname}/../package.json`).version, -}; - -export function addCachablePlugin(babelConfig: TransformOptions) { - if (Array.isArray(babelConfig.plugins) && babelConfig.plugins.length > 0) { - const plugins = Object.create(null); - plugins[CACHE_BUSTING_PLUGIN.path] = CACHE_BUSTING_PLUGIN.version; - - for (const plugin of babelConfig.plugins) { - let absolutePathToPlugin: string; - if (Array.isArray(plugin) && typeof plugin[0] === 'string') { - absolutePathToPlugin = plugin[0] as string; - } else if (typeof plugin === 'string') { - absolutePathToPlugin = plugin; - } else { - throw new Error(`[Embroider] a babel plugin without an absolute path was from: ${plugin}`); - } - - plugins[absolutePathToPlugin] = maybeNodeModuleVersion(absolutePathToPlugin); - } - - babelConfig.plugins.push([ - CACHE_BUSTING_PLUGIN.path, - { - plugins, - }, - ]); - } -} - -class ParsedEmberAsset { - kind: 'parsed-ember' = 'parsed-ember'; - relativePath: string; - fileAsset: EmberAsset; - html: PreparedEmberHTML; - - constructor(asset: EmberAsset) { - this.fileAsset = asset; - this.html = new PreparedEmberHTML(asset); - this.relativePath = asset.relativePath; - } - - validFor(other: EmberAsset) { - return this.fileAsset.mtime === other.mtime && this.fileAsset.size === other.size; - } -} - -class BuiltEmberAsset { - kind: 'built-ember' = 'built-ember'; - relativePath: string; - parsedAsset: ParsedEmberAsset; - source: string; - - constructor(asset: ParsedEmberAsset) { - this.parsedAsset = asset; - this.source = asset.html.dom.serialize(); - this.relativePath = asset.relativePath; - } -} - -class ConcatenatedAsset { - kind: 'concatenated-asset' = 'concatenated-asset'; - constructor( - public relativePath: string, - public sources: (OnDiskAsset | InMemoryAsset)[], - private resolvableExtensions: RegExp - ) {} - get sourcemapPath() { - return this.relativePath.replace(this.resolvableExtensions, '') + '.map'; - } -} - -type InternalAsset = OnDiskAsset | InMemoryAsset | BuiltEmberAsset | ConcatenatedAsset; - -export class AppBuilder { - // for each relativePath, an Asset we have already emitted - private assets: Map = new Map(); - - constructor( - private root: string, - private app: Package, - private adapter: AppAdapter, - private options: Required, - private macrosConfig: MacrosConfig - ) { - // this uses globalConfig because it's a way for packages to ask "is - // Embroider doing this build?". So it's necessarily global, not scoped to - // any subgraph of dependencies. - macrosConfig.setGlobalConfig(__filename, `@embroider/core`, { - // this is hard-coded to true because it literally means "embroider is - // building this Ember app". You can see non-true when using the Embroider - // macros in a classic build. - active: true, - }); - } - - private scriptPriority(pkg: Package) { - switch (pkg.name) { - case 'loader.js': - return 0; - case 'ember-source': - return 10; - default: - return 1000; - } - } - - @Memoize() - private get resolvableExtensionsPattern(): RegExp { - return extensionsPattern(this.adapter.resolvableExtensions()); - } - - private impliedAssets( - type: keyof ImplicitAssetPaths, - engine: Engine, - emberENV?: EmberENV - ): (OnDiskAsset | InMemoryAsset)[] { - let result: (OnDiskAsset | InMemoryAsset)[] = this.impliedAddonAssets(type, engine).map( - (sourcePath: string): OnDiskAsset => { - let stats = statSync(sourcePath); - return { - kind: 'on-disk', - relativePath: explicitRelative(this.root, sourcePath), - sourcePath, - mtime: stats.mtimeMs, - size: stats.size, - }; - } - ); - - if (type === 'implicit-scripts') { - result.unshift({ - kind: 'in-memory', - relativePath: '_testing_prefix_.js', - source: `var runningTests=false;`, - }); - - result.unshift({ - kind: 'in-memory', - relativePath: '_ember_env_.js', - source: `window.EmberENV={ ...(window.EmberENV || {}), ...${JSON.stringify(emberENV, null, 2)} };`, - }); - - result.push({ - kind: 'in-memory', - relativePath: '_loader_.js', - source: `loader.makeDefaultExport=false;`, - }); - } - - if (type === 'implicit-test-scripts') { - // this is the traditional test-support-suffix.js - result.push({ - kind: 'in-memory', - relativePath: '_testing_suffix_.js', - source: ` - var runningTests=true; - if (typeof Testem !== 'undefined' && (typeof QUnit !== 'undefined' || typeof Mocha !== 'undefined')) { - Testem.hookIntoTestFramework(); - }`, - }); - - // whether or not anybody was actually using @embroider/macros - // explicitly as an addon, we ensure its test-support file is always - // present. - if (!result.find(s => s.kind === 'on-disk' && s.sourcePath.endsWith('embroider-macros-test-support.js'))) { - result.unshift({ - kind: 'on-disk', - sourcePath: require.resolve('@embroider/macros/src/vendor/embroider-macros-test-support'), - mtime: 0, - size: 0, - relativePath: 'embroider-macros-test-support.js', - }); - } - } - - return result; - } - - private impliedAddonAssets(type: keyof ImplicitAssetPaths, engine: Engine): string[] { - let result: Array = []; - for (let addon of sortBy(Array.from(engine.addons), this.scriptPriority.bind(this))) { - let implicitScripts = addon.meta[type]; - if (implicitScripts) { - let styles = []; - let options = { basedir: addon.root }; - for (let mod of implicitScripts) { - if (type === 'implicit-styles') { - // exclude engines because they will handle their own css importation - if (!addon.isLazyEngine()) { - styles.push(resolve.sync(mod, options)); - } - } else { - result.push(resolve.sync(mod, options)); - } - } - if (styles.length) { - result = [...styles, ...result]; - } - } - } - return result; - } - - // unlike our full config, this one just needs to know how to parse all the - // syntax our app can contain. - @Memoize() - private babelParserConfig(): TransformOptions { - let babel = cloneDeep(this.adapter.babelConfig()); - - if (!babel.plugins) { - babel.plugins = []; - } - - // Our stage3 code is always allowed to use dynamic import. We may emit it - // ourself when splitting routes. - babel.plugins.push(require.resolve('@babel/plugin-syntax-dynamic-import')); - return babel; - } - - @Memoize() - private babelConfig(resolverConfig: ResolverConfig) { - let babel = cloneDeep(this.adapter.babelConfig()); - - if (!babel.plugins) { - babel.plugins = []; - } - - // Our stage3 code is always allowed to use dynamic import. We may emit it - // ourself when splitting routes. - babel.plugins.push(require.resolve('@babel/plugin-syntax-dynamic-import')); - - // https://github.com/webpack/webpack/issues/12154 - babel.plugins.push(require.resolve('./rename-require-plugin')); - - babel.plugins.push([require.resolve('babel-plugin-ember-template-compilation'), this.etcOptions(resolverConfig)]); - - // this is @embroider/macros configured for full stage3 resolution - babel.plugins.push(...this.macrosConfig.babelPluginConfig()); - - let colocationOptions: ColocationOptions = { - appRoot: this.root, - - // This extra weirdness is a compromise in favor of build performance. - // - // 1. When auto-upgrading an addon from v1 to v2, we definitely want to - // run any custom AST transforms in stage1. - // - // 2. In general case, AST transforms are allowed to manipulate Javascript - // scope. This means that running transforms -- even when we're doing - // source-to-source compilation that emits handlebars and not wire - // format -- implies changing .hbs files into .js files. - // - // 3. So stage1 may need to rewrite .hbs to .hbs.js (to avoid colliding - // with an existing co-located .js file). - // - // 4. But stage1 doesn't necessarily want to run babel over the - // corresponding JS file. Most of the time, that's just an - // unnecessarily expensive second parse. (We only run it in stage1 to - // eliminate an addon's custom babel plugins, and many addons don't - // have any.) - // - // 5. Therefore, the work of template-colocation gets defered until here, - // and it may see co-located templates named `.hbs.js` instead of the - // usual `.hbs. - templateExtensions: ['.hbs', '.hbs.js'], - - // All of the above only applies to auto-upgraded packages that were - // authored in v1. V2 packages don't get any of this complexity, they're - // supposed to take care of colocating their own templates explicitly. - packageGuard: true, - }; - babel.plugins.push([ - require.resolve('@embroider/shared-internals/src/template-colocation-plugin'), - colocationOptions, - ]); - - for (let p of this.adapter.jsPlugins(resolverConfig)) { - babel.plugins.push(p); - } - - // we can use globally shared babel runtime by default - babel.plugins.push([ - require.resolve('@babel/plugin-transform-runtime'), - { absoluteRuntime: __dirname, useESModules: true, regenerator: false }, - ]); - - const portable = makePortable(babel, { basedir: this.root }, this.portableHints); - addCachablePlugin(portable.config); - return portable; - } - - private insertEmberApp( - asset: ParsedEmberAsset, - appFiles: Engine[], - prepared: Map, - emberENV: EmberENV - ) { - let html = asset.html; - - if (this.fastbootConfig) { - // ignore scripts like ember-cli-livereload.js which are not really associated with - // "the app". - let ignoreScripts = html.dom.window.document.querySelectorAll('script'); - ignoreScripts.forEach(script => { - script.setAttribute('data-fastboot-ignore', ''); - }); - } - - // our tests entrypoint already includes a correct module dependency on the - // app, so we only insert the app when we're not inserting tests - if (!asset.fileAsset.includeTests) { - let appJS = this.topAppJSAsset(appFiles, prepared); - html.insertScriptTag(html.javascript, appJS.relativePath, { type: 'module' }); - } - - if (this.fastbootConfig) { - // any extra fastboot app files get inserted into our html.javascript - // section, after the app has been inserted. - for (let script of this.fastbootConfig.extraAppFiles) { - html.insertScriptTag(html.javascript, script, { tag: 'fastboot-script' }); - } - } - - html.insertStyleLink(html.styles, `assets/${this.app.name}.css`); - - const parentEngine = appFiles.find(e => !e.parent) as Engine; - let vendorJS = this.implicitScriptsAsset(prepared, parentEngine, emberENV); - if (vendorJS) { - html.insertScriptTag(html.implicitScripts, vendorJS.relativePath); - } - - if (this.fastbootConfig) { - // any extra fastboot vendor files get inserted into our - // html.implicitScripts section, after the regular implicit script - // (vendor.js) have been inserted. - for (let script of this.fastbootConfig.extraVendorFiles) { - html.insertScriptTag(html.implicitScripts, script, { tag: 'fastboot-script' }); - } - } - - let implicitStyles = this.implicitStylesAsset(prepared, parentEngine); - if (implicitStyles) { - html.insertStyleLink(html.implicitStyles, implicitStyles.relativePath); - } - - if (!asset.fileAsset.includeTests) { - return; - } - - // Test-related assets happen below this point - - let testJS = this.testJSEntrypoint(appFiles, prepared); - html.insertScriptTag(html.testJavascript, testJS.relativePath, { type: 'module' }); - - let implicitTestScriptsAsset = this.implicitTestScriptsAsset(prepared, parentEngine); - if (implicitTestScriptsAsset) { - html.insertScriptTag(html.implicitTestScripts, implicitTestScriptsAsset.relativePath); - } - - let implicitTestStylesAsset = this.implicitTestStylesAsset(prepared, parentEngine); - if (implicitTestStylesAsset) { - html.insertStyleLink(html.implicitTestStyles, implicitTestStylesAsset.relativePath); - } - } - - private implicitScriptsAsset( - prepared: Map, - application: Engine, - emberENV: EmberENV - ): InternalAsset | undefined { - let asset = prepared.get('assets/vendor.js'); - if (!asset) { - let implicitScripts = this.impliedAssets('implicit-scripts', application, emberENV); - if (implicitScripts.length > 0) { - asset = new ConcatenatedAsset('assets/vendor.js', implicitScripts, this.resolvableExtensionsPattern); - prepared.set(asset.relativePath, asset); - } - } - return asset; - } - - private implicitStylesAsset(prepared: Map, application: Engine): InternalAsset | undefined { - let asset = prepared.get('assets/vendor.css'); - if (!asset) { - let implicitStyles = this.impliedAssets('implicit-styles', application); - if (implicitStyles.length > 0) { - // we reverse because we want the synthetic vendor style at the top - asset = new ConcatenatedAsset('assets/vendor.css', implicitStyles.reverse(), this.resolvableExtensionsPattern); - prepared.set(asset.relativePath, asset); - } - } - return asset; - } - - private implicitTestScriptsAsset( - prepared: Map, - application: Engine - ): InternalAsset | undefined { - let testSupportJS = prepared.get('assets/test-support.js'); - if (!testSupportJS) { - let implicitTestScripts = this.impliedAssets('implicit-test-scripts', application); - if (implicitTestScripts.length > 0) { - testSupportJS = new ConcatenatedAsset( - 'assets/test-support.js', - implicitTestScripts, - this.resolvableExtensionsPattern - ); - prepared.set(testSupportJS.relativePath, testSupportJS); - } - } - return testSupportJS; - } - - private implicitTestStylesAsset( - prepared: Map, - application: Engine - ): InternalAsset | undefined { - let asset = prepared.get('assets/test-support.css'); - if (!asset) { - let implicitTestStyles = this.impliedAssets('implicit-test-styles', application); - if (implicitTestStyles.length > 0) { - asset = new ConcatenatedAsset('assets/test-support.css', implicitTestStyles, this.resolvableExtensionsPattern); - prepared.set(asset.relativePath, asset); - } - } - return asset; - } - - // recurse to find all active addons that don't cross an engine boundary. - // Inner engines themselves will be returned, but not those engines' children. - // The output set's insertion order is the proper ember-cli compatible - // ordering of the addons. - private findActiveAddons(pkg: Package, engine: EngineSummary, isChild = false): void { - for (let child of this.adapter.activeAddonChildren(pkg)) { - if (!child.isEngine()) { - this.findActiveAddons(child, engine, true); - } - engine.addons.add(child); - } - // ensure addons are applied in the correct order, if set (via @embroider/compat/v1-addon) - if (!isChild) { - engine.addons = new Set( - [...engine.addons].sort((a, b) => { - return (a.meta['order-index'] || 0) - (b.meta['order-index'] || 0); - }) - ); - } - } - - private partitionEngines(appJSPath: string): EngineSummary[] { - let queue: EngineSummary[] = [ - { - package: this.app, - addons: new Set(), - parent: undefined, - sourcePath: appJSPath, - destPath: this.root, - modulePrefix: this.modulePrefix, - appRelativePath: '.', - }, - ]; - let done: EngineSummary[] = []; - let seenEngines: Set = new Set(); - while (true) { - let current = queue.shift(); - if (!current) { - break; - } - this.findActiveAddons(current.package, current); - for (let addon of current.addons) { - if (addon.isEngine() && !seenEngines.has(addon)) { - seenEngines.add(addon); - queue.push({ - package: addon, - addons: new Set(), - parent: current, - sourcePath: mangledEngineRoot(addon), - destPath: addon.root, - modulePrefix: addon.name, - appRelativePath: explicitRelative(this.root, addon.root), - }); - } - } - done.push(current); - } - return done; - } - - @Memoize() - private get activeFastboot() { - return this.adapter.activeAddonChildren(this.app).find(a => a.name === 'ember-cli-fastboot'); - } - - @Memoize() - private get fastbootConfig(): - | { packageJSON: PackageInfo; extraAppFiles: string[]; extraVendorFiles: string[] } - | undefined { - if (this.activeFastboot) { - // this is relying on work done in stage1 by @embroider/compat/src/compat-adapters/ember-cli-fastboot.ts - let packageJSON = readJSONSync(join(this.activeFastboot.root, '_fastboot_', 'package.json')); - let { extraAppFiles, extraVendorFiles } = packageJSON['embroider-fastboot']; - delete packageJSON['embroider-fastboot']; - extraVendorFiles.push('assets/embroider_macros_fastboot_init.js'); - return { packageJSON, extraAppFiles, extraVendorFiles }; - } - } - - private appDiffers: { differ: AppDiffer; engine: EngineSummary }[] | undefined; - - private updateAppJS(inputPaths: OutputPaths): Engine[] { - let appJSPath = this.adapter.appJSSrcDir(inputPaths); - if (!this.appDiffers) { - let engines = this.partitionEngines(appJSPath); - this.appDiffers = engines.map(engine => { - let differ: AppDiffer; - if (this.activeFastboot) { - differ = new AppDiffer( - engine.destPath, - engine.sourcePath, - [...engine.addons], - true, - this.adapter.fastbootJSSrcDir(inputPaths), - this.babelParserConfig() - ); - } else { - differ = new AppDiffer(engine.destPath, engine.sourcePath, [...engine.addons]); - } - return { - differ, - engine, - }; - }); - } - // this is in reverse order because we need deeper engines to update before - // their parents, because they aren't really valid packages until they - // update, and their parents will go looking for their own `app-js` content. - this.appDiffers - .slice() - .reverse() - .forEach(a => a.differ.update()); - return this.appDiffers.map(a => { - return { - ...a.engine, - appFiles: new AppFiles(a.differ, this.resolvableExtensionsPattern, this.adapter.podModulePrefix()), - }; - }); - } - - private prepareAsset(asset: Asset, appFiles: Engine[], prepared: Map, emberENV: EmberENV) { - if (asset.kind === 'ember') { - let prior = this.assets.get(asset.relativePath); - let parsed: ParsedEmberAsset; - if (prior && prior.kind === 'built-ember' && prior.parsedAsset.validFor(asset)) { - // we can reuse the parsed html - parsed = prior.parsedAsset; - parsed.html.clear(); - } else { - parsed = new ParsedEmberAsset(asset); - } - this.insertEmberApp(parsed, appFiles, prepared, emberENV); - prepared.set(asset.relativePath, new BuiltEmberAsset(parsed)); - } else { - prepared.set(asset.relativePath, asset); - } - } - - private prepareAssets(requestedAssets: Asset[], appFiles: Engine[], emberENV: EmberENV): Map { - let prepared: Map = new Map(); - for (let asset of requestedAssets) { - this.prepareAsset(asset, appFiles, prepared, emberENV); - } - return prepared; - } - - private assetIsValid(asset: InternalAsset, prior: InternalAsset | undefined): boolean { - if (!prior) { - return false; - } - switch (asset.kind) { - case 'on-disk': - return prior.kind === 'on-disk' && prior.size === asset.size && prior.mtime === asset.mtime; - case 'in-memory': - return prior.kind === 'in-memory' && stringOrBufferEqual(prior.source, asset.source); - case 'built-ember': - return prior.kind === 'built-ember' && prior.source === asset.source; - case 'concatenated-asset': - return ( - prior.kind === 'concatenated-asset' && - prior.sources.length === asset.sources.length && - prior.sources.every((priorFile, index) => { - let newFile = asset.sources[index]; - return this.assetIsValid(newFile, priorFile); - }) - ); - } - } - - private updateOnDiskAsset(asset: OnDiskAsset) { - let destination = join(this.root, asset.relativePath); - ensureDirSync(dirname(destination)); - copySync(asset.sourcePath, destination, { dereference: true }); - } - - private updateInMemoryAsset(asset: InMemoryAsset) { - let destination = join(this.root, asset.relativePath); - ensureDirSync(dirname(destination)); - writeFileSync(destination, asset.source, 'utf8'); - } - - private updateBuiltEmberAsset(asset: BuiltEmberAsset) { - let destination = join(this.root, asset.relativePath); - ensureDirSync(dirname(destination)); - writeFileSync(destination, asset.source, 'utf8'); - } - - private async updateConcatenatedAsset(asset: ConcatenatedAsset) { - let concat = new SourceMapConcat({ - outputFile: join(this.root, asset.relativePath), - mapCommentType: asset.relativePath.endsWith('.js') ? 'line' : 'block', - baseDir: this.root, - }); - if (process.env.EMBROIDER_CONCAT_STATS) { - let MeasureConcat = (await import('./measure-concat')).default; - concat = new MeasureConcat(asset.relativePath, concat, this.root); - } - for (let source of asset.sources) { - switch (source.kind) { - case 'on-disk': - concat.addFile(explicitRelative(this.root, source.sourcePath)); - break; - case 'in-memory': - if (typeof source.source !== 'string') { - throw new Error(`attempted to concatenated a Buffer-backed in-memory asset`); - } - concat.addSpace(source.source); - break; - default: - assertNever(source); - } - } - await concat.end(); - } - - private async updateAssets(requestedAssets: Asset[], appFiles: Engine[], emberENV: EmberENV) { - let assets = this.prepareAssets(requestedAssets, appFiles, emberENV); - for (let asset of assets.values()) { - if (this.assetIsValid(asset, this.assets.get(asset.relativePath))) { - continue; - } - debug('rebuilding %s', asset.relativePath); - switch (asset.kind) { - case 'on-disk': - this.updateOnDiskAsset(asset); - break; - case 'in-memory': - this.updateInMemoryAsset(asset); - break; - case 'built-ember': - this.updateBuiltEmberAsset(asset); - break; - case 'concatenated-asset': - await this.updateConcatenatedAsset(asset); - break; - default: - assertNever(asset); - } - } - for (let oldAsset of this.assets.values()) { - if (!assets.has(oldAsset.relativePath)) { - unlinkSync(join(this.root, oldAsset.relativePath)); - } - } - this.assets = assets; - return [...assets.values()]; - } - - private gatherAssets(inputPaths: OutputPaths): Asset[] { - // first gather all the assets out of addons - let assets: Asset[] = []; - for (let pkg of this.adapter.allActiveAddons) { - if (pkg.meta['public-assets']) { - for (let [filename, appRelativeURL] of Object.entries(pkg.meta['public-assets'] || {})) { - let sourcePath = resolvePath(pkg.root, filename); - let stats = statSync(sourcePath); - assets.push({ - kind: 'on-disk', - sourcePath, - relativePath: appRelativeURL, - mtime: stats.mtimeMs, - size: stats.size, - }); - } - } - } - - if (this.activeFastboot) { - const source = ` - (function(){ - var key = '_embroider_macros_runtime_config'; - if (!window[key]){ window[key] = [];} - window[key].push(function(m) { - m.setGlobalConfig('fastboot', Object.assign({}, m.getGlobalConfig().fastboot, { isRunning: true })); - }); - }())`; - assets.push({ - kind: 'in-memory', - source, - relativePath: 'assets/embroider_macros_fastboot_init.js', - }); - } - - // and finally tack on the ones from our app itself - return assets.concat(this.adapter.assets(inputPaths)); - } - - async build(inputPaths: OutputPaths) { - if (this.adapter.env !== 'production') { - this.macrosConfig.enablePackageDevelopment(this.root); - this.macrosConfig.enableRuntimeMode(); - } - for (let pkgRoot of this.adapter.developingAddons()) { - this.macrosConfig.enablePackageDevelopment(pkgRoot); - } - - // on the first build, we lock down the macros config. on subsequent builds, - // this doesn't do anything anyway because it's idempotent. - this.macrosConfig.finalize(); - - let appFiles = this.updateAppJS(inputPaths); - let emberENV = this.adapter.emberENV(); - let assets = this.gatherAssets(inputPaths); - - let finalAssets = await this.updateAssets(assets, appFiles, emberENV); - - let assetPaths = assets.map(asset => asset.relativePath); - - if (this.activeFastboot) { - // when using fastboot, our own package.json needs to be in the output so fastboot can read it. - assetPaths.push('package.json'); - } - - for (let asset of finalAssets) { - // our concatenated assets all have map files that ride along. Here we're - // telling the final stage packager to be sure and serve the map files - // too. - if (asset.kind === 'concatenated-asset') { - assetPaths.push(asset.sourcemapPath); - } - } - - let meta: AppMeta = { - type: 'app', - version: 2, - assets: assetPaths, - babel: { - filename: '_babel_config_.js', - isParallelSafe: true, // TODO - majorVersion: this.adapter.babelMajorVersion(), - fileFilter: '_babel_filter_.js', - }, - 'root-url': this.adapter.rootURL(), - }; - - if (!this.adapter.strictV2Format()) { - meta['auto-upgraded'] = true; - } - - let pkg = this.combinePackageJSON(meta); - writeFileSync(join(this.root, 'package.json'), JSON.stringify(pkg, null, 2), 'utf8'); - - let resolverConfig = this.adapter.resolverConfig(appFiles); - this.addResolverConfig(resolverConfig); - let babelConfig = this.babelConfig(resolverConfig); - this.addBabelConfig(babelConfig); - } - - private combinePackageJSON(meta: AppMeta): object { - let pkgLayers: any[] = [this.app.packageJSON]; - let fastbootConfig = this.fastbootConfig; - if (fastbootConfig) { - // fastboot-specific package.json output is allowed to add to our original package.json - pkgLayers.push(fastbootConfig.packageJSON); - } - // but our own new v2 app metadata takes precedence over both - pkgLayers.push({ keywords: ['ember-addon'], 'ember-addon': meta }); - return combinePackageJSON(...pkgLayers); - } - - private etcOptions(resolverConfig: ResolverConfig): EtcOptions { - let transforms = this.adapter.htmlbarsPlugins(); - - let { plugins: macroPlugins, setConfig } = MacrosConfig.transforms(); - setConfig(this.macrosConfig); - for (let macroPlugin of macroPlugins) { - transforms.push(macroPlugin as any); - } - - for (let t of this.adapter.hbsTransforms(resolverConfig)) { - transforms.push(t); - } - - return { - transforms, - compilerPath: resolve.sync(this.adapter.templateCompilerPath(), { basedir: this.root }), - enableLegacyModules: ['ember-cli-htmlbars', 'ember-cli-htmlbars-inline-precompile', 'htmlbars-inline-precompile'], - }; - } - - @Memoize() - private get portableHints(): PortableHint[] { - return this.options.pluginHints.map(hint => { - let cursor = join(this.app.root, 'package.json'); - for (let i = 0; i < hint.resolve.length; i++) { - let target = hint.resolve[i]; - if (i < hint.resolve.length - 1) { - target = join(target, 'package.json'); - } - cursor = resolve.sync(target, { basedir: dirname(cursor) }); - } - - return { - requireFile: cursor, - useMethod: hint.useMethod, - packageVersion: maybeNodeModuleVersion(cursor), - }; - }); - } - - private addBabelConfig(pconfig: { config: TransformOptions; isParallelSafe: boolean }) { - if (!pconfig.isParallelSafe) { - warn('Your build is slower because some babel plugins are non-serializable'); - } - writeFileSync( - join(this.root, '_babel_config_.js'), - `module.exports = ${JSON.stringify(pconfig.config, null, 2)}`, - 'utf8' - ); - writeFileSync( - join(this.root, '_babel_filter_.js'), - babelFilterTemplate({ skipBabel: this.options.skipBabel, appRoot: this.root }), - 'utf8' - ); - } - - private addResolverConfig(config: ResolverConfig) { - outputJSONSync(join(this.root, '.embroider', 'resolver.json'), config); - } - - private shouldSplitRoute(routeName: string) { - return ( - !this.options.splitAtRoutes || - this.options.splitAtRoutes.find(pattern => { - if (typeof pattern === 'string') { - return pattern === routeName; - } else { - return pattern.test(routeName); - } - }) - ); - } - - private splitRoute( - routeName: string, - files: RouteFiles, - addToParent: (routeName: string, filename: string) => void, - addLazyBundle: (routeNames: string[], files: string[]) => void - ) { - let shouldSplit = routeName && this.shouldSplitRoute(routeName); - let ownFiles = []; - let ownNames = new Set() as Set; - - if (files.template) { - if (shouldSplit) { - ownFiles.push(files.template); - ownNames.add(routeName); - } else { - addToParent(routeName, files.template); - } - } - - if (files.controller) { - if (shouldSplit) { - ownFiles.push(files.controller); - ownNames.add(routeName); - } else { - addToParent(routeName, files.controller); - } - } - - if (files.route) { - if (shouldSplit) { - ownFiles.push(files.route); - ownNames.add(routeName); - } else { - addToParent(routeName, files.route); - } - } - - for (let [childName, childFiles] of files.children) { - this.splitRoute( - `${routeName}.${childName}`, - childFiles, - - (childRouteName: string, childFile: string) => { - // this is our child calling "addToParent" - if (shouldSplit) { - ownFiles.push(childFile); - ownNames.add(childRouteName); - } else { - addToParent(childRouteName, childFile); - } - }, - (routeNames: string[], files: string[]) => { - addLazyBundle(routeNames, files); - } - ); - } - - if (ownFiles.length > 0) { - addLazyBundle([...ownNames], ownFiles); - } - } - - private topAppJSAsset(engines: Engine[], prepared: Map): InternalAsset { - let [app, ...childEngines] = engines; - let relativePath = `assets/${this.app.name}.js`; - return this.appJSAsset(relativePath, app, childEngines, prepared, { - autoRun: this.adapter.autoRun(), - appBoot: !this.adapter.autoRun() ? this.adapter.appBoot() : '', - mainModule: explicitRelative(dirname(relativePath), this.adapter.mainModule()), - appConfig: this.adapter.mainModuleConfig(), - }); - } - - @Memoize() - private get staticAppPathsPattern(): RegExp | undefined { - if (this.options.staticAppPaths.length > 0) { - return new RegExp( - '^(?:' + this.options.staticAppPaths.map(staticAppPath => escapeRegExp(staticAppPath)).join('|') + ')(?:$|/)' - ); - } - } - - private requiredOtherFiles(appFiles: AppFiles): readonly string[] { - let pattern = this.staticAppPathsPattern; - if (pattern) { - return appFiles.otherAppFiles.filter(f => { - return !pattern!.test(f); - }); - } else { - return appFiles.otherAppFiles; - } - } - - private appJSAsset( - relativePath: string, - engine: Engine, - childEngines: Engine[], - prepared: Map, - entryParams?: Partial[0]> - ): InternalAsset { - let { appFiles } = engine; - let cached = prepared.get(relativePath); - if (cached) { - return cached; - } - - let eagerModules = []; - - let requiredAppFiles = [this.requiredOtherFiles(appFiles)]; - if (!this.options.staticComponents) { - requiredAppFiles.push(appFiles.components); - } - if (!this.options.staticHelpers) { - requiredAppFiles.push(appFiles.helpers); - } - if (!this.options.staticModifiers) { - requiredAppFiles.push(appFiles.modifiers); - } - - let styles = []; - // only import styles from engines with a parent (this excludeds the parent application) as their styles - // will be inserted via a direct tag. - if (engine.parent && engine.package.isLazyEngine()) { - let implicitStyles = this.impliedAssets('implicit-styles', engine); - for (let style of implicitStyles) { - styles.push({ - path: explicitRelative('assets/_engine_', style.relativePath), - }); - } - - let engineMeta = engine.package.meta as AddonMeta; - if (engineMeta && engineMeta['implicit-styles']) { - for (let style of engineMeta['implicit-styles']) { - styles.push({ - path: explicitRelative(dirname(relativePath), join(engine.appRelativePath, style)), - }); - } - } - } - - let lazyEngines: { names: string[]; path: string }[] = []; - for (let childEngine of childEngines) { - let asset = this.appJSAsset( - `assets/_engine_/${encodeURIComponent(childEngine.package.name)}.js`, - childEngine, - [], - prepared - ); - if (childEngine.package.isLazyEngine()) { - lazyEngines.push({ - names: [childEngine.package.name], - path: explicitRelative(dirname(relativePath), asset.relativePath), - }); - } else { - eagerModules.push(explicitRelative(dirname(relativePath), asset.relativePath)); - } - } - let lazyRoutes: { names: string[]; path: string }[] = []; - for (let [routeName, routeFiles] of appFiles.routeFiles.children) { - this.splitRoute( - routeName, - routeFiles, - (_: string, filename: string) => { - requiredAppFiles.push([filename]); - }, - (routeNames: string[], files: string[]) => { - let routeEntrypoint = `assets/_route_/${encodeURIComponent(routeNames[0])}.js`; - if (!prepared.has(routeEntrypoint)) { - prepared.set(routeEntrypoint, this.routeEntrypoint(engine, routeEntrypoint, files)); - } - lazyRoutes.push({ - names: routeNames, - path: this.importPaths(engine, routeEntrypoint).buildtime, - }); - } - ); - } - - let [fastboot, nonFastboot] = partition(excludeDotFiles(flatten(requiredAppFiles)), file => - appFiles.isFastbootOnly.get(file) - ); - let amdModules = nonFastboot.map(file => this.importPaths(engine, file)); - let fastbootOnlyAmdModules = fastboot.map(file => this.importPaths(engine, file)); - - // this is a backward-compatibility feature: addons can force inclusion of - // modules. - this.gatherImplicitModules('implicit-modules', engine, amdModules); - - let params = { amdModules, fastbootOnlyAmdModules, lazyRoutes, lazyEngines, eagerModules, styles }; - if (entryParams) { - Object.assign(params, entryParams); - } - - let source = entryTemplate(params); - - let asset: InternalAsset = { - kind: 'in-memory', - source, - relativePath, - }; - prepared.set(relativePath, asset); - return asset; - } - - @Memoize() - private get modulePrefix() { - return this.adapter.modulePrefix(); - } - - private importPaths(engine: Engine, engineRelativePath: string) { - let noHBS = engineRelativePath.replace(this.resolvableExtensionsPattern, '').replace(/\.hbs$/, ''); - return { - runtime: `${engine.modulePrefix}/${noHBS}`, - buildtime: posix.join(engine.package.name, engineRelativePath), - }; - } - - private routeEntrypoint(engine: Engine, relativePath: string, files: string[]) { - let [fastboot, nonFastboot] = partition(files, file => engine.appFiles.isFastbootOnly.get(file)); - - let asset: InternalAsset = { - kind: 'in-memory', - source: routeEntryTemplate({ - files: nonFastboot.map(f => this.importPaths(engine, f)), - fastbootOnlyFiles: fastboot.map(f => this.importPaths(engine, f)), - }), - relativePath, - }; - return asset; - } - - private testJSEntrypoint(engines: Engine[], prepared: Map): InternalAsset { - let asset = prepared.get(`assets/test.js`); - if (asset) { - return asset; - } - - // We're only building tests from the first engine (the app). This is the - // normal thing to do -- tests from engines don't automatically roll up into - // the app. - let engine = engines[0]; - - const myName = 'assets/test.js'; - - // tests necessarily also include the app. This is where we account for - // that. The classic solution was to always include the app's separate - // script tag in the tests HTML, but that isn't as easy for final stage - // packagers to understand. It's better to express it here as a direct - // module dependency. - let eagerModules: string[] = [ - explicitRelative(dirname(myName), this.topAppJSAsset(engines, prepared).relativePath), - ]; - - let amdModules: { runtime: string; buildtime: string }[] = []; - // this is a backward-compatibility feature: addons can force inclusion of - // test support modules. - this.gatherImplicitModules('implicit-test-modules', engine, amdModules); - - let { appFiles } = engine; - for (let relativePath of appFiles.tests) { - amdModules.push(this.importPaths(engine, relativePath)); - } - - let source = entryTemplate({ - amdModules, - eagerModules, - testSuffix: true, - }); - - asset = { - kind: 'in-memory', - source, - relativePath: myName, - }; - prepared.set(asset.relativePath, asset); - return asset; - } - - private gatherImplicitModules( - section: 'implicit-modules' | 'implicit-test-modules', - engine: Engine, - lazyModules: { runtime: string; buildtime: string }[] - ) { - for (let addon of engine.addons) { - let implicitModules = addon.meta[section]; - if (implicitModules) { - let renamedModules = inverseRenamedModules(addon.meta, this.resolvableExtensionsPattern); - for (let name of implicitModules) { - let packageName = addon.name; - - if (addon.isV2Addon()) { - let renamedMeta = addon.meta['renamed-packages']; - if (renamedMeta) { - Object.entries(renamedMeta).forEach(([key, value]) => { - if (value === addon!.name) { - packageName = key; - } - }); - } - } - - let runtime = join(packageName, name).replace(this.resolvableExtensionsPattern, ''); - let runtimeRenameLookup = runtime.split('\\').join('/'); - if (renamedModules && renamedModules[runtimeRenameLookup]) { - runtime = renamedModules[runtimeRenameLookup]; - } - runtime = runtime.split(sep).join('/'); - lazyModules.push({ - runtime, - buildtime: posix.join(packageName, name), - }); - } - } - } - } -} - -const entryTemplate = compile(` -import { importSync as i, macroCondition, getGlobalConfig } from '@embroider/macros'; -let w = window; -let d = w.define; - -{{#if styles}} - if (macroCondition(!getGlobalConfig().fastboot?.isRunning)) { - {{#each styles as |stylePath| ~}} - i("{{js-string-escape stylePath.path}}"); - {{/each}} - } -{{/if}} - -{{#each amdModules as |amdModule| ~}} - d("{{js-string-escape amdModule.runtime}}", function(){ return i("{{js-string-escape amdModule.buildtime}}");}); -{{/each}} - -{{#if fastbootOnlyAmdModules}} - if (macroCondition(getGlobalConfig().fastboot?.isRunning)) { - {{#each fastbootOnlyAmdModules as |amdModule| ~}} - d("{{js-string-escape amdModule.runtime}}", function(){ return i("{{js-string-escape amdModule.buildtime}}");}); - {{/each}} - } -{{/if}} - -{{#each eagerModules as |eagerModule| ~}} - i("{{js-string-escape eagerModule}}"); -{{/each}} - -{{#if lazyRoutes}} -w._embroiderRouteBundles_ = [ - {{#each lazyRoutes as |route|}} - { - names: {{{json-stringify route.names}}}, - load: function() { - return import("{{js-string-escape route.path}}"); - } - }, - {{/each}} -] -{{/if}} - -{{#if lazyEngines}} -w._embroiderEngineBundles_ = [ - {{#each lazyEngines as |engine|}} - { - names: {{{json-stringify engine.names}}}, - load: function() { - return import("{{js-string-escape engine.path}}"); - } - }, - {{/each}} -] -{{/if}} - -{{#if autoRun ~}} -if (!runningTests) { - i("{{js-string-escape mainModule}}").default.create({{{json-stringify appConfig}}}); -} -{{else if appBoot ~}} - {{{ appBoot }}} -{{/if}} - -{{#if testSuffix ~}} - {{!- TODO: both of these suffixes should get dynamically generated so they incorporate - any content-for added by addons. -}} - - - {{!- this is the traditional tests-suffix.js -}} - i('../tests/test-helper'); - EmberENV.TESTS_FILE_LOADED = true; -{{/if}} -`) as (params: { - amdModules: { runtime: string; buildtime: string }[]; - fastbootOnlyAmdModules?: { runtime: string; buildtime: string }[]; - eagerModules?: string[]; - autoRun?: boolean; - appBoot?: string; - mainModule?: string; - appConfig?: unknown; - testSuffix?: boolean; - lazyRoutes?: { names: string[]; path: string }[]; - lazyEngines?: { names: string[]; path: string }[]; - styles?: { path: string }[]; -}) => string; - -const routeEntryTemplate = compile(` -import { importSync as i } from '@embroider/macros'; -let d = window.define; -{{#each files as |amdModule| ~}} -d("{{js-string-escape amdModule.runtime}}", function(){ return i("{{js-string-escape amdModule.buildtime}}");}); -{{/each}} -{{#if fastbootOnlyFiles}} - import { macroCondition, getGlobalConfig } from '@embroider/macros'; - if (macroCondition(getGlobalConfig().fastboot?.isRunning)) { - {{#each fastbootOnlyFiles as |amdModule| ~}} - d("{{js-string-escape amdModule.runtime}}", function(){ return i("{{js-string-escape amdModule.buildtime}}");}); - {{/each}} - } -{{/if}} -`) as (params: { - files: { runtime: string; buildtime: string }[]; - fastbootOnlyFiles: { runtime: string; buildtime: string }[]; -}) => string; - -function stringOrBufferEqual(a: string | Buffer, b: string | Buffer): boolean { - if (typeof a === 'string' && typeof b === 'string') { - return a === b; - } - if (a instanceof Buffer && b instanceof Buffer) { - return Buffer.compare(a, b) === 0; - } - return false; -} - -const babelFilterTemplate = compile(` -const { babelFilter } = require(${JSON.stringify(require.resolve('./index.js'))}); -module.exports = babelFilter({{{json-stringify skipBabel}}}, "{{{js-string-escape appRoot}}}"); -`) as (params: { skipBabel: Options['skipBabel']; appRoot: string }) => string; - -// meta['renamed-modules'] has mapping from classic filename to real filename. -// This takes that and converts it to the inverst mapping from real import path -// to classic import path. -function inverseRenamedModules(meta: AddonPackage['meta'], extensions: RegExp) { - let renamed = meta['renamed-modules']; - if (renamed) { - let inverted = {} as { [name: string]: string }; - for (let [classic, real] of Object.entries(renamed)) { - inverted[real.replace(extensions, '')] = classic.replace(extensions, ''); - } - return inverted; - } -} - -function combinePackageJSON(...layers: object[]) { - function custom(objValue: any, srcValue: any, key: string, _object: any, _source: any, stack: { size: number }) { - if (key === 'keywords' && stack.size === 0) { - if (Array.isArray(objValue)) { - return objValue.concat(srcValue); - } - } - } - return mergeWith({}, ...layers, custom); -} diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 5ef0a821b..308836bc1 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -13,7 +13,6 @@ export { default as Options, optionsWithDefaults } from './options'; export { default as toBroccoliPlugin } from './to-broccoli-plugin'; export { default as WaitForTrees, OutputPaths } from './wait-for-trees'; export { compile as jsHandlebarsCompile } from './js-handlebars'; -export { AppAdapter, AppBuilder, EmberENV } from './app'; export { todo, unsupported, warn, debug, expectWarning, throwOnWarnings } from './messages'; export { mangledEngineRoot } from './engine-mangler'; export { diff --git a/packages/core/tests/app.test.ts b/packages/core/tests/app.test.ts deleted file mode 100644 index 9b0704233..000000000 --- a/packages/core/tests/app.test.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { excludeDotFiles, addCachablePlugin, CACHE_BUSTING_PLUGIN } from '../src/app'; - -describe('dot files can be excluded', () => { - test('excludeDotFiles works', () => { - expect(excludeDotFiles([])).toEqual([]); - expect(excludeDotFiles(['.foo.js'])).toEqual([]); - expect(excludeDotFiles(['bar/.foo.js'])).toEqual([]); - expect(excludeDotFiles(['.foo.js', 'bar/.foo.js'])).toEqual([]); - expect(excludeDotFiles(['foo.bar.baz', '.foo.js'])).toEqual(['foo.bar.baz']); - expect(excludeDotFiles(['foo/bar/baz/.foo.js'])).toEqual([]); - }); -}); - -describe('cacheable-plugin', function () { - test('noop', function () { - const input = {}; - addCachablePlugin(input); - expect(input).toEqual({}); - }); - - test('no plugins', function () { - const input = { plugins: [] }; - addCachablePlugin(input); - expect(input).toEqual({ plugins: [] }); - }); - - test('some plugins', function () { - const input = { - plugins: [__dirname, [__dirname, []], [`${__dirname}/../`, []], __dirname, [__dirname, []]], - }; - - addCachablePlugin(input); - - expect(input).toEqual({ - plugins: [ - __dirname, - [__dirname, []], - [`${__dirname}/../`, []], - __dirname, - [__dirname, []], - - [ - CACHE_BUSTING_PLUGIN.path, - { - plugins: { - [CACHE_BUSTING_PLUGIN.path]: CACHE_BUSTING_PLUGIN.version, - [__dirname]: CACHE_BUSTING_PLUGIN.version, - [`${__dirname}/../`]: CACHE_BUSTING_PLUGIN.version, - }, - }, - ], - ], - }); - }); -}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7eb567e33..cc1cb01e9 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -222,6 +222,12 @@ importers: debug: specifier: ^4.3.2 version: 4.3.2(supports-color@8.1.0) + escape-string-regexp: + specifier: ^4.0.0 + version: 4.0.0 + fast-sourcemap-concat: + specifier: ^1.4.0 + version: 1.4.0 fs-extra: specifier: ^9.1.0 version: 9.1.0 @@ -4360,7 +4366,7 @@ packages: ignore: 4.0.6 import-fresh: 3.3.0 js-yaml: 3.14.1 - minimatch: 3.0.4 + minimatch: 3.1.2 strip-json-comments: 3.1.1 transitivePeerDependencies: - supports-color @@ -4669,7 +4675,7 @@ packages: dependencies: '@humanwhocodes/object-schema': 1.2.1 debug: 4.3.2(supports-color@8.1.0) - minimatch: 3.0.4 + minimatch: 3.1.2 transitivePeerDependencies: - supports-color dev: true From b0636d0d8eb05b33203a040e6c04b81d3b6a1782 Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Mon, 22 May 2023 14:09:51 -0400 Subject: [PATCH 07/72] adjust the runtime require.resolves --- packages/compat/package.json | 1 + packages/compat/src/compat-app.ts | 106 ++++++++---------- .../src/rename-require-plugin.ts | 0 packages/core/package.json | 4 - .../core/tests/rename-require-plugin.test.ts | 18 --- .../src/babel-plugin-cache-busting.ts | 5 + packages/shared-internals/src/index.ts | 7 ++ .../src/template-colocation-plugin.ts | 2 + pnpm-lock.yaml | 17 +-- 9 files changed, 67 insertions(+), 93 deletions(-) rename packages/{core => compat}/src/rename-require-plugin.ts (100%) delete mode 100644 packages/core/tests/rename-require-plugin.test.ts diff --git a/packages/compat/package.json b/packages/compat/package.json index fa411972b..bdd064c12 100644 --- a/packages/compat/package.json +++ b/packages/compat/package.json @@ -26,6 +26,7 @@ "@babel/code-frame": "^7.14.5", "@babel/core": "^7.14.5", "@babel/plugin-syntax-dynamic-import": "^7.8.3", + "@babel/plugin-transform-runtime": "^7.14.5", "@babel/preset-env": "^7.14.5", "@babel/traverse": "^7.14.5", "@embroider/macros": "workspace:*", diff --git a/packages/compat/src/compat-app.ts b/packages/compat/src/compat-app.ts index 854d7e57a..5fa0f7105 100644 --- a/packages/compat/src/compat-app.ts +++ b/packages/compat/src/compat-app.ts @@ -19,6 +19,9 @@ import { debug, warn, jsHandlebarsCompile, + templateColocationPluginPath, + cacheBustingPluginVersion, + cacheBustingPluginPath, } from '@embroider/core'; import V1InstanceCache from './v1-instance-cache'; import V1App from './v1-app'; @@ -119,50 +122,7 @@ class ConcatenatedAsset { type InternalAsset = OnDiskAsset | InMemoryAsset | BuiltEmberAsset | ConcatenatedAsset; -// This runs at broccoli-pipeline-construction time, whereas our actual -// CompatAppAdapter instance only becomes available during tree-building -// time. -function setup(legacyEmberAppInstance: object, options: Required) { - let oldPackage = V1InstanceCache.forApp(legacyEmberAppInstance, options).app; - - let { appJS } = oldPackage.processAppJS(); - let htmlTree = oldPackage.htmlTree; - let publicTree = oldPackage.publicTree; - let configTree = oldPackage.config; - let appBootTree = oldPackage.appBoot; - - if (options.extraPublicTrees.length > 0) { - publicTree = mergeTrees([publicTree, ...options.extraPublicTrees].filter(Boolean) as BroccoliNode[]); - } - - let inTrees = { - appJS, - htmlTree, - publicTree, - configTree, - appBootTree, - }; - - let instantiate = async (root: string, appSrcDir: string, packageCache: PackageCache) => { - let appPackage = packageCache.get(appSrcDir); - let macrosConfig = MacrosConfig.for(legacyEmberAppInstance, appSrcDir); - - return new CompatAppAdapter( - root, - appPackage, - options, - oldPackage, - configTree, - packageCache.get(join(root, 'node_modules', '@embroider', 'synthesized-vendor')), - packageCache.get(join(root, 'node_modules', '@embroider', 'synthesized-styles')), - macrosConfig - ); - }; - - return { inTrees, instantiate }; -} - -class CompatAppAdapter { +class CompatAppBuilder { // for each relativePath, an Asset we have already emitted private assets: Map = new Map(); @@ -658,10 +618,7 @@ class CompatAppAdapter { // supposed to take care of colocating their own templates explicitly. packageGuard: true, }; - babel.plugins.push([ - require.resolve('@embroider/shared-internals/src/template-colocation-plugin'), - colocationOptions, - ]); + babel.plugins.push([templateColocationPluginPath, colocationOptions]); for (let p of this.jsPlugins(resolverConfig)) { babel.plugins.push(p); @@ -1568,6 +1525,8 @@ interface ExtraTree { __prevStageTree: BroccoliNode; } +// This runs at broccoli-pipeline-construction time, whereas our actual +// CompatAppBuilder instance only becomes available during tree-building time. export default class CompatApp { private inTrees: TreeNames; private annotation = '@embroider/compat/app'; @@ -1581,8 +1540,44 @@ export default class CompatApp { private outputPath: string | undefined; private packageCache: PackageCache | undefined; - constructor(legacyEmberAppInstance: object, private prevStage: Stage, options?: Options) { - let { inTrees, instantiate } = setup(legacyEmberAppInstance, optionsWithDefaults(options)); + constructor(legacyEmberAppInstance: object, private prevStage: Stage, _options?: Options) { + let options = optionsWithDefaults(_options); + let oldPackage = V1InstanceCache.forApp(legacyEmberAppInstance, options).app; + + let { appJS } = oldPackage.processAppJS(); + let htmlTree = oldPackage.htmlTree; + let publicTree = oldPackage.publicTree; + let configTree = oldPackage.config; + let appBootTree = oldPackage.appBoot; + + if (options.extraPublicTrees.length > 0) { + publicTree = mergeTrees([publicTree, ...options.extraPublicTrees].filter(Boolean) as BroccoliNode[]); + } + + let inTrees = { + appJS, + htmlTree, + publicTree, + configTree, + appBootTree, + }; + + let instantiate = async (root: string, appSrcDir: string, packageCache: PackageCache) => { + let appPackage = packageCache.get(appSrcDir); + let macrosConfig = MacrosConfig.for(legacyEmberAppInstance, appSrcDir); + + return new CompatAppBuilder( + root, + appPackage, + options, + oldPackage, + configTree, + packageCache.get(join(root, 'node_modules', '@embroider', 'synthesized-vendor')), + packageCache.get(join(root, 'node_modules', '@embroider', 'synthesized-styles')), + macrosConfig + ); + }; + this.inTrees = inTrees; this.instantiate = instantiate; } @@ -1771,7 +1766,7 @@ function stringOrBufferEqual(a: string | Buffer, b: string | Buffer): boolean { } const babelFilterTemplate = jsHandlebarsCompile(` -const { babelFilter } = require(${JSON.stringify(require.resolve('./index.js'))}); +const { babelFilter } = require(${JSON.stringify(require.resolve('@embroider/core'))}); module.exports = babelFilter({{{json-stringify skipBabel}}}, "{{{js-string-escape appRoot}}}"); `) as (params: { skipBabel: Options['skipBabel']; appRoot: string }) => string; @@ -1800,15 +1795,10 @@ function combinePackageJSON(...layers: object[]) { return mergeWith({}, ...layers, custom); } -const CACHE_BUSTING_PLUGIN = { - path: require.resolve('@embroider/shared-internals/src/babel-plugin-cache-busting.js'), - version: readJSONSync(`${__dirname}/../package.json`).version, -}; - function addCachablePlugin(babelConfig: TransformOptions) { if (Array.isArray(babelConfig.plugins) && babelConfig.plugins.length > 0) { const plugins = Object.create(null); - plugins[CACHE_BUSTING_PLUGIN.path] = CACHE_BUSTING_PLUGIN.version; + plugins[cacheBustingPluginPath] = cacheBustingPluginVersion; for (const plugin of babelConfig.plugins) { let absolutePathToPlugin: string; @@ -1824,7 +1814,7 @@ function addCachablePlugin(babelConfig: TransformOptions) { } babelConfig.plugins.push([ - CACHE_BUSTING_PLUGIN.path, + cacheBustingPluginPath, { plugins, }, diff --git a/packages/core/src/rename-require-plugin.ts b/packages/compat/src/rename-require-plugin.ts similarity index 100% rename from packages/core/src/rename-require-plugin.ts rename to packages/compat/src/rename-require-plugin.ts diff --git a/packages/core/package.json b/packages/core/package.json index 5ceee02c2..487512640 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -22,9 +22,6 @@ "dependencies": { "@babel/core": "^7.14.5", "@babel/parser": "^7.14.5", - "@babel/plugin-syntax-dynamic-import": "^7.8.3", - "@babel/plugin-transform-runtime": "^7.14.5", - "@babel/runtime": "^7.14.5", "@babel/traverse": "^7.14.5", "@embroider/macros": "workspace:*", "@embroider/shared-internals": "workspace:*", @@ -35,7 +32,6 @@ "broccoli-plugin": "^4.0.7", "broccoli-source": "^3.0.1", "debug": "^4.3.2", - "escape-string-regexp": "^4.0.0", "fast-sourcemap-concat": "^1.4.0", "filesize": "^5.0.0", "fs-extra": "^9.1.0", diff --git a/packages/core/tests/rename-require-plugin.test.ts b/packages/core/tests/rename-require-plugin.test.ts deleted file mode 100644 index 97dd44f04..000000000 --- a/packages/core/tests/rename-require-plugin.test.ts +++ /dev/null @@ -1,18 +0,0 @@ -import renameRequire from '../src/rename-require-plugin'; -import { transformSync } from '@babel/core'; - -describe('babel-plugin-adjust-imports', function () { - test('can rename require', function () { - let code = transformSync( - ` - import require from "require"; - function whatever() { - require("x"); - } - `, - { plugins: [renameRequire] } - )!.code!; - expect(code).not.toMatch(/ require /); - expect(code).not.toMatch(/ require\(/); - }); -}); diff --git a/packages/shared-internals/src/babel-plugin-cache-busting.ts b/packages/shared-internals/src/babel-plugin-cache-busting.ts index 16d25884e..98abaa271 100644 --- a/packages/shared-internals/src/babel-plugin-cache-busting.ts +++ b/packages/shared-internals/src/babel-plugin-cache-busting.ts @@ -1,3 +1,5 @@ +import { readJSONSync } from 'fs-extra'; + export default function makePlugin(): any { // Dear future @rwjblue, // @@ -9,3 +11,6 @@ export default function makePlugin(): any { // Contributor return {}; } + +export const pluginPath = __filename; +export const version = readJSONSync(`${__dirname}/../package.json`); diff --git a/packages/shared-internals/src/index.ts b/packages/shared-internals/src/index.ts index 5f91776c5..bdabe88f0 100644 --- a/packages/shared-internals/src/index.ts +++ b/packages/shared-internals/src/index.ts @@ -12,4 +12,11 @@ export { hbsToJS } from './hbs-to-js'; export { default as templateColocationPlugin, Options as TemplateColocationPluginOptions, + pluginPath as templateColocationPluginPath, } from './template-colocation-plugin'; + +export { + default as cacheBustingPlugin, + pluginPath as cacheBustingPluginPath, + version as cacheBustingPluginVersion, +} from './babel-plugin-cache-busting'; diff --git a/packages/shared-internals/src/template-colocation-plugin.ts b/packages/shared-internals/src/template-colocation-plugin.ts index a9d0299ac..3e6646726 100644 --- a/packages/shared-internals/src/template-colocation-plugin.ts +++ b/packages/shared-internals/src/template-colocation-plugin.ts @@ -6,6 +6,8 @@ import { dirname } from 'path'; import { explicitRelative, PackageCache } from '.'; import { ImportUtil } from 'babel-import-util'; +export const pluginPath = __filename; + // these options are designed so the defaults are appropriate for use within an // addon's dev pipeline, whereas when we use it within Embroider we diverge from // the defaults. That means less options for addon authors to need to know diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index cc1cb01e9..8c8acfc2f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -156,6 +156,9 @@ importers: '@babel/plugin-syntax-dynamic-import': specifier: ^7.8.3 version: 7.8.3(@babel/core@7.19.6) + '@babel/plugin-transform-runtime': + specifier: ^7.14.5 + version: 7.18.6(@babel/core@7.19.6) '@babel/preset-env': specifier: ^7.14.5 version: 7.16.11(@babel/core@7.19.6) @@ -343,15 +346,6 @@ importers: '@babel/parser': specifier: ^7.14.5 version: 7.14.5 - '@babel/plugin-syntax-dynamic-import': - specifier: ^7.8.3 - version: 7.8.3(@babel/core@7.19.6) - '@babel/plugin-transform-runtime': - specifier: ^7.14.5 - version: 7.18.6(@babel/core@7.19.6) - '@babel/runtime': - specifier: ^7.14.5 - version: 7.18.6 '@babel/traverse': specifier: ^7.14.5 version: 7.14.5 @@ -382,9 +376,6 @@ importers: debug: specifier: ^4.3.2 version: 4.3.2(supports-color@8.1.0) - escape-string-regexp: - specifier: ^4.0.0 - version: 4.0.0 fast-sourcemap-concat: specifier: ^1.4.0 version: 1.4.0 @@ -6800,7 +6791,7 @@ packages: glob: 7.2.3 pkg-up: 2.0.0 reselect: 3.0.1 - resolve: 1.20.0 + resolve: 1.22.2 /babel-plugin-module-resolver@4.1.0: resolution: {integrity: sha512-MlX10UDheRr3lb3P0WcaIdtCSRlxdQsB1sBqL7W0raF070bGl1HQQq5K3T2vf2XAYie+ww+5AKC/WrkjRO2knA==} From 25079f3870ea11d3edc4e137094402789ae18cc3 Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Tue, 30 May 2023 09:08:56 -0400 Subject: [PATCH 08/72] marking private --- packages/compat/src/compat-app.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/compat/src/compat-app.ts b/packages/compat/src/compat-app.ts index 5fa0f7105..6470b05f9 100644 --- a/packages/compat/src/compat-app.ts +++ b/packages/compat/src/compat-app.ts @@ -163,7 +163,7 @@ class CompatAppBuilder { return this.oldPackage.env; } - extractAssets(treePaths: OutputPaths): Asset[] { + private extractAssets(treePaths: OutputPaths): Asset[] { let assets: Asset[] = []; // Everything in our traditional public tree is an on-disk asset @@ -200,7 +200,7 @@ class CompatAppBuilder { } @Memoize() - findTestemAsset(): Asset | undefined { + private findTestemAsset(): Asset | undefined { let sourcePath; try { sourcePath = resolveSync('ember-cli/lib/broccoli/testem.js', { basedir: this.root }); From 9fcca0b1c3dd0fae97ebd0ad560fa5ba3f621d02 Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Fri, 2 Jun 2023 10:52:28 -0400 Subject: [PATCH 09/72] eliminating tiny methods that are leftover from the AppAdapter split --- packages/compat/src/compat-app.ts | 110 +++++++++--------------------- 1 file changed, 32 insertions(+), 78 deletions(-) diff --git a/packages/compat/src/compat-app.ts b/packages/compat/src/compat-app.ts index 6470b05f9..8de5106be 100644 --- a/packages/compat/src/compat-app.ts +++ b/packages/compat/src/compat-app.ts @@ -54,10 +54,10 @@ import { unlinkSync, writeFileSync, } from 'fs-extra'; -import type { Options as EtcOptions, Transform } from 'babel-plugin-ember-template-compilation'; +import type { Options as EtcOptions } from 'babel-plugin-ember-template-compilation'; import type { Options as ResolverTransformOptions } from './resolver-transform'; import type { Options as AdjustImportsOptions } from './babel-plugin-adjust-imports'; -import type { PluginItem, TransformOptions } from '@babel/core'; +import type { TransformOptions } from '@babel/core'; import { PreparedEmberHTML } from '@embroider/core/src/ember-html'; import { InMemoryAsset, OnDiskAsset, ImplicitAssetPaths } from '@embroider/core/src/asset'; import { makePortable } from '@embroider/core/src/portable-babel-config'; @@ -318,26 +318,6 @@ class CompatAppBuilder { } } - private autoRun(): boolean { - return this.oldPackage.autoRun; - } - - private appBoot(): string | undefined { - return this.oldPackage.appBoot.readAppBoot(); - } - - private mainModule(): string { - return 'app'; - } - - private mainModuleConfig(): unknown { - return this.configTree.readConfig().APP; - } - - private emberENV(): EmberENV { - return this.configTree.readConfig().EmberENV; - } - private modulePrefix(): string { return this.configTree.readConfig().modulePrefix; } @@ -354,10 +334,6 @@ class CompatAppBuilder { return 'ember-source/vendor/ember/ember-template-compiler'; } - private strictV2Format() { - return false; - } - @Memoize() private activeRules() { return activePackageRules(this.options.packageRules.concat(defaultAddonPackageRules()), [ @@ -366,29 +342,6 @@ class CompatAppBuilder { ]); } - private hbsTransforms(resolverConfig: CompatResolverOptions): Transform[] { - if ( - this.options.staticComponents || - this.options.staticHelpers || - this.options.staticModifiers || - (globalThis as any).embroider_audit - ) { - let opts: ResolverTransformOptions = { - appRoot: resolverConfig.appRoot, - }; - return [[require.resolve('./resolver-transform'), opts]]; - } else { - return []; - } - } - - private jsPlugins(resolverConfig: CompatResolverOptions): PluginItem[] { - let pluginConfig: AdjustImportsOptions = { - appRoot: resolverConfig.appRoot, - }; - return [[require.resolve('./babel-plugin-adjust-imports'), pluginConfig]]; - } - private resolverConfig(engines: Engine[]): CompatResolverOptions { let renamePackages = Object.assign({}, ...this.allActiveAddons.map(dep => dep.meta['renamed-packages'])); let renameModules = Object.assign({}, ...this.allActiveAddons.map(dep => dep.meta['renamed-modules'])); @@ -430,14 +383,6 @@ class CompatAppBuilder { return config; } - private htmlbarsPlugins(): Transform[] { - return this.oldPackage.htmlbarsPlugins; - } - - private babelMajorVersion() { - return this.oldPackage.babelMajorVersion(); - } - private scriptPriority(pkg: Package) { switch (pkg.name) { case 'loader.js': @@ -546,15 +491,11 @@ class CompatAppBuilder { return result; } - private originalBabelConfig() { - return this.oldPackage.babelConfig(); - } - // unlike our full config, this one just needs to know how to parse all the // syntax our app can contain. @Memoize() private babelParserConfig(): TransformOptions { - let babel = cloneDeep(this.originalBabelConfig()); + let babel = cloneDeep(this.oldPackage.babelConfig()); if (!babel.plugins) { babel.plugins = []; @@ -568,7 +509,7 @@ class CompatAppBuilder { @Memoize() private babelConfig(resolverConfig: CompatResolverOptions) { - let babel = cloneDeep(this.originalBabelConfig()); + let babel = cloneDeep(this.oldPackage.babelConfig()); if (!babel.plugins) { babel.plugins = []; @@ -620,9 +561,15 @@ class CompatAppBuilder { }; babel.plugins.push([templateColocationPluginPath, colocationOptions]); - for (let p of this.jsPlugins(resolverConfig)) { - babel.plugins.push(p); - } + babel.plugins.push([ + require.resolve('./babel-plugin-adjust-imports'), + (() => { + let pluginConfig: AdjustImportsOptions = { + appRoot: resolverConfig.appRoot, + }; + return pluginConfig; + })(), + ]); // we can use globally shared babel runtime by default babel.plugins.push([ @@ -1073,7 +1020,7 @@ class CompatAppBuilder { this.macrosConfig.finalize(); let appFiles = this.updateAppJS(inputPaths); - let emberENV = this.emberENV(); + let emberENV = this.configTree.readConfig().EmberENV; let assets = this.gatherAssets(inputPaths); let finalAssets = await this.updateAssets(assets, appFiles, emberENV); @@ -1101,15 +1048,14 @@ class CompatAppBuilder { babel: { filename: '_babel_config_.js', isParallelSafe: true, // TODO - majorVersion: this.babelMajorVersion(), + majorVersion: this.oldPackage.babelMajorVersion(), fileFilter: '_babel_filter_.js', }, 'root-url': this.rootURL(), }; - if (!this.strictV2Format()) { - meta['auto-upgraded'] = true; - } + // all compat apps are auto-upgraded, there's no v2 app format here + meta['auto-upgraded'] = true; let pkg = this.combinePackageJSON(meta); writeFileSync(join(this.root, 'package.json'), JSON.stringify(pkg, null, 2), 'utf8'); @@ -1133,7 +1079,7 @@ class CompatAppBuilder { } private etcOptions(resolverConfig: CompatResolverOptions): EtcOptions { - let transforms = this.htmlbarsPlugins(); + let transforms = this.oldPackage.htmlbarsPlugins; let { plugins: macroPlugins, setConfig } = MacrosConfig.transforms(); setConfig(this.macrosConfig); @@ -1141,8 +1087,16 @@ class CompatAppBuilder { transforms.push(macroPlugin as any); } - for (let t of this.hbsTransforms(resolverConfig)) { - transforms.push(t); + if ( + this.options.staticComponents || + this.options.staticHelpers || + this.options.staticModifiers || + (globalThis as any).embroider_audit + ) { + let opts: ResolverTransformOptions = { + appRoot: resolverConfig.appRoot, + }; + transforms.push([require.resolve('./resolver-transform'), opts]); } return { @@ -1271,10 +1225,10 @@ class CompatAppBuilder { let [app, ...childEngines] = engines; let relativePath = `assets/${this.appPackage.name}.js`; return this.appJSAsset(relativePath, app, childEngines, prepared, { - autoRun: this.autoRun(), - appBoot: !this.autoRun() ? this.appBoot() : '', - mainModule: explicitRelative(dirname(relativePath), this.mainModule()), - appConfig: this.mainModuleConfig(), + autoRun: this.oldPackage.autoRun, + appBoot: !this.oldPackage.autoRun ? this.oldPackage.appBoot.readAppBoot() : '', + mainModule: explicitRelative(dirname(relativePath), 'app'), + appConfig: this.configTree.readConfig().APP, }); } From d236b8eefef60af588b3c333d67c0c45184b9880 Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Fri, 2 Jun 2023 11:17:15 -0400 Subject: [PATCH 10/72] remove redundant lint --- packages/compat/src/compat-app.ts | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/packages/compat/src/compat-app.ts b/packages/compat/src/compat-app.ts index 8de5106be..01b9e1a25 100644 --- a/packages/compat/src/compat-app.ts +++ b/packages/compat/src/compat-app.ts @@ -1471,10 +1471,6 @@ class CompatAppBuilder { } } -interface BuilderInstance { - build(inputPaths: OutputPaths): Promise; -} - interface ExtraTree { __prevStageTree: BroccoliNode; } @@ -1484,13 +1480,9 @@ interface ExtraTree { export default class CompatApp { private inTrees: TreeNames; private annotation = '@embroider/compat/app'; - private instantiate: ( - root: string, - appSrcDir: string, - packageCache: PackageCache - ) => Promise>; + private instantiate: (root: string, appSrcDir: string, packageCache: PackageCache) => Promise; - private active: BuilderInstance | undefined; + private active: CompatAppBuilder | undefined; private outputPath: string | undefined; private packageCache: PackageCache | undefined; @@ -1545,7 +1537,6 @@ export default class CompatApp { this.packageCache = packageCache; this.active = await this.instantiate(outputPath, this.prevStage.inputPath, packageCache); } - delete (treePaths as any).__prevStageTree; await this.active.build(this.deAugment(treePaths)); this.deferReady.resolve(); }); From fa9e5363d9c657150da9761ebc776815d81dc1b0 Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Fri, 2 Jun 2023 12:04:45 -0400 Subject: [PATCH 11/72] eliminate V1DummyApp class --- packages/compat/src/v1-app.ts | 58 ++++++++++-------------- packages/compat/src/v1-instance-cache.ts | 2 +- 2 files changed, 26 insertions(+), 34 deletions(-) diff --git a/packages/compat/src/v1-app.ts b/packages/compat/src/v1-app.ts index 107e52d37..a2c79cdbb 100644 --- a/packages/compat/src/v1-app.ts +++ b/packages/compat/src/v1-app.ts @@ -51,29 +51,35 @@ export default class V1App { // used to signal that this is a dummy app owned by a particular addon owningAddon: Package | undefined; - static create(app: EmberAppInstance): V1App { - if (app.project.pkg.keywords?.includes('ember-addon')) { - // we are a dummy app, which is unfortunately weird and special - return new V1DummyApp(app); - } else { - return new V1App(app); - } - } - private _publicAssets: { [filePath: string]: string } = Object.create(null); private _implicitScripts: string[] = []; private _implicitStyles: string[] = []; packageCache: MovablePackageCache; - protected constructor(protected app: EmberAppInstance) { + private get isDummy(): boolean { + return this.app.project.pkg.keywords?.includes('ember-addon') ?? false; + } + + constructor(protected app: EmberAppInstance) { this.packageCache = new MovablePackageCache(MacrosConfig.for(app, this.root), this.root); + + if (this.isDummy) { + this.owningAddon = new OwningAddon(this.app.project.root, this.packageCache); + this.packageCache.seed(this.owningAddon); + this.packageCache.seed(new DummyPackage(this.root, this.owningAddon, this.packageCache)); + } } - // always the name from package.json. Not the one that apps may have weirdly - // customized. get name(): string { - return this.app.project.pkg.name; + if (this.isDummy) { + // here we accept the ember-cli behavior + return this.app.name; + } else { + // always the name from package.json. Not the one that apps may have weirdly + // customized. + return this.app.project.pkg.name; + } } get env(): string { @@ -82,7 +88,12 @@ export default class V1App { @Memoize() get root(): string { - return dirname(pkgUpSync({ cwd: this.app.project.root })!); + if (this.isDummy) { + // this is the Known Hack for finding the true root of the dummy app. + return join(this.app.project.configPath(), '..', '..'); + } else { + return dirname(pkgUpSync({ cwd: this.app.project.root })!); + } } @Memoize() @@ -774,25 +785,6 @@ function throwIfMissing( return asset; } -class V1DummyApp extends V1App { - constructor(app: EmberAppInstance) { - super(app); - this.owningAddon = new OwningAddon(this.app.project.root, this.packageCache); - this.packageCache.seed(this.owningAddon); - this.packageCache.seed(new DummyPackage(this.root, this.owningAddon, this.packageCache)); - } - - get name(): string { - // here we accept the ember-cli behavior - return this.app.name; - } - - get root(): string { - // this is the Known Hack for finding the true root of the dummy app. - return join(this.app.project.configPath(), '..', '..'); - } -} - interface Preprocessors { preprocessJs(tree: Node, a: string, b: string, options: object): Node; preprocessCss(tree: Node, a: string, b: string, options: object): Node; diff --git a/packages/compat/src/v1-instance-cache.ts b/packages/compat/src/v1-instance-cache.ts index 2ad5b31cc..fe2c363a2 100644 --- a/packages/compat/src/v1-instance-cache.ts +++ b/packages/compat/src/v1-instance-cache.ts @@ -29,7 +29,7 @@ export default class V1InstanceCache { orderIdx: number; private constructor(oldApp: any, private options: Required) { - this.app = V1App.create(oldApp); + this.app = new V1App(oldApp); this.orderIdx = 0; // no reason to do this on demand because oldApp already eagerly loaded From 8ebb41dbcb1299d524a40adc304930b06567b122 Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Tue, 6 Jun 2023 15:17:58 -0400 Subject: [PATCH 12/72] merging V1App into CompatApp --- packages/compat/package.json | 1 + packages/compat/src/compat-addons.ts | 10 +- packages/compat/src/compat-app.ts | 932 +++++++++++++++++++++-- packages/compat/src/default-pipeline.ts | 7 +- packages/compat/src/v1-addon.ts | 6 +- packages/compat/src/v1-app.ts | 791 ------------------- packages/compat/src/v1-instance-cache.ts | 8 +- pnpm-lock.yaml | 298 +++++--- 8 files changed, 1061 insertions(+), 992 deletions(-) delete mode 100644 packages/compat/src/v1-app.ts diff --git a/packages/compat/package.json b/packages/compat/package.json index 329feb588..84fbf7c6d 100644 --- a/packages/compat/package.json +++ b/packages/compat/package.json @@ -28,6 +28,7 @@ "@babel/plugin-syntax-dynamic-import": "^7.8.3", "@babel/plugin-transform-runtime": "^7.14.5", "@babel/preset-env": "^7.14.5", + "@babel/runtime": "^7.18.6", "@babel/traverse": "^7.14.5", "@embroider/macros": "workspace:*", "@types/babel__code-frame": "^7.0.2", diff --git a/packages/compat/src/compat-addons.ts b/packages/compat/src/compat-addons.ts index 189b20ce9..867cfc433 100644 --- a/packages/compat/src/compat-addons.ts +++ b/packages/compat/src/compat-addons.ts @@ -1,15 +1,15 @@ import { Node } from 'broccoli-node-api'; import { join, relative, dirname, isAbsolute, sep } from 'path'; import { emptyDirSync, ensureSymlinkSync, ensureDirSync, realpathSync, copySync, writeJSONSync } from 'fs-extra'; -import { Stage, Package, PackageCache, WaitForTrees, mangledEngineRoot } from '@embroider/core'; +import { Stage, Package, PackageCache, WaitForTrees, mangledEngineRoot, EmberAppInstance } from '@embroider/core'; import V1InstanceCache from './v1-instance-cache'; import { MovedPackageCache } from './moved-package-cache'; import { Memoize } from 'typescript-memoize'; import buildCompatAddon from './build-compat-addon'; import Options, { optionsWithDefaults } from './options'; -import V1App from './v1-app'; import TreeSync from 'tree-sync'; import { WatchedDir } from 'broccoli-source'; +import CompatApp from './compat-app'; // This build stage expects to be run with broccoli memoization enabled in order // to get good rebuild performance. We turn it on by default here, but you can @@ -30,7 +30,7 @@ export default class CompatAddons implements Stage { private v1Cache: V1InstanceCache; readonly inputPath: string; - constructor(legacyEmberAppInstance: object, maybeOptions?: Options) { + constructor(legacyEmberAppInstance: EmberAppInstance, maybeOptions?: Options) { let options = optionsWithDefaults(maybeOptions); let v1Cache = V1InstanceCache.forApp(legacyEmberAppInstance, options); @@ -40,7 +40,7 @@ export default class CompatAddons implements Stage { ensureDirSync(options.workspaceDir!); this.destDir = realpathSync(options.workspaceDir!); - this.packageCache = v1Cache.app.packageCache.moveAddons(this.destDir); + this.packageCache = v1Cache.app.movablePackageCache.moveAddons(this.destDir); this.inputPath = v1Cache.app.root; this.treeSyncMap = new WeakMap(); this.v1Cache = v1Cache; @@ -221,7 +221,7 @@ export default class CompatAddons implements Stage { } } - private getSyntheticPackages(v1App: V1App, movedAddons: Node[]): { synthVendor: Node; synthStyles: Node } { + private getSyntheticPackages(v1App: CompatApp, movedAddons: Node[]): { synthVendor: Node; synthStyles: Node } { let index = 0; let upgradedAddonTrees = []; for (let [oldPkg] of this.packageCache.moved.entries()) { diff --git a/packages/compat/src/compat-app.ts b/packages/compat/src/compat-app.ts index 01b9e1a25..14abc1590 100644 --- a/packages/compat/src/compat-app.ts +++ b/packages/compat/src/compat-app.ts @@ -1,20 +1,15 @@ import { Node as BroccoliNode } from 'broccoli-node-api'; -import mergeTrees from 'broccoli-merge-trees'; import { - Stage, PackageCache, OutputPaths, Asset, EmberAsset, - Package, AddonPackage, Engine, WaitForTrees, - AddonMeta, AppMeta, explicitRelative, extensionsPattern, - PackageInfo, TemplateColocationPluginOptions, debug, warn, @@ -22,14 +17,11 @@ import { templateColocationPluginPath, cacheBustingPluginVersion, cacheBustingPluginPath, + Stage, } from '@embroider/core'; -import V1InstanceCache from './v1-instance-cache'; -import V1App from './v1-app'; import walkSync from 'walk-sync'; -import { dirname, join, resolve as resolvePath, sep, posix } from 'path'; +import { resolve as resolvePath, posix } from 'path'; import { JSDOM } from 'jsdom'; -import resolve from 'resolve'; -import { V1Config } from './v1-config'; import Options, { optionsWithDefaults } from './options'; import { CompatResolverOptions } from './resolver-transform'; import { activePackageRules, PackageRules } from './dependency-rules'; @@ -39,25 +31,12 @@ import flatten from 'lodash/flatten'; import partition from 'lodash/partition'; import mergeWith from 'lodash/mergeWith'; import cloneDeep from 'lodash/cloneDeep'; -import { Memoize } from 'typescript-memoize'; import { sync as resolveSync } from 'resolve'; -import { MacrosConfig } from '@embroider/macros/src/node'; import bind from 'bind-decorator'; -import { - pathExistsSync, - copySync, - ensureDirSync, - outputJSONSync, - readJSONSync, - statSync, - readdirSync, - unlinkSync, - writeFileSync, -} from 'fs-extra'; +import { outputJSONSync, readJSONSync, statSync, unlinkSync, writeFileSync } from 'fs-extra'; import type { Options as EtcOptions } from 'babel-plugin-ember-template-compilation'; import type { Options as ResolverTransformOptions } from './resolver-transform'; import type { Options as AdjustImportsOptions } from './babel-plugin-adjust-imports'; -import type { TransformOptions } from '@babel/core'; import { PreparedEmberHTML } from '@embroider/core/src/ember-html'; import { InMemoryAsset, OnDiskAsset, ImplicitAssetPaths } from '@embroider/core/src/asset'; import { makePortable } from '@embroider/core/src/portable-babel-config'; @@ -65,10 +44,53 @@ import { AppFiles, EngineSummary, RouteFiles } from '@embroider/core/src/app-fil import { mangledEngineRoot } from '@embroider/core/src/engine-mangler'; import { PortableHint, maybeNodeModuleVersion } from '@embroider/core/src/portable'; import AppDiffer from '@embroider/core/src/app-differ'; -import SourceMapConcat from 'fast-sourcemap-concat'; import assertNever from 'assert-never'; +import { Memoize } from 'typescript-memoize'; +import { sync as pkgUpSync } from 'pkg-up'; +import { join, dirname, isAbsolute, sep } from 'path'; +import buildFunnel from 'broccoli-funnel'; +import mergeTrees from 'broccoli-merge-trees'; +import { WatchedDir } from 'broccoli-source'; +import resolve from 'resolve'; +import { V1Config, WriteV1Config } from './v1-config'; +import { WriteV1AppBoot, ReadV1AppBoot } from './v1-appboot'; +import { + AddonMeta, + Package, + EmberAppInstance, + OutputFileToInputFileMap, + PackageInfo, + AddonInstance, +} from '@embroider/core'; +import { writeJSONSync, ensureDirSync, copySync, readdirSync, pathExistsSync, existsSync } from 'fs-extra'; +import AddToTree from './add-to-tree'; +import DummyPackage, { OwningAddon } from './dummy-package'; +import { TransformOptions } from '@babel/core'; +import { isEmbroiderMacrosPlugin, MacrosConfig } from '@embroider/macros/src/node'; +import resolvePackagePath from 'resolve-package-path'; +import Concat from 'broccoli-concat'; +import mapKeys from 'lodash/mapKeys'; +import SynthesizeTemplateOnlyComponents from './synthesize-template-only-components'; +import { isEmberAutoImportDynamic, isInlinePrecompilePlugin } from './detect-babel-plugins'; +import prepHtmlbarsAstPluginsForUnwrap from './prepare-htmlbars-ast-plugins'; +import { readFileSync } from 'fs'; +import type { Options as HTMLBarsOptions } from 'ember-cli-htmlbars'; +import semver from 'semver'; +import { MovablePackageCache } from './moved-package-cache'; +import type { Transform } from 'babel-plugin-ember-template-compilation'; +import SourceMapConcat from 'fast-sourcemap-concat'; import escapeRegExp from 'escape-string-regexp'; +type EmberCliHTMLBarsAddon = AddonInstance & { + htmlbarsOptions(): HTMLBarsOptions; +}; + +interface Group { + outputFiles: OutputFileToInputFileMap; + implicitKey: '_implicitStyles' | '_implicitScripts'; + vendorOutputPath: 'string'; +} + interface TreeNames { appJS: BroccoliNode; htmlTree: BroccoliNode; @@ -130,7 +152,7 @@ class CompatAppBuilder { private root: string, private appPackage: Package, private options: Required, - private oldPackage: V1App, + private compatApp: CompatApp, private configTree: V1Config, private synthVendor: Package, private synthStyles: Package, @@ -153,16 +175,12 @@ class CompatAppBuilder { @Memoize() private fastbootJSSrcDir(_treePaths: OutputPaths) { - let target = join(this.oldPackage.root, 'fastboot'); + let target = join(this.root, 'fastboot'); if (pathExistsSync(target)) { return target; } } - private get env() { - return this.oldPackage.env; - } - private extractAssets(treePaths: OutputPaths): Asset[] { let assets: Asset[] = []; @@ -185,7 +203,7 @@ class CompatAppBuilder { // ember-cli traditionally outputs a dummy testem.js file to prevent // spurious errors when running tests under "ember s". - if (this.oldPackage.shouldBuildTests) { + if (this.compatApp.shouldBuildTests) { let testemAsset = this.findTestemAsset(); if (testemAsset) { assets.push(testemAsset); @@ -218,8 +236,8 @@ class CompatAppBuilder { } private developingAddons(): string[] { - if (this.oldPackage.owningAddon) { - return [this.oldPackage.owningAddon.root]; + if (this.compatApp.owningAddon) { + return [this.compatApp.owningAddon.root]; } return []; } @@ -285,7 +303,7 @@ class CompatAppBuilder { { entrypoint: 'index.html', includeTests: false }, { entrypoint: 'tests/index.html', includeTests: true }, ]; - if (!this.oldPackage.shouldBuildTests) { + if (!this.compatApp.shouldBuildTests) { classicEntrypoints.pop(); } for (let { entrypoint, includeTests } of classicEntrypoints) { @@ -304,13 +322,13 @@ class CompatAppBuilder { let styles = [...dom.window.document.querySelectorAll('link[rel="stylesheet"]')] as HTMLLinkElement[]; return { - javascript: definitelyReplace(dom, this.oldPackage.findAppScript(scripts, entrypoint)), - styles: definitelyReplace(dom, this.oldPackage.findAppStyles(styles, entrypoint)), - implicitScripts: definitelyReplace(dom, this.oldPackage.findVendorScript(scripts, entrypoint)), - implicitStyles: definitelyReplace(dom, this.oldPackage.findVendorStyles(styles, entrypoint)), - testJavascript: maybeReplace(dom, this.oldPackage.findTestScript(scripts)), - implicitTestScripts: maybeReplace(dom, this.oldPackage.findTestSupportScript(scripts)), - implicitTestStyles: maybeReplace(dom, this.oldPackage.findTestSupportStyles(styles)), + javascript: definitelyReplace(dom, this.compatApp.findAppScript(scripts, entrypoint)), + styles: definitelyReplace(dom, this.compatApp.findAppStyles(styles, entrypoint)), + implicitScripts: definitelyReplace(dom, this.compatApp.findVendorScript(scripts, entrypoint)), + implicitStyles: definitelyReplace(dom, this.compatApp.findVendorStyles(styles, entrypoint)), + testJavascript: maybeReplace(dom, this.compatApp.findTestScript(scripts)), + implicitTestScripts: maybeReplace(dom, this.compatApp.findTestSupportScript(scripts)), + implicitTestStyles: maybeReplace(dom, this.compatApp.findTestSupportStyles(styles)), }; }, }; @@ -495,7 +513,7 @@ class CompatAppBuilder { // syntax our app can contain. @Memoize() private babelParserConfig(): TransformOptions { - let babel = cloneDeep(this.oldPackage.babelConfig()); + let babel = cloneDeep(this.compatApp.babelConfig()); if (!babel.plugins) { babel.plugins = []; @@ -509,7 +527,7 @@ class CompatAppBuilder { @Memoize() private babelConfig(resolverConfig: CompatResolverOptions) { - let babel = cloneDeep(this.oldPackage.babelConfig()); + let babel = cloneDeep(this.compatApp.babelConfig()); if (!babel.plugins) { babel.plugins = []; @@ -1007,7 +1025,7 @@ class CompatAppBuilder { } async build(inputPaths: OutputPaths) { - if (this.env !== 'production') { + if (this.compatApp.env !== 'production') { this.macrosConfig.enablePackageDevelopment(this.root); this.macrosConfig.enableRuntimeMode(); } @@ -1048,7 +1066,7 @@ class CompatAppBuilder { babel: { filename: '_babel_config_.js', isParallelSafe: true, // TODO - majorVersion: this.oldPackage.babelMajorVersion(), + majorVersion: this.compatApp.babelMajorVersion(), fileFilter: '_babel_filter_.js', }, 'root-url': this.rootURL(), @@ -1079,7 +1097,7 @@ class CompatAppBuilder { } private etcOptions(resolverConfig: CompatResolverOptions): EtcOptions { - let transforms = this.oldPackage.htmlbarsPlugins; + let transforms = this.compatApp.htmlbarsPlugins; let { plugins: macroPlugins, setConfig } = MacrosConfig.transforms(); setConfig(this.macrosConfig); @@ -1225,8 +1243,8 @@ class CompatAppBuilder { let [app, ...childEngines] = engines; let relativePath = `assets/${this.appPackage.name}.js`; return this.appJSAsset(relativePath, app, childEngines, prepared, { - autoRun: this.oldPackage.autoRun, - appBoot: !this.oldPackage.autoRun ? this.oldPackage.appBoot.readAppBoot() : '', + autoRun: this.compatApp.autoRun, + appBoot: !this.compatApp.autoRun ? this.compatApp.appBoot.readAppBoot() : '', mainModule: explicitRelative(dirname(relativePath), 'app'), appConfig: this.configTree.readConfig().APP, }); @@ -1486,15 +1504,741 @@ export default class CompatApp { private outputPath: string | undefined; private packageCache: PackageCache | undefined; - constructor(legacyEmberAppInstance: object, private prevStage: Stage, _options?: Options) { + // used to signal that this is a dummy app owned by a particular addon + owningAddon: Package | undefined; + + private _publicAssets: { [filePath: string]: string } = Object.create(null); + private _implicitScripts: string[] = []; + private _implicitStyles: string[] = []; + + movablePackageCache: MovablePackageCache; + + private get isDummy(): boolean { + return this.legacyEmberAppInstance.project.pkg.keywords?.includes('ember-addon') ?? false; + } + + get name(): string { + if (this.isDummy) { + // here we accept the ember-cli behavior + return this.legacyEmberAppInstance.name; + } else { + // always the name from package.json. Not the one that apps may have weirdly + // customized. + return this.legacyEmberAppInstance.project.pkg.name; + } + } + + get env(): string { + return this.legacyEmberAppInstance.env; + } + + @Memoize() + get root(): string { + if (this.isDummy) { + // this is the Known Hack for finding the true root of the dummy app. + return join(this.legacyEmberAppInstance.project.configPath(), '..', '..'); + } else { + return dirname(pkgUpSync({ cwd: this.legacyEmberAppInstance.project.root })!); + } + } + + @Memoize() + private get emberCLILocation() { + const emberCLIPackage = resolvePackagePath('ember-cli', this.root); + + if (emberCLIPackage === null) { + throw new Error(`Embroider: cannot resolve ember-cli's package.json`); + } + + return dirname(emberCLIPackage); + } + + @Memoize() + get hasCompiledStyles() { + return semver.gte(JSON.parse(readFileSync(`${this.emberCLILocation}/package.json`, 'utf8')).version, '3.18.0'); + } + + private requireFromEmberCLI(specifier: string) { + // eslint-disable-next-line @typescript-eslint/no-require-imports + return require(resolve.sync(specifier, { basedir: this.emberCLILocation })); + } + + private get configReplace() { + return this.requireFromEmberCLI('broccoli-config-replace'); + } + + private get configLoader() { + return this.requireFromEmberCLI('broccoli-config-loader'); + } + + private get appUtils() { + return this.requireFromEmberCLI('./lib/utilities/ember-app-utils'); + } + + @Memoize() + get addonTreeCache(): Map { + return new Map(); + } + + @Memoize() + get preprocessRegistry() { + return this.requireFromEmberCLI('ember-cli-preprocess-registry/preprocessors'); + } + + get shouldBuildTests(): boolean { + return this.legacyEmberAppInstance.tests || false; + } + + configPath(): string { + return this.legacyEmberAppInstance.project.configPath(); + } + + private get configTree() { + return new this.configLoader(dirname(this.configPath()), { + env: this.legacyEmberAppInstance.env, + tests: this.legacyEmberAppInstance.tests || false, + project: this.legacyEmberAppInstance.project, + }); + } + + @Memoize() + get config(): V1Config { + return new V1Config(this.configTree, this.legacyEmberAppInstance.env); + } + + get autoRun(): boolean { + return this.legacyEmberAppInstance.options.autoRun; + } + + @Memoize() + get appBoot(): ReadV1AppBoot { + let env = this.legacyEmberAppInstance.env; + let appBootContentTree = new WriteV1AppBoot(); + + let patterns = this.configReplacePatterns; + + appBootContentTree = new this.configReplace(appBootContentTree, this.configTree, { + configPath: join('environments', `${env}.json`), + files: ['config/app-boot.js'], + patterns, + }); + + return new ReadV1AppBoot(appBootContentTree); + } + + private get storeConfigInMeta(): boolean { + return this.legacyEmberAppInstance.options.storeConfigInMeta; + } + + @Memoize() + private get configReplacePatterns() { + return this.appUtils.configReplacePatterns({ + addons: this.legacyEmberAppInstance.project.addons, + autoRun: this.autoRun, + storeConfigInMeta: this.storeConfigInMeta, + }); + } + + get htmlTree() { + if (this.legacyEmberAppInstance.tests) { + return mergeTrees([this.indexTree, this.testIndexTree]); + } else { + return this.indexTree; + } + } + + private get indexTree() { + let indexFilePath = this.legacyEmberAppInstance.options.outputPaths.app.html; + let index = buildFunnel(this.legacyEmberAppInstance.trees.app, { + allowEmpty: true, + include: [`index.html`], + getDestinationPath: () => indexFilePath, + annotation: 'app/index.html', + }); + return new this.configReplace(index, this.configTree, { + configPath: join('environments', `${this.legacyEmberAppInstance.env}.json`), + files: [indexFilePath], + patterns: this.configReplacePatterns, + annotation: 'ConfigReplace/indexTree', + }); + } + + private get testIndexTree() { + let index = buildFunnel(this.legacyEmberAppInstance.trees.tests, { + allowEmpty: true, + include: [`index.html`], + destDir: 'tests', + annotation: 'tests/index.html', + }); + return new this.configReplace(index, this.configTree, { + configPath: join('environments', `test.json`), + files: ['tests/index.html'], + patterns: this.configReplacePatterns, + annotation: 'ConfigReplace/testIndexTree', + }); + } + + @Memoize() + babelConfig(): TransformOptions { + // this finds all the built-in babel configuration that comes with ember-cli-babel + const babelAddon = (this.legacyEmberAppInstance.project as any).findAddonByName('ember-cli-babel'); + const babelConfig = babelAddon.buildBabelOptions({ + 'ember-cli-babel': { + ...this.legacyEmberAppInstance.options['ember-cli-babel'], + includeExternalHelpers: true, + compileModules: false, + disableDebugTooling: false, + disablePresetEnv: false, + disableEmberModulesAPIPolyfill: false, + disableDecoratorTransforms: false, + }, + }); + + let plugins = babelConfig.plugins as any[]; + let presets = babelConfig.presets; + + // this finds any custom babel configuration that's on the app (either + // because the app author explicitly added some, or because addons have + // pushed plugins into it). + let appBabel = this.legacyEmberAppInstance.options.babel; + if (appBabel) { + if (appBabel.plugins) { + plugins = appBabel.plugins.concat(plugins); + } + if (appBabel.presets) { + presets = appBabel.presets.concat(presets); + } + } + + plugins = plugins.filter(p => { + // even if the app was using @embroider/macros, we drop it from the config + // here in favor of our globally-configured one. + return ( + !isEmbroiderMacrosPlugin(p) && + // similarly, if the app was already using an inline template compiler + // babel plugin, we remove it here because we have our own + // always-installed version of that (v2 addons are allowed to assume it + // will be present in the final app build, the app doesn't get to turn + // that off or configure it.) + !isInlinePrecompilePlugin(p) && + !isEmberAutoImportDynamic(p) + ); + }); + + const config: TransformOptions = { + babelrc: false, + plugins, + presets, + // this is here because broccoli-middleware can't render a codeFrame full + // of terminal codes. It would be nice to add something like + // https://github.com/mmalecki/ansispan to broccoli-middleware so we can + // leave color enabled. + highlightCode: false, + }; + + return config; + } + + @Memoize() + babelMajorVersion(): 7 { + let babelAddon = this.legacyEmberAppInstance.project.addons.find((a: any) => a.name === 'ember-cli-babel'); + if (babelAddon) { + let babelAddonMajor = Number(babelAddon.pkg.version.split('.')[0]); + let babelMajor: number | undefined = babelAddonMajor; + if (babelAddonMajor >= 8) { + // `ember-cli-babel` v8 breaks lockstep with Babel, because it now + // defines `@babel/core` as a peer dependency, so we need to check the + // project's version of `@babel/core`: + let babelVersion = this.legacyEmberAppInstance.project.pkg.devDependencies?.['@babel/core']; + if (babelVersion) { + babelMajor = semver.coerce(babelVersion)?.major; + } else { + babelMajor = 7; + } + } + if (babelMajor !== 7) { + throw new Error('`@embroider/compat` only supports apps and addons that use Babel v7.'); + } + return babelMajor; + } + // if we didn't have our own babel plugin at all, it's safe to parse our + // code with 7. + return 7; + } + + @Memoize() + private transformedNodeFiles(): Map { + // any app.imports from node_modules that need custom transforms will need + // to get copied into our own synthesized vendor package. app.imports from + // node_modules that *don't* need custom transforms can just stay where they + // are. + let transformed = new Map(); + for (let transformConfig of this.legacyEmberAppInstance._customTransformsMap.values()) { + for (let filename of transformConfig.files as string[]) { + let preresolved = this.preresolvedNodeFile(filename); + if (preresolved) { + transformed.set(filename, preresolved); + } + } + } + return transformed; + } + + private preresolvedNodeFile(filename: string) { + // this regex is an exact copy of how ember-cli does this, so we align. + let match = filename.match(/^node_modules\/((@[^/]+\/)?[^/]+)\//); + if (match) { + // ember-cli has already done its own resolution of + // `app.import('node_modules/something/...')`, so we go find its answer. + for (let { name, path } of this.legacyEmberAppInstance._nodeModules.values()) { + if (match[1] === name) { + return filename.replace(match[0], path + sep); + } + } + throw new Error(`bug: expected ember-cli to already have a resolved path for asset ${filename}`); + } + } + + private combinedVendor(addonTrees: BroccoliNode[]): BroccoliNode { + let trees = addonTrees.map(tree => + buildFunnel(tree, { + allowEmpty: true, + srcDir: 'vendor', + destDir: 'vendor', + }) + ); + if (this.vendorTree) { + trees.push( + buildFunnel(this.vendorTree, { + destDir: 'vendor', + }) + ); + } + + const tree = mergeTrees(trees, { overwrite: true }); + + const outputGroups: Group[] = [ + // scripts + { + outputFiles: this.legacyEmberAppInstance._scriptOutputFiles, + implicitKey: '_implicitScripts', + vendorOutputPath: this.legacyEmberAppInstance.options.outputPaths.vendor.js, + }, + // styles + { + outputFiles: this.legacyEmberAppInstance._styleOutputFiles, + implicitKey: '_implicitStyles', + vendorOutputPath: this.legacyEmberAppInstance.options.outputPaths.vendor.css, + }, + ]; + + const concatentations = []; + + // support: app.import / outputFile / using + for (let entry of outputGroups) { + const { outputFiles, implicitKey, vendorOutputPath } = entry; + for (let importPath of Object.keys(outputFiles)) { + const headerFiles = outputFiles[importPath]; + + if (importPath === vendorOutputPath) { + // these are the default ember-cli output files vendor.js or + // vendor.css. Let embroider handle these. + this[implicitKey] = headerFiles; + } else if (headerFiles.length === 0) { + // something went really wrong, open an issue + throw new Error('Embroider: EWUT'); + } else if (headerFiles.length === 1) { + // app.import(x, { outputFile: y }); where only one app.imports had this outputFile + // + // No concat needed. Simply serialize the remapping in the addon's + // manifest, this ensures it is included in the final output with no extra work. + this._publicAssets[headerFiles[0]] = importPath; + } else { + // app.import(x, { outputFile: y }); where multiple app.imports share one outputFile + // Concat needed. Perform concat, and include the outputFile in the + // addon's manifest. This ensures it is included in the final output + this._publicAssets[importPath] = importPath; + + concatentations.push( + new Concat(tree, { + headerFiles, + outputFile: importPath, + annotation: `Package ${importPath}`, + separator: '\n;', + sourceMapConfig: this.legacyEmberAppInstance.options['sourcemaps'], + }) + ); + } + } + } + + this.addOtherAssets(); + return mergeTrees([tree, ...concatentations], { overwrite: true }); + } + + addOtherAssets() { + for (let asset of this.legacyEmberAppInstance.otherAssetPaths) { + this._publicAssets[`${asset.src}/${asset.file}`] = `${asset.dest}/${asset.file}`; + } + } + + private addNodeAssets(inputTree: BroccoliNode): BroccoliNode { + let transformedNodeFiles = this.transformedNodeFiles(); + + return new AddToTree(inputTree, outputPath => { + for (let [localDestPath, sourcePath] of transformedNodeFiles) { + let destPath = join(outputPath, localDestPath); + ensureDirSync(dirname(destPath)); + copySync(sourcePath, destPath); + } + + let remapAsset = this.remapAsset.bind(this); + + let addonMeta: AddonMeta = { + type: 'addon', + version: 2, + 'implicit-scripts': this._implicitScripts.map(remapAsset), + 'implicit-styles': this._implicitStyles.map(remapAsset), + 'implicit-test-scripts': this.legacyEmberAppInstance.legacyTestFilesToAppend.map(remapAsset), + 'implicit-test-styles': this.legacyEmberAppInstance.vendorTestStaticStyles.map(remapAsset), + 'public-assets': mapKeys(this._publicAssets, (_, key) => remapAsset(key)), + }; + let meta: PackageInfo = { + name: '@embroider/synthesized-vendor', + version: '0.0.0', + keywords: ['ember-addon'], + 'ember-addon': addonMeta, + }; + writeJSONSync(join(outputPath, 'package.json'), meta, { spaces: 2 }); + }); + } + + synthesizeVendorPackage(addonTrees: BroccoliNode[]): BroccoliNode { + return this.applyCustomTransforms(this.addNodeAssets(this.combinedVendor(addonTrees))); + } + + private combinedStyles(addonTrees: BroccoliNode[]): BroccoliNode { + let trees: BroccoliNode[] = addonTrees.map(tree => + buildFunnel(tree, { + allowEmpty: true, + srcDir: '_app_styles_', + }) + ); + let appStyles = this.legacyEmberAppInstance.trees.styles as BroccoliNode | undefined; + if (appStyles) { + // Workaround for https://github.com/ember-cli/ember-cli/issues/9020 + // + // The default app styles tree is unwatched and relies on side effects + // elsewhere in ember-cli's build pipeline to actually get rebuilds to + // work. Here we need it to actually be watched properly if we want to + // rely on it, particularly when using BROCCOLI_ENABLED_MEMOIZE. + if ((appStyles as any)._watched === false && (appStyles as any)._directoryPath) { + appStyles = new WatchedDir((appStyles as any)._directoryPath); + } + trees.push(appStyles); + } + return mergeTrees(trees, { overwrite: true, annotation: 'embroider-v1-app-combined-styles' }); + } + + synthesizeStylesPackage(addonTrees: BroccoliNode[]): BroccoliNode { + let options = { + // we're deliberately not allowing this to be customized. It's an + // internal implementation detail, and respecting outputPaths here is + // unnecessary complexity. The corresponding code that adjusts the HTML + // is in updateHTML in app.ts. + outputPaths: { app: `/assets/${this.name}.css` }, + registry: this.legacyEmberAppInstance.registry, + minifyCSS: this.legacyEmberAppInstance.options.minifyCSS.options, + }; + + let nestedInput = buildFunnel(this.combinedStyles(addonTrees), { destDir: 'app/styles' }); + let styles = this.preprocessors.preprocessCss(nestedInput, '/app/styles', '/assets', options); + + return new AddToTree(styles, outputPath => { + let addonMeta: AddonMeta = { + type: 'addon', + version: 2, + 'public-assets': {}, + }; + let assetPath = join(outputPath, 'assets'); + if (pathExistsSync(assetPath)) { + for (let file of readdirSync(assetPath)) { + addonMeta['public-assets']![`./assets/${file}`] = `/assets/${file}`; + } + } + let meta: PackageInfo = { + name: '@embroider/synthesized-styles', + version: '0.0.0', + keywords: ['ember-addon'], + 'ember-addon': addonMeta, + }; + writeJSONSync(join(outputPath, 'package.json'), meta, { spaces: 2 }); + }); + } + + // this is taken nearly verbatim from ember-cli. + private applyCustomTransforms(externalTree: BroccoliNode) { + for (let customTransformEntry of this.legacyEmberAppInstance._customTransformsMap) { + let transformName = customTransformEntry[0]; + let transformConfig = customTransformEntry[1]; + + let transformTree = buildFunnel(externalTree, { + files: transformConfig.files, + annotation: `Funnel (custom transform: ${transformName})`, + }); + + externalTree = mergeTrees([externalTree, transformConfig.callback(transformTree, transformConfig.options)], { + annotation: `TreeMerger (custom transform: ${transformName})`, + overwrite: true, + }); + } + return externalTree; + } + + private remapAsset(asset: string) { + if (this.transformedNodeFiles().has(asset)) { + // transformed node assets become local paths, because we have copied + // those ones into our synthesized vendor package. + return './' + asset; + } + let preresolved = this.preresolvedNodeFile(asset); + if (preresolved) { + // non-transformed node assets point directly at their pre-resolved + // original files (this is an absolute path). + return preresolved; + } + // non node assets are local paths. They need an explicit `/` or `.` at + // the start. + if (asset.startsWith('.') || isAbsolute(asset)) { + return asset; + } + return './' + asset; + } + + private preprocessJS(tree: BroccoliNode): BroccoliNode { + // we're saving all our babel compilation for the final stage packager + this.legacyEmberAppInstance.registry.remove('js', 'ember-cli-babel'); + + // auto-import is supported natively so we don't need it here + this.legacyEmberAppInstance.registry.remove('js', 'ember-auto-import-analyzer'); + + tree = buildFunnel(tree, { destDir: this.name }); + + tree = this.preprocessors.preprocessJs(tree, `/`, '/', { + annotation: 'v1-app-preprocess-js', + registry: this.legacyEmberAppInstance.registry, + }); + + tree = buildFunnel(tree, { srcDir: this.name }); + + return tree; + } + + get htmlbarsPlugins(): Transform[] { + let addon = this.legacyEmberAppInstance.project.addons.find( + (a: AddonInstance) => a.name === 'ember-cli-htmlbars' + ) as unknown as EmberCliHTMLBarsAddon; + let options = addon.htmlbarsOptions(); + if (options?.plugins?.ast) { + // even if the app was using @embroider/macros, we drop it from the config + // here in favor of our globally-configured one. + options.plugins.ast = options.plugins.ast.filter((p: any) => !isEmbroiderMacrosPlugin(p)); + prepHtmlbarsAstPluginsForUnwrap(this.legacyEmberAppInstance.registry); + + // classically, this list was backwards for silly historic reasons. But + // we're the compatibility system, so we're putting it back into + // reasonable order. + options.plugins.ast.reverse(); + + return options.plugins.ast; + } else { + return []; + } + } + + // our own appTree. Not to be confused with the one that combines the app js + // from all addons too. + private get appTree(): BroccoliNode { + return this.preprocessJS( + buildFunnel(this.legacyEmberAppInstance.trees.app, { + exclude: ['styles/**', '*.html'], + }) + ); + } + + private get testsTree(): BroccoliNode | undefined { + if (this.shouldBuildTests && this.legacyEmberAppInstance.trees.tests) { + return this.preprocessJS( + buildFunnel(this.legacyEmberAppInstance.trees.tests, { + destDir: 'tests', + }) + ); + } + } + + private get lintTree(): BroccoliNode | undefined { + if (this.shouldBuildTests) { + return this.legacyEmberAppInstance.getLintTests(); + } + } + + get vendorTree(): BroccoliNode | undefined { + return this.ensureTree(this.legacyEmberAppInstance.trees.vendor); + } + + private ensureTree(maybeTree: string | BroccoliNode | undefined): BroccoliNode | undefined { + if (typeof maybeTree === 'string') { + // this is deliberately mimicking how ember-cli does it. We don't use + // `this.root` on purpose, because that can differ from what ember-cli + // considers the project.root. And we don't use path.resolve even though + // that seems possibly more correct, because ember-cli always assumes the + // input is relative. + let resolvedPath = join(this.legacyEmberAppInstance.project.root, maybeTree); + if (existsSync(resolvedPath)) { + return new WatchedDir(maybeTree); + } else { + return undefined; + } + } + return maybeTree; + } + + @Memoize() + private get preprocessors(): Preprocessors { + return this.requireFromEmberCLI('ember-cli-preprocess-registry/preprocessors'); + } + + get publicTree(): BroccoliNode | undefined { + return this.ensureTree(this.legacyEmberAppInstance.trees.public); + } + + processAppJS(): { appJS: BroccoliNode } { + let appTree = this.appTree; + let testsTree = this.testsTree; + let lintTree = this.lintTree; + let config = new WriteV1Config(this.config, this.storeConfigInMeta); + let patterns = this.configReplacePatterns; + let configReplaced = new this.configReplace(config, this.configTree, { + configPath: join('environments', `${this.legacyEmberAppInstance.env}.json`), + files: ['config/environment.js'], + patterns, + }); + + let trees: BroccoliNode[] = []; + trees.push(appTree); + trees.push( + new SynthesizeTemplateOnlyComponents(appTree, { allowedPaths: ['components'], templateExtensions: ['.hbs'] }) + ); + + trees.push(configReplaced); + if (testsTree) { + trees.push(testsTree); + } + if (lintTree) { + trees.push(lintTree); + } + return { + appJS: mergeTrees(trees, { overwrite: true }), + }; + } + + private withoutRootURL(src: string) { + let rootURL = this.config.readConfig().rootURL; + if ((src.startsWith(rootURL) && rootURL) || (!rootURL && !src.startsWith('/'))) { + src = '/' + src.slice(rootURL.length); + } else if (src.startsWith('/' + rootURL)) { + src = src.slice(rootURL.length); + } + return src; + } + + findAppScript(scripts: HTMLScriptElement[], entrypoint: string): HTMLScriptElement { + let appJS = scripts.find( + script => this.withoutRootURL(script.src) === this.legacyEmberAppInstance.options.outputPaths.app.js + ); + return throwIfMissing( + appJS, + this.legacyEmberAppInstance.options.outputPaths.app.js, + scripts.map(s => s.src), + entrypoint, + 'app javascript' + ); + } + + findAppStyles(styles: HTMLLinkElement[], entrypoint: string): HTMLLinkElement { + let style = styles.find( + style => this.withoutRootURL(style.href) === this.legacyEmberAppInstance.options.outputPaths.app.css.app + ); + return throwIfMissing( + style, + this.legacyEmberAppInstance.options.outputPaths.app.css.app, + styles.map(s => s.href), + entrypoint, + 'app css' + ); + } + + findVendorScript(scripts: HTMLScriptElement[], entrypoint: string): HTMLScriptElement { + let vendor = scripts.find( + script => this.withoutRootURL(script.src) === this.legacyEmberAppInstance.options.outputPaths.vendor.js + ); + return throwIfMissing( + vendor, + this.legacyEmberAppInstance.options.outputPaths.vendor.js, + scripts.map(s => s.src), + entrypoint, + 'vendor javascript' + ); + } + + findVendorStyles(styles: HTMLLinkElement[], entrypoint: string): HTMLLinkElement { + let vendorStyle = styles.find( + style => this.withoutRootURL(style.href) === this.legacyEmberAppInstance.options.outputPaths.vendor.css + ); + return throwIfMissing( + vendorStyle, + this.legacyEmberAppInstance.options.outputPaths.vendor.css, + styles.map(s => s.href), + entrypoint, + 'vendor css' + ); + } + + findTestSupportStyles(styles: HTMLLinkElement[]): HTMLLinkElement | undefined { + return styles.find( + style => this.withoutRootURL(style.href) === this.legacyEmberAppInstance.options.outputPaths.testSupport.css + ); + } + + findTestSupportScript(scripts: HTMLScriptElement[]): HTMLScriptElement | undefined { + return scripts.find( + script => + this.withoutRootURL(script.src) === this.legacyEmberAppInstance.options.outputPaths.testSupport.js.testSupport + ); + } + + findTestScript(scripts: HTMLScriptElement[]): HTMLScriptElement | undefined { + return scripts.find( + script => this.withoutRootURL(script.src) === this.legacyEmberAppInstance.options.outputPaths.tests.js + ); + } + + constructor(private legacyEmberAppInstance: EmberAppInstance, _options?: Options) { let options = optionsWithDefaults(_options); - let oldPackage = V1InstanceCache.forApp(legacyEmberAppInstance, options).app; - let { appJS } = oldPackage.processAppJS(); - let htmlTree = oldPackage.htmlTree; - let publicTree = oldPackage.publicTree; - let configTree = oldPackage.config; - let appBootTree = oldPackage.appBoot; + this.movablePackageCache = new MovablePackageCache(MacrosConfig.for(legacyEmberAppInstance, this.root), this.root); + + if (this.isDummy) { + this.owningAddon = new OwningAddon(legacyEmberAppInstance.project.root, this.movablePackageCache); + this.movablePackageCache.seed(this.owningAddon); + this.movablePackageCache.seed(new DummyPackage(this.root, this.owningAddon, this.movablePackageCache)); + } + + let { appJS } = this.processAppJS(); + let htmlTree = this.htmlTree; + let publicTree = this.publicTree; + let configTree = this.config; + let appBootTree = this.appBoot; if (options.extraPublicTrees.length > 0) { publicTree = mergeTrees([publicTree, ...options.extraPublicTrees].filter(Boolean) as BroccoliNode[]); @@ -1516,7 +2260,7 @@ export default class CompatApp { root, appPackage, options, - oldPackage, + this, configTree, packageCache.get(join(root, 'node_modules', '@embroider', 'synthesized-vendor')), packageCache.get(join(root, 'node_modules', '@embroider', 'synthesized-styles')), @@ -1528,29 +2272,33 @@ export default class CompatApp { this.instantiate = instantiate; } - @Memoize() - get tree(): BroccoliNode { - return new WaitForTrees(this.augment(this.inTrees), this.annotation, async treePaths => { - if (!this.active) { - let { outputPath, packageCache } = await this.prevStage.ready(); - this.outputPath = outputPath; - this.packageCache = packageCache; - this.active = await this.instantiate(outputPath, this.prevStage.inputPath, packageCache); - } - await this.active.build(this.deAugment(treePaths)); - this.deferReady.resolve(); - }); - } - - get inputPath(): string { - return this.prevStage.inputPath; - } + asStage(prevStage: Stage): Stage { + let tree = () => + new WaitForTrees(this.augment(this.inTrees, prevStage.tree), this.annotation, async treePaths => { + if (!this.active) { + let { outputPath, packageCache } = await prevStage.ready(); + this.outputPath = outputPath; + this.packageCache = packageCache; + this.active = await this.instantiate(outputPath, prevStage.inputPath, packageCache); + } + await this.active.build(this.deAugment(treePaths)); + this.deferReady.resolve(); + }); - async ready(): Promise<{ outputPath: string; packageCache: PackageCache }> { - await this.deferReady.promise; return { - outputPath: this.outputPath!, - packageCache: this.packageCache!, + get inputPath() { + return prevStage.inputPath; + }, + ready: async () => { + await this.deferReady.promise; + return { + outputPath: this.outputPath!, + packageCache: this.packageCache!, + }; + }, + get tree() { + return tree(); + }, }; } @@ -1561,8 +2309,8 @@ export default class CompatApp { return { resolve: resolve!, promise }; } - private augment(inTrees: TreeNames): TreeNames & ExtraTree { - return Object.assign({ __prevStageTree: this.prevStage.tree }, inTrees); + private augment(inTrees: TreeNames, prevStageTree: BroccoliNode): TreeNames & ExtraTree { + return Object.assign({ __prevStageTree: prevStageTree }, inTrees); } private deAugment(treePaths: OutputPaths): OutputPaths { @@ -1770,3 +2518,27 @@ function addCachablePlugin(babelConfig: TransformOptions) { function excludeDotFiles(files: string[]) { return files.filter(file => !file.startsWith('.') && !file.includes('/.')); } +function throwIfMissing( + asset: T | undefined, + needle: string, + haystack: string[], + entryfile: string, + context: string +): T { + if (!asset) { + throw new Error( + `Could not find ${context}: "${needle}" in ${entryfile}. Found the following instead:\n${haystack + .map(asset => ` - ${asset}`) + .join( + '\n' + )}\n\nFor more information about this error: https://github.com/thoov/stitch/wiki/Could-not-find-asset-in-entry-file-error-help` + ); + } + + return asset; +} + +interface Preprocessors { + preprocessJs(tree: BroccoliNode, a: string, b: string, options: object): BroccoliNode; + preprocessCss(tree: BroccoliNode, a: string, b: string, options: object): BroccoliNode; +} diff --git a/packages/compat/src/default-pipeline.ts b/packages/compat/src/default-pipeline.ts index 95f56a934..ec976d1cc 100644 --- a/packages/compat/src/default-pipeline.ts +++ b/packages/compat/src/default-pipeline.ts @@ -49,15 +49,16 @@ export default function defaultPipeline( return mergeTrees([addons.tree, writeFile('.stage1-output', () => outputPath)]); } - let embroiderApp = new App(emberApp, addons, options); + let embroiderApp = new App(emberApp, options); + let appStage = embroiderApp.asStage(addons); if (process.env.STAGE2_ONLY || !packager) { - return mergeTrees([embroiderApp.tree, writeFile('.stage2-output', () => outputPath)]); + return mergeTrees([appStage.tree, writeFile('.stage2-output', () => outputPath)]); } let BroccoliPackager = toBroccoliPlugin(packager); let variants = (options && options.variants) || defaultVariants(emberApp); - return new BroccoliPackager(embroiderApp, variants, options && options.packagerOptions); + return new BroccoliPackager(appStage, variants, options && options.packagerOptions); } function hasFastboot(emberApp: EmberAppInstance | EmberAppInstance) { diff --git a/packages/compat/src/v1-addon.ts b/packages/compat/src/v1-addon.ts index 38e25c385..6380b4a40 100644 --- a/packages/compat/src/v1-addon.ts +++ b/packages/compat/src/v1-addon.ts @@ -18,7 +18,6 @@ import ObserveTree from './observe-tree'; import type { Options as HTMLBarsOptions } from 'ember-cli-htmlbars'; import { isEmbroiderMacrosPlugin } from '@embroider/macros/src/node'; import { TransformOptions, PluginItem } from '@babel/core'; -import V1App from './v1-app'; import modulesCompat from './modules-compat'; import writeFile from 'broccoli-file-creator'; import SynthesizeTemplateOnlyComponents from './synthesize-template-only-components'; @@ -33,6 +32,7 @@ import { fromPairs } from 'lodash'; import prepHtmlbarsAstPluginsForUnwrap from './prepare-htmlbars-ast-plugins'; import getRealAddon from './get-real-addon'; import type { Options as EtcOptions } from 'babel-plugin-ember-template-compilation'; +import CompatApp from './compat-app'; const stockTreeNames: AddonTreePath[] = Object.freeze([ 'addon', @@ -82,7 +82,7 @@ export default class V1Addon { constructor( protected addonInstance: AddonInstance, protected addonOptions: Required, - protected app: V1App, + protected app: CompatApp, private packageCache: PackageCache, private orderIdx: number ) { @@ -1068,7 +1068,7 @@ export interface V1AddonConstructor { new ( addonInstance: any, options: Required, - app: V1App, + app: CompatApp, packageCache: PackageCache, orderIdx: number ): V1Addon; diff --git a/packages/compat/src/v1-app.ts b/packages/compat/src/v1-app.ts deleted file mode 100644 index a2c79cdbb..000000000 --- a/packages/compat/src/v1-app.ts +++ /dev/null @@ -1,791 +0,0 @@ -import { Memoize } from 'typescript-memoize'; -import { sync as pkgUpSync } from 'pkg-up'; -import { join, dirname, isAbsolute, sep } from 'path'; -import buildFunnel from 'broccoli-funnel'; -import mergeTrees from 'broccoli-merge-trees'; -import { WatchedDir } from 'broccoli-source'; -import resolve from 'resolve'; -import { Node } from 'broccoli-node-api'; -import { V1Config, WriteV1Config } from './v1-config'; -import { WriteV1AppBoot, ReadV1AppBoot } from './v1-appboot'; -import { - AddonMeta, - Package, - EmberAppInstance, - OutputFileToInputFileMap, - PackageInfo, - AddonInstance, -} from '@embroider/core'; -import { writeJSONSync, ensureDirSync, copySync, readdirSync, pathExistsSync, existsSync } from 'fs-extra'; -import AddToTree from './add-to-tree'; -import DummyPackage, { OwningAddon } from './dummy-package'; -import { TransformOptions } from '@babel/core'; -import { isEmbroiderMacrosPlugin, MacrosConfig } from '@embroider/macros/src/node'; -import resolvePackagePath from 'resolve-package-path'; -import Concat from 'broccoli-concat'; -import mapKeys from 'lodash/mapKeys'; -import SynthesizeTemplateOnlyComponents from './synthesize-template-only-components'; -import { isEmberAutoImportDynamic, isInlinePrecompilePlugin } from './detect-babel-plugins'; -import prepHtmlbarsAstPluginsForUnwrap from './prepare-htmlbars-ast-plugins'; -import { readFileSync } from 'fs'; -import type { Options as HTMLBarsOptions } from 'ember-cli-htmlbars'; -import semver from 'semver'; -import { MovablePackageCache } from './moved-package-cache'; - -import type { Transform } from 'babel-plugin-ember-template-compilation'; - -// This controls and types the interface between our new world and the classic -// v1 app instance. - -type EmberCliHTMLBarsAddon = AddonInstance & { - htmlbarsOptions(): HTMLBarsOptions; -}; - -interface Group { - outputFiles: OutputFileToInputFileMap; - implicitKey: '_implicitStyles' | '_implicitScripts'; - vendorOutputPath: 'string'; -} - -export default class V1App { - // used to signal that this is a dummy app owned by a particular addon - owningAddon: Package | undefined; - - private _publicAssets: { [filePath: string]: string } = Object.create(null); - private _implicitScripts: string[] = []; - private _implicitStyles: string[] = []; - - packageCache: MovablePackageCache; - - private get isDummy(): boolean { - return this.app.project.pkg.keywords?.includes('ember-addon') ?? false; - } - - constructor(protected app: EmberAppInstance) { - this.packageCache = new MovablePackageCache(MacrosConfig.for(app, this.root), this.root); - - if (this.isDummy) { - this.owningAddon = new OwningAddon(this.app.project.root, this.packageCache); - this.packageCache.seed(this.owningAddon); - this.packageCache.seed(new DummyPackage(this.root, this.owningAddon, this.packageCache)); - } - } - - get name(): string { - if (this.isDummy) { - // here we accept the ember-cli behavior - return this.app.name; - } else { - // always the name from package.json. Not the one that apps may have weirdly - // customized. - return this.app.project.pkg.name; - } - } - - get env(): string { - return this.app.env; - } - - @Memoize() - get root(): string { - if (this.isDummy) { - // this is the Known Hack for finding the true root of the dummy app. - return join(this.app.project.configPath(), '..', '..'); - } else { - return dirname(pkgUpSync({ cwd: this.app.project.root })!); - } - } - - @Memoize() - private get emberCLILocation() { - const emberCLIPackage = resolvePackagePath('ember-cli', this.root); - - if (emberCLIPackage === null) { - throw new Error(`Embroider: cannot resolve ember-cli's package.json`); - } - - return dirname(emberCLIPackage); - } - - @Memoize() - get hasCompiledStyles() { - return semver.gte(JSON.parse(readFileSync(`${this.emberCLILocation}/package.json`, 'utf8')).version, '3.18.0'); - } - - private requireFromEmberCLI(specifier: string) { - // eslint-disable-next-line @typescript-eslint/no-require-imports - return require(resolve.sync(specifier, { basedir: this.emberCLILocation })); - } - - private get configReplace() { - return this.requireFromEmberCLI('broccoli-config-replace'); - } - - private get configLoader() { - return this.requireFromEmberCLI('broccoli-config-loader'); - } - - private get appUtils() { - return this.requireFromEmberCLI('./lib/utilities/ember-app-utils'); - } - - @Memoize() - get addonTreeCache(): Map { - return new Map(); - } - - @Memoize() - get preprocessRegistry() { - return this.requireFromEmberCLI('ember-cli-preprocess-registry/preprocessors'); - } - - get shouldBuildTests(): boolean { - return this.app.tests || false; - } - - configPath(): string { - return this.app.project.configPath(); - } - - private get configTree() { - return new this.configLoader(dirname(this.configPath()), { - env: this.app.env, - tests: this.app.tests || false, - project: this.app.project, - }); - } - - @Memoize() - get config(): V1Config { - return new V1Config(this.configTree, this.app.env); - } - - get autoRun(): boolean { - return this.app.options.autoRun; - } - - @Memoize() - get appBoot(): ReadV1AppBoot { - let env = this.app.env; - let appBootContentTree = new WriteV1AppBoot(); - - let patterns = this.configReplacePatterns; - - appBootContentTree = new this.configReplace(appBootContentTree, this.configTree, { - configPath: join('environments', `${env}.json`), - files: ['config/app-boot.js'], - patterns, - }); - - return new ReadV1AppBoot(appBootContentTree); - } - - private get storeConfigInMeta(): boolean { - return this.app.options.storeConfigInMeta; - } - - @Memoize() - private get configReplacePatterns() { - return this.appUtils.configReplacePatterns({ - addons: this.app.project.addons, - autoRun: this.autoRun, - storeConfigInMeta: this.storeConfigInMeta, - }); - } - - get htmlTree() { - if (this.app.tests) { - return mergeTrees([this.indexTree, this.testIndexTree]); - } else { - return this.indexTree; - } - } - - private get indexTree() { - let indexFilePath = this.app.options.outputPaths.app.html; - let index = buildFunnel(this.app.trees.app, { - allowEmpty: true, - include: [`index.html`], - getDestinationPath: () => indexFilePath, - annotation: 'app/index.html', - }); - return new this.configReplace(index, this.configTree, { - configPath: join('environments', `${this.app.env}.json`), - files: [indexFilePath], - patterns: this.configReplacePatterns, - annotation: 'ConfigReplace/indexTree', - }); - } - - private get testIndexTree() { - let index = buildFunnel(this.app.trees.tests, { - allowEmpty: true, - include: [`index.html`], - destDir: 'tests', - annotation: 'tests/index.html', - }); - return new this.configReplace(index, this.configTree, { - configPath: join('environments', `test.json`), - files: ['tests/index.html'], - patterns: this.configReplacePatterns, - annotation: 'ConfigReplace/testIndexTree', - }); - } - - @Memoize() - babelConfig(): TransformOptions { - // this finds all the built-in babel configuration that comes with ember-cli-babel - const babelAddon = (this.app.project as any).findAddonByName('ember-cli-babel'); - const babelConfig = babelAddon.buildBabelOptions({ - 'ember-cli-babel': { - ...this.app.options['ember-cli-babel'], - includeExternalHelpers: true, - compileModules: false, - disableDebugTooling: false, - disablePresetEnv: false, - disableEmberModulesAPIPolyfill: false, - disableDecoratorTransforms: false, - }, - }); - - let plugins = babelConfig.plugins as any[]; - let presets = babelConfig.presets; - - // this finds any custom babel configuration that's on the app (either - // because the app author explicitly added some, or because addons have - // pushed plugins into it). - let appBabel = this.app.options.babel; - if (appBabel) { - if (appBabel.plugins) { - plugins = appBabel.plugins.concat(plugins); - } - if (appBabel.presets) { - presets = appBabel.presets.concat(presets); - } - } - - plugins = plugins.filter(p => { - // even if the app was using @embroider/macros, we drop it from the config - // here in favor of our globally-configured one. - return ( - !isEmbroiderMacrosPlugin(p) && - // similarly, if the app was already using an inline template compiler - // babel plugin, we remove it here because we have our own - // always-installed version of that (v2 addons are allowed to assume it - // will be present in the final app build, the app doesn't get to turn - // that off or configure it.) - !isInlinePrecompilePlugin(p) && - !isEmberAutoImportDynamic(p) - ); - }); - - const config: TransformOptions = { - babelrc: false, - plugins, - presets, - // this is here because broccoli-middleware can't render a codeFrame full - // of terminal codes. It would be nice to add something like - // https://github.com/mmalecki/ansispan to broccoli-middleware so we can - // leave color enabled. - highlightCode: false, - }; - - return config; - } - - @Memoize() - babelMajorVersion(): 7 { - let babelAddon = this.app.project.addons.find((a: any) => a.name === 'ember-cli-babel'); - if (babelAddon) { - let babelAddonMajor = Number(babelAddon.pkg.version.split('.')[0]); - let babelMajor: number | undefined = babelAddonMajor; - if (babelAddonMajor >= 8) { - // `ember-cli-babel` v8 breaks lockstep with Babel, because it now - // defines `@babel/core` as a peer dependency, so we need to check the - // project's version of `@babel/core`: - let babelVersion = this.app.project.pkg.devDependencies?.['@babel/core']; - if (babelVersion) { - babelMajor = semver.coerce(babelVersion)?.major; - } else { - babelMajor = 7; - } - } - if (babelMajor !== 7) { - throw new Error('`@embroider/compat` only supports apps and addons that use Babel v7.'); - } - return babelMajor; - } - // if we didn't have our own babel plugin at all, it's safe to parse our - // code with 7. - return 7; - } - - @Memoize() - private transformedNodeFiles(): Map { - // any app.imports from node_modules that need custom transforms will need - // to get copied into our own synthesized vendor package. app.imports from - // node_modules that *don't* need custom transforms can just stay where they - // are. - let transformed = new Map(); - for (let transformConfig of this.app._customTransformsMap.values()) { - for (let filename of transformConfig.files as string[]) { - let preresolved = this.preresolvedNodeFile(filename); - if (preresolved) { - transformed.set(filename, preresolved); - } - } - } - return transformed; - } - - private preresolvedNodeFile(filename: string) { - // this regex is an exact copy of how ember-cli does this, so we align. - let match = filename.match(/^node_modules\/((@[^/]+\/)?[^/]+)\//); - if (match) { - // ember-cli has already done its own resolution of - // `app.import('node_modules/something/...')`, so we go find its answer. - for (let { name, path } of this.app._nodeModules.values()) { - if (match[1] === name) { - return filename.replace(match[0], path + sep); - } - } - throw new Error(`bug: expected ember-cli to already have a resolved path for asset ${filename}`); - } - } - - private combinedVendor(addonTrees: Node[]): Node { - let trees = addonTrees.map(tree => - buildFunnel(tree, { - allowEmpty: true, - srcDir: 'vendor', - destDir: 'vendor', - }) - ); - if (this.vendorTree) { - trees.push( - buildFunnel(this.vendorTree, { - destDir: 'vendor', - }) - ); - } - - const tree = mergeTrees(trees, { overwrite: true }); - - const outputGroups: Group[] = [ - // scripts - { - outputFiles: this.app._scriptOutputFiles, - implicitKey: '_implicitScripts', - vendorOutputPath: this.app.options.outputPaths.vendor.js, - }, - // styles - { - outputFiles: this.app._styleOutputFiles, - implicitKey: '_implicitStyles', - vendorOutputPath: this.app.options.outputPaths.vendor.css, - }, - ]; - - const concatentations = []; - - // support: app.import / outputFile / using - for (let entry of outputGroups) { - const { outputFiles, implicitKey, vendorOutputPath } = entry; - for (let importPath of Object.keys(outputFiles)) { - const headerFiles = outputFiles[importPath]; - - if (importPath === vendorOutputPath) { - // these are the default ember-cli output files vendor.js or - // vendor.css. Let embroider handle these. - this[implicitKey] = headerFiles; - } else if (headerFiles.length === 0) { - // something went really wrong, open an issue - throw new Error('Embroider: EWUT'); - } else if (headerFiles.length === 1) { - // app.import(x, { outputFile: y }); where only one app.imports had this outputFile - // - // No concat needed. Simply serialize the remapping in the addon's - // manifest, this ensures it is included in the final output with no extra work. - this._publicAssets[headerFiles[0]] = importPath; - } else { - // app.import(x, { outputFile: y }); where multiple app.imports share one outputFile - // Concat needed. Perform concat, and include the outputFile in the - // addon's manifest. This ensures it is included in the final output - this._publicAssets[importPath] = importPath; - - concatentations.push( - new Concat(tree, { - headerFiles, - outputFile: importPath, - annotation: `Package ${importPath}`, - separator: '\n;', - sourceMapConfig: this.app.options['sourcemaps'], - }) - ); - } - } - } - - this.addOtherAssets(); - return mergeTrees([tree, ...concatentations], { overwrite: true }); - } - - addOtherAssets() { - for (let asset of this.app.otherAssetPaths) { - this._publicAssets[`${asset.src}/${asset.file}`] = `${asset.dest}/${asset.file}`; - } - } - - private addNodeAssets(inputTree: Node): Node { - let transformedNodeFiles = this.transformedNodeFiles(); - - return new AddToTree(inputTree, outputPath => { - for (let [localDestPath, sourcePath] of transformedNodeFiles) { - let destPath = join(outputPath, localDestPath); - ensureDirSync(dirname(destPath)); - copySync(sourcePath, destPath); - } - - let remapAsset = this.remapAsset.bind(this); - - let addonMeta: AddonMeta = { - type: 'addon', - version: 2, - 'implicit-scripts': this._implicitScripts.map(remapAsset), - 'implicit-styles': this._implicitStyles.map(remapAsset), - 'implicit-test-scripts': this.app.legacyTestFilesToAppend.map(remapAsset), - 'implicit-test-styles': this.app.vendorTestStaticStyles.map(remapAsset), - 'public-assets': mapKeys(this._publicAssets, (_, key) => remapAsset(key)), - }; - let meta: PackageInfo = { - name: '@embroider/synthesized-vendor', - version: '0.0.0', - keywords: ['ember-addon'], - 'ember-addon': addonMeta, - }; - writeJSONSync(join(outputPath, 'package.json'), meta, { spaces: 2 }); - }); - } - - synthesizeVendorPackage(addonTrees: Node[]): Node { - return this.applyCustomTransforms(this.addNodeAssets(this.combinedVendor(addonTrees))); - } - - private combinedStyles(addonTrees: Node[]): Node { - let trees: Node[] = addonTrees.map(tree => - buildFunnel(tree, { - allowEmpty: true, - srcDir: '_app_styles_', - }) - ); - let appStyles = this.app.trees.styles as Node | undefined; - if (appStyles) { - // Workaround for https://github.com/ember-cli/ember-cli/issues/9020 - // - // The default app styles tree is unwatched and relies on side effects - // elsewhere in ember-cli's build pipeline to actually get rebuilds to - // work. Here we need it to actually be watched properly if we want to - // rely on it, particularly when using BROCCOLI_ENABLED_MEMOIZE. - if ((appStyles as any)._watched === false && (appStyles as any)._directoryPath) { - appStyles = new WatchedDir((appStyles as any)._directoryPath); - } - trees.push(appStyles); - } - return mergeTrees(trees, { overwrite: true, annotation: 'embroider-v1-app-combined-styles' }); - } - - synthesizeStylesPackage(addonTrees: Node[]): Node { - let options = { - // we're deliberately not allowing this to be customized. It's an - // internal implementation detail, and respecting outputPaths here is - // unnecessary complexity. The corresponding code that adjusts the HTML - // is in updateHTML in app.ts. - outputPaths: { app: `/assets/${this.name}.css` }, - registry: this.app.registry, - minifyCSS: this.app.options.minifyCSS.options, - }; - - let nestedInput = buildFunnel(this.combinedStyles(addonTrees), { destDir: 'app/styles' }); - let styles = this.preprocessors.preprocessCss(nestedInput, '/app/styles', '/assets', options); - - return new AddToTree(styles, outputPath => { - let addonMeta: AddonMeta = { - type: 'addon', - version: 2, - 'public-assets': {}, - }; - let assetPath = join(outputPath, 'assets'); - if (pathExistsSync(assetPath)) { - for (let file of readdirSync(assetPath)) { - addonMeta['public-assets']![`./assets/${file}`] = `/assets/${file}`; - } - } - let meta: PackageInfo = { - name: '@embroider/synthesized-styles', - version: '0.0.0', - keywords: ['ember-addon'], - 'ember-addon': addonMeta, - }; - writeJSONSync(join(outputPath, 'package.json'), meta, { spaces: 2 }); - }); - } - - // this is taken nearly verbatim from ember-cli. - private applyCustomTransforms(externalTree: Node) { - for (let customTransformEntry of this.app._customTransformsMap) { - let transformName = customTransformEntry[0]; - let transformConfig = customTransformEntry[1]; - - let transformTree = buildFunnel(externalTree, { - files: transformConfig.files, - annotation: `Funnel (custom transform: ${transformName})`, - }); - - externalTree = mergeTrees([externalTree, transformConfig.callback(transformTree, transformConfig.options)], { - annotation: `TreeMerger (custom transform: ${transformName})`, - overwrite: true, - }); - } - return externalTree; - } - - private remapAsset(asset: string) { - if (this.transformedNodeFiles().has(asset)) { - // transformed node assets become local paths, because we have copied - // those ones into our synthesized vendor package. - return './' + asset; - } - let preresolved = this.preresolvedNodeFile(asset); - if (preresolved) { - // non-transformed node assets point directly at their pre-resolved - // original files (this is an absolute path). - return preresolved; - } - // non node assets are local paths. They need an explicit `/` or `.` at - // the start. - if (asset.startsWith('.') || isAbsolute(asset)) { - return asset; - } - return './' + asset; - } - - private preprocessJS(tree: Node): Node { - // we're saving all our babel compilation for the final stage packager - this.app.registry.remove('js', 'ember-cli-babel'); - - // auto-import is supported natively so we don't need it here - this.app.registry.remove('js', 'ember-auto-import-analyzer'); - - tree = buildFunnel(tree, { destDir: this.name }); - - tree = this.preprocessors.preprocessJs(tree, `/`, '/', { - annotation: 'v1-app-preprocess-js', - registry: this.app.registry, - }); - - tree = buildFunnel(tree, { srcDir: this.name }); - - return tree; - } - - get htmlbarsPlugins(): Transform[] { - let addon = this.app.project.addons.find( - (a: AddonInstance) => a.name === 'ember-cli-htmlbars' - ) as unknown as EmberCliHTMLBarsAddon; - let options = addon.htmlbarsOptions(); - if (options?.plugins?.ast) { - // even if the app was using @embroider/macros, we drop it from the config - // here in favor of our globally-configured one. - options.plugins.ast = options.plugins.ast.filter((p: any) => !isEmbroiderMacrosPlugin(p)); - prepHtmlbarsAstPluginsForUnwrap(this.app.registry); - - // classically, this list was backwards for silly historic reasons. But - // we're the compatibility system, so we're putting it back into - // reasonable order. - options.plugins.ast.reverse(); - - return options.plugins.ast; - } else { - return []; - } - } - - // our own appTree. Not to be confused with the one that combines the app js - // from all addons too. - private get appTree(): Node { - return this.preprocessJS( - buildFunnel(this.app.trees.app, { - exclude: ['styles/**', '*.html'], - }) - ); - } - - private get testsTree(): Node | undefined { - if (this.shouldBuildTests && this.app.trees.tests) { - return this.preprocessJS( - buildFunnel(this.app.trees.tests, { - destDir: 'tests', - }) - ); - } - } - - private get lintTree(): Node | undefined { - if (this.shouldBuildTests) { - return this.app.getLintTests(); - } - } - - get vendorTree(): Node | undefined { - return this.ensureTree(this.app.trees.vendor); - } - - private ensureTree(maybeTree: string | Node | undefined): Node | undefined { - if (typeof maybeTree === 'string') { - // this is deliberately mimicking how ember-cli does it. We don't use - // `this.root` on purpose, because that can differ from what ember-cli - // considers the project.root. And we don't use path.resolve even though - // that seems possibly more correct, because ember-cli always assumes the - // input is relative. - let resolvedPath = join(this.app.project.root, maybeTree); - if (existsSync(resolvedPath)) { - return new WatchedDir(maybeTree); - } else { - return undefined; - } - } - return maybeTree; - } - - @Memoize() - private get preprocessors(): Preprocessors { - return this.requireFromEmberCLI('ember-cli-preprocess-registry/preprocessors'); - } - - get publicTree(): Node | undefined { - return this.ensureTree(this.app.trees.public); - } - - processAppJS(): { appJS: Node } { - let appTree = this.appTree; - let testsTree = this.testsTree; - let lintTree = this.lintTree; - let config = new WriteV1Config(this.config, this.storeConfigInMeta); - let patterns = this.configReplacePatterns; - let configReplaced = new this.configReplace(config, this.configTree, { - configPath: join('environments', `${this.app.env}.json`), - files: ['config/environment.js'], - patterns, - }); - - let trees: Node[] = []; - trees.push(appTree); - trees.push( - new SynthesizeTemplateOnlyComponents(appTree, { allowedPaths: ['components'], templateExtensions: ['.hbs'] }) - ); - - trees.push(configReplaced); - if (testsTree) { - trees.push(testsTree); - } - if (lintTree) { - trees.push(lintTree); - } - return { - appJS: mergeTrees(trees, { overwrite: true }), - }; - } - - private withoutRootURL(src: string) { - let rootURL = this.config.readConfig().rootURL; - if ((src.startsWith(rootURL) && rootURL) || (!rootURL && !src.startsWith('/'))) { - src = '/' + src.slice(rootURL.length); - } else if (src.startsWith('/' + rootURL)) { - src = src.slice(rootURL.length); - } - return src; - } - - findAppScript(scripts: HTMLScriptElement[], entrypoint: string): HTMLScriptElement { - let appJS = scripts.find(script => this.withoutRootURL(script.src) === this.app.options.outputPaths.app.js); - return throwIfMissing( - appJS, - this.app.options.outputPaths.app.js, - scripts.map(s => s.src), - entrypoint, - 'app javascript' - ); - } - - findAppStyles(styles: HTMLLinkElement[], entrypoint: string): HTMLLinkElement { - let style = styles.find(style => this.withoutRootURL(style.href) === this.app.options.outputPaths.app.css.app); - return throwIfMissing( - style, - this.app.options.outputPaths.app.css.app, - styles.map(s => s.href), - entrypoint, - 'app css' - ); - } - - findVendorScript(scripts: HTMLScriptElement[], entrypoint: string): HTMLScriptElement { - let vendor = scripts.find(script => this.withoutRootURL(script.src) === this.app.options.outputPaths.vendor.js); - return throwIfMissing( - vendor, - this.app.options.outputPaths.vendor.js, - scripts.map(s => s.src), - entrypoint, - 'vendor javascript' - ); - } - - findVendorStyles(styles: HTMLLinkElement[], entrypoint: string): HTMLLinkElement { - let vendorStyle = styles.find(style => this.withoutRootURL(style.href) === this.app.options.outputPaths.vendor.css); - return throwIfMissing( - vendorStyle, - this.app.options.outputPaths.vendor.css, - styles.map(s => s.href), - entrypoint, - 'vendor css' - ); - } - - findTestSupportStyles(styles: HTMLLinkElement[]): HTMLLinkElement | undefined { - return styles.find(style => this.withoutRootURL(style.href) === this.app.options.outputPaths.testSupport.css); - } - - findTestSupportScript(scripts: HTMLScriptElement[]): HTMLScriptElement | undefined { - return scripts.find( - script => this.withoutRootURL(script.src) === this.app.options.outputPaths.testSupport.js.testSupport - ); - } - - findTestScript(scripts: HTMLScriptElement[]): HTMLScriptElement | undefined { - return scripts.find(script => this.withoutRootURL(script.src) === this.app.options.outputPaths.tests.js); - } -} - -function throwIfMissing( - asset: T | undefined, - needle: string, - haystack: string[], - entryfile: string, - context: string -): T { - if (!asset) { - throw new Error( - `Could not find ${context}: "${needle}" in ${entryfile}. Found the following instead:\n${haystack - .map(asset => ` - ${asset}`) - .join( - '\n' - )}\n\nFor more information about this error: https://github.com/thoov/stitch/wiki/Could-not-find-asset-in-entry-file-error-help` - ); - } - - return asset; -} - -interface Preprocessors { - preprocessJs(tree: Node, a: string, b: string, options: object): Node; - preprocessCss(tree: Node, a: string, b: string, options: object): Node; -} diff --git a/packages/compat/src/v1-instance-cache.ts b/packages/compat/src/v1-instance-cache.ts index fe2c363a2..ab0b05e1d 100644 --- a/packages/compat/src/v1-instance-cache.ts +++ b/packages/compat/src/v1-instance-cache.ts @@ -2,12 +2,12 @@ // packages is supposed to go through here. This lets us control the boundary // between the new and old words. -import V1App from './v1-app'; import V1Addon, { V1AddonConstructor } from './v1-addon'; import { pathExistsSync } from 'fs-extra'; import { AddonInstance, getOrCreate } from '@embroider/core'; import Options from './options'; import isEqual from 'lodash/isEqual'; +import CompatApp from './compat-app'; export default class V1InstanceCache { static caches: WeakMap = new WeakMap(); @@ -25,11 +25,11 @@ export default class V1InstanceCache { // other packages and each gets an instance. private addons: Map = new Map(); - app: V1App; + app: CompatApp; orderIdx: number; private constructor(oldApp: any, private options: Required) { - this.app = new V1App(oldApp); + this.app = new CompatApp(oldApp, options); this.orderIdx = 0; // no reason to do this on demand because oldApp already eagerly loaded @@ -75,7 +75,7 @@ export default class V1InstanceCache { this.orderIdx += 1; let Klass = this.adapterClass(addonInstance); - let v1Addon = new Klass(addonInstance, this.options, this.app, this.app.packageCache, this.orderIdx); + let v1Addon = new Klass(addonInstance, this.options, this.app, this.app.movablePackageCache, this.orderIdx); let pkgs = getOrCreate(this.addons, v1Addon.root, () => []); pkgs.push(v1Addon); } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a5da85ee2..43e0bef15 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -159,6 +159,9 @@ importers: '@babel/preset-env': specifier: ^7.14.5 version: 7.16.11(@babel/core@7.19.6) + '@babel/runtime': + specifier: ^7.18.6 + version: 7.21.5 '@babel/traverse': specifier: ^7.14.5 version: 7.14.5 @@ -1659,7 +1662,7 @@ importers: version: /ember-data@4.4.0(@babel/core@7.19.6)(webpack@5.78.0) ember-data-latest: specifier: npm:ember-data@latest - version: /ember-data@4.11.3(@babel/core@7.19.6)(@ember/string@3.0.1)(@glimmer/tracking@1.1.2)(ember-source@5.1.0-beta.1)(webpack@5.78.0) + version: /ember-data@4.12.0(@babel/core@7.19.6)(@ember/string@3.0.1)(@glimmer/tracking@1.1.2)(ember-source@5.1.0-beta.1)(webpack@5.78.0) ember-engines: specifier: ^0.8.23 version: 0.8.23(@ember/legacy-built-in-components@0.4.1)(ember-source@5.1.0-beta.1) @@ -3353,13 +3356,13 @@ packages: engines: {node: '>=6.9.0'} dependencies: regenerator-runtime: 0.13.11 + dev: true /@babel/runtime@7.21.5: resolution: {integrity: sha512-8jI69toZqqcsnqGGqwGS4Qb1VwLOEp4hz+CXPywcvjs60u3B4Pom/U/7rm4W8tMOYEB+E9wgD0mW1l3r8qlI9Q==} engines: {node: '>=6.9.0'} dependencies: regenerator-runtime: 0.13.11 - dev: true /@babel/template@7.20.7: resolution: {integrity: sha512-8SegXApWe6VoNw0r9JHpSteLKTpTiLZ4rMlGIm9JQ18KiCtyQiAMEazujAHrUS5flrcqYZa75ukev3P6QmUwUw==} @@ -3504,26 +3507,23 @@ packages: - supports-color dev: true - /@ember-data/adapter@4.11.3(@ember-data/store@4.11.3)(@ember/string@3.0.1)(ember-inflector@4.0.2)(webpack@5.78.0): - resolution: {integrity: sha512-G7dbaPnYMW8VYxIT75KAkzax2mkWTs2TYxS7+qbphs6esXpO9Y/iNp5fTqLaACb9JqUypwEA/rlfC7/zkcGbBw==} - engines: {node: ^14.8.0 || 16.* || >= 18.*} + /@ember-data/adapter@4.12.0(@ember-data/store@4.12.0)(@ember/string@3.0.1)(ember-inflector@4.0.2): + resolution: {integrity: sha512-sY7Zm73LSN1x1jO+lTV0+Vtdis6rBFAuRD3sln1BOW0y9che5WK+qyQs8FhjC6m9D/FFIKqUucWvaPO4/GazuQ==} + engines: {node: 16.* || >= 18.*} peerDependencies: - '@ember-data/store': 4.11.3 + '@ember-data/store': 4.12.0 '@ember/string': ^3.0.1 ember-inflector: ^4.0.2 dependencies: - '@ember-data/private-build-infra': 4.11.3 - '@ember-data/store': 4.11.3(@babel/core@7.19.6)(@ember-data/model@4.11.3)(@ember-data/record-data@4.11.3)(@ember-data/tracking@4.11.3)(@ember/string@3.0.1)(@glimmer/tracking@1.1.2)(ember-source@5.1.0-beta.1)(webpack@5.78.0) - '@ember/edition-utils': 1.2.0 + '@ember-data/private-build-infra': 4.12.0 + '@ember-data/store': 4.12.0(@babel/core@7.19.6)(@ember-data/graph@4.12.0)(@ember-data/json-api@4.12.0)(@ember-data/legacy-compat@4.12.0)(@ember-data/model@4.12.0)(@ember-data/tracking@4.12.0)(@ember/string@3.0.1)(@glimmer/tracking@1.1.2)(ember-source@5.1.0-beta.1) '@ember/string': 3.0.1 '@embroider/macros': 1.10.0 - ember-auto-import: 2.6.3(webpack@5.78.0) ember-cli-babel: 7.26.11 ember-cli-test-info: 1.0.0 ember-inflector: 4.0.2 transitivePeerDependencies: - supports-color - - webpack dev: true /@ember-data/adapter@4.4.0(@babel/core@7.19.6)(webpack@5.78.0): @@ -3554,16 +3554,6 @@ packages: - supports-color dev: true - /@ember-data/canary-features@4.11.3: - resolution: {integrity: sha512-RTLY2N9t1SXr4e90VBKi+3PIitwjTMBU8BcEhnKovT//sGlywohHq7T36H6nJuITRtki3On9PpbJOhhQZuyAlQ==} - engines: {node: ^14.8.0 || 16.* || >= 18.*} - dependencies: - '@embroider/macros': 1.10.0 - ember-cli-babel: 7.26.11 - transitivePeerDependencies: - - supports-color - dev: true - /@ember-data/canary-features@4.4.0: resolution: {integrity: sha512-PkizCdM5SXCWiAKwq9ybIfhsLtM596IXtBz0cRH+d4YV3sBIsQjnAf+IYsJpwxf+XKvzXnzPLeIcFd7+NL6YPw==} engines: {node: 12.* || >= 14.*} @@ -3589,13 +3579,13 @@ packages: - supports-color dev: true - /@ember-data/debug@4.11.3(@ember/string@3.0.1)(webpack@5.78.0): - resolution: {integrity: sha512-3pA5u3qy+pjtwcoyMzs7WijRrSQz5z+Vgn9b5Y4cEOHn8loS9riLCMScnFaQT3HjxQgq+3NkNb52sJafHPzs4Q==} - engines: {node: ^14.8.0 || 16.* || >= 18.*} + /@ember-data/debug@4.12.0(@ember/string@3.0.1)(webpack@5.78.0): + resolution: {integrity: sha512-6SNJjoV3zKnjjZEu9/tOjeWdN70mxmkvHd+0Y7kjasmjLBgIkZk20+B/nFm25MpmmpfZEsvdUY3HIfu+iPy+5A==} + engines: {node: 16.* || >= 18.*} peerDependencies: '@ember/string': ^3.0.1 dependencies: - '@ember-data/private-build-infra': 4.11.3 + '@ember-data/private-build-infra': 4.12.0 '@ember/edition-utils': 1.2.0 '@ember/string': 3.0.1 '@embroider/macros': 1.10.0 @@ -3623,6 +3613,59 @@ packages: - webpack dev: true + /@ember-data/graph@4.12.0(@ember-data/store@4.12.0): + resolution: {integrity: sha512-5crSekONC8cm/sPS4OnNNG1TrnCb4rqrM72Ux8i8xlomYpLq75R2gY4ibY1HRNstrEoAB09rzONTB0bRJHlTQw==} + engines: {node: 16.* || >= 18.*} + peerDependencies: + '@ember-data/store': 4.12.0 + dependencies: + '@ember-data/private-build-infra': 4.12.0 + '@ember-data/store': 4.12.0(@babel/core@7.19.6)(@ember-data/graph@4.12.0)(@ember-data/json-api@4.12.0)(@ember-data/legacy-compat@4.12.0)(@ember-data/model@4.12.0)(@ember-data/tracking@4.12.0)(@ember/string@3.0.1)(@glimmer/tracking@1.1.2)(ember-source@5.1.0-beta.1) + '@ember/edition-utils': 1.2.0 + '@embroider/macros': 1.10.0 + ember-cli-babel: 7.26.11 + transitivePeerDependencies: + - supports-color + dev: true + + /@ember-data/json-api@4.12.0(@ember-data/graph@4.12.0)(@ember-data/store@4.12.0): + resolution: {integrity: sha512-vtxuB7akuSfsEBvLX/8h4zGyIozynyq5Bf9I02ftIoIIwD21wN+g/ZG91KU6sNZzyeycTZEKpoYaITM84pLTTg==} + engines: {node: 16.* || >= 18.*} + peerDependencies: + '@ember-data/graph': 4.12.0 + '@ember-data/store': 4.12.0 + dependencies: + '@ember-data/graph': 4.12.0(@ember-data/store@4.12.0) + '@ember-data/private-build-infra': 4.12.0 + '@ember-data/store': 4.12.0(@babel/core@7.19.6)(@ember-data/graph@4.12.0)(@ember-data/json-api@4.12.0)(@ember-data/legacy-compat@4.12.0)(@ember-data/model@4.12.0)(@ember-data/tracking@4.12.0)(@ember/string@3.0.1)(@glimmer/tracking@1.1.2)(ember-source@5.1.0-beta.1) + '@ember/edition-utils': 1.2.0 + '@embroider/macros': 1.10.0 + ember-cli-babel: 7.26.11 + transitivePeerDependencies: + - supports-color + dev: true + + /@ember-data/legacy-compat@4.12.0(@ember-data/graph@4.12.0)(@ember-data/json-api@4.12.0): + resolution: {integrity: sha512-QVZczGMbTk8Ch+xiZt7KQk5UX2AdUsVdR3rSB/pJVZrWcUWo6ToAR2mPl97/cWd6VYFXBZgMamsxkeBO4q5HXA==} + engines: {node: 16.* || >= 18} + peerDependencies: + '@ember-data/graph': 4.12.0 + '@ember-data/json-api': 4.12.0 + peerDependenciesMeta: + '@ember-data/graph': + optional: true + '@ember-data/json-api': + optional: true + dependencies: + '@ember-data/graph': 4.12.0(@ember-data/store@4.12.0) + '@ember-data/json-api': 4.12.0(@ember-data/graph@4.12.0)(@ember-data/store@4.12.0) + '@ember-data/private-build-infra': 4.12.0 + '@embroider/macros': 1.10.0 + ember-cli-babel: 7.26.11 + transitivePeerDependencies: + - supports-color + dev: true + /@ember-data/model@3.28.0(@babel/core@7.19.6): resolution: {integrity: sha512-G4lK0eRVZ8IU6Jh6+Wk060qLgj5ziPAKXXIqDsiDHaoBSd0uKj5tU+lUxHBxVM2fU3+yRr1WK5HIBu9PiSPWmw==} engines: {node: 12.* || >= 14.*} @@ -3644,40 +3687,46 @@ packages: - supports-color dev: true - /@ember-data/model@4.11.3(@babel/core@7.19.6)(@ember-data/record-data@4.11.3)(@ember-data/store@4.11.3)(@ember-data/tracking@4.11.3)(@ember/string@3.0.1)(ember-inflector@4.0.2)(ember-source@5.1.0-beta.1)(webpack@5.78.0): - resolution: {integrity: sha512-nkDru5TZmOp4J1xp65D1bR3hBJ3u5KhKKfDpWeGnHW2YDCVUdLORRwW7vfrPnnXDIoJij42DwDVCiTY25Xhrqw==} - engines: {node: ^14.8.0 || 16.* || >= 18.*} + /@ember-data/model@4.12.0(@babel/core@7.19.6)(@ember-data/debug@4.12.0)(@ember-data/graph@4.12.0)(@ember-data/json-api@4.12.0)(@ember-data/legacy-compat@4.12.0)(@ember-data/store@4.12.0)(@ember-data/tracking@4.12.0)(@ember/string@3.0.1)(ember-inflector@4.0.2)(ember-source@5.1.0-beta.1): + resolution: {integrity: sha512-gE9LRmUkrJy9hJ+WeNns/GOMQC311R18SOvbsIVk5z/u2tgD5l0BjLSeqCaG/CjO+fCRsM8Ne/Ivm07c/CyezQ==} + engines: {node: 16.* || >= 18.*} peerDependencies: - '@ember-data/record-data': 4.11.3 - '@ember-data/store': 4.11.3 - '@ember-data/tracking': 4.11.3 + '@ember-data/debug': 4.12.0 + '@ember-data/graph': 4.12.0 + '@ember-data/json-api': 4.12.0 + '@ember-data/legacy-compat': 4.12.0 + '@ember-data/store': 4.12.0 + '@ember-data/tracking': 4.12.0 '@ember/string': ^3.0.1 ember-inflector: ^4.0.2 peerDependenciesMeta: - '@ember-data/record-data': + '@ember-data/debug': + optional: true + '@ember-data/graph': + optional: true + '@ember-data/json-api': optional: true dependencies: - '@ember-data/canary-features': 4.11.3 - '@ember-data/private-build-infra': 4.11.3 - '@ember-data/record-data': 4.11.3(@ember-data/store@4.11.3)(webpack@5.78.0) - '@ember-data/store': 4.11.3(@babel/core@7.19.6)(@ember-data/model@4.11.3)(@ember-data/record-data@4.11.3)(@ember-data/tracking@4.11.3)(@ember/string@3.0.1)(@glimmer/tracking@1.1.2)(ember-source@5.1.0-beta.1)(webpack@5.78.0) - '@ember-data/tracking': 4.11.3 + '@ember-data/debug': 4.12.0(@ember/string@3.0.1)(webpack@5.78.0) + '@ember-data/graph': 4.12.0(@ember-data/store@4.12.0) + '@ember-data/json-api': 4.12.0(@ember-data/graph@4.12.0)(@ember-data/store@4.12.0) + '@ember-data/legacy-compat': 4.12.0(@ember-data/graph@4.12.0)(@ember-data/json-api@4.12.0) + '@ember-data/private-build-infra': 4.12.0 + '@ember-data/store': 4.12.0(@babel/core@7.19.6)(@ember-data/graph@4.12.0)(@ember-data/json-api@4.12.0)(@ember-data/legacy-compat@4.12.0)(@ember-data/model@4.12.0)(@ember-data/tracking@4.12.0)(@ember/string@3.0.1)(@glimmer/tracking@1.1.2)(ember-source@5.1.0-beta.1) + '@ember-data/tracking': 4.12.0 '@ember/edition-utils': 1.2.0 '@ember/string': 3.0.1 '@embroider/macros': 1.10.0 - ember-auto-import: 2.6.3(webpack@5.78.0) ember-cached-decorator-polyfill: 1.0.1(@babel/core@7.19.6)(ember-source@5.1.0-beta.1) ember-cli-babel: 7.26.11 ember-cli-string-utils: 1.1.0 ember-cli-test-info: 1.0.0 - ember-compatibility-helpers: 1.2.6(@babel/core@7.19.6) ember-inflector: 4.0.2 inflection: 2.0.1 transitivePeerDependencies: - '@babel/core' - ember-source - supports-color - - webpack dev: true /@ember-data/model@4.4.0(@babel/core@7.19.6)(webpack@5.78.0): @@ -3738,14 +3787,13 @@ packages: - supports-color dev: true - /@ember-data/private-build-infra@4.11.3: - resolution: {integrity: sha512-bXFQMEegUc+vKn/vD7FmAkq7ECE0okZ2sbtv/0RXqYn7TLk44rvGzpqSUXUowpCaGI/87MmaW8JaZMMdqF9wuw==} - engines: {node: ^14.8.0 || 16.* || >= 18.*} + /@ember-data/private-build-infra@4.12.0: + resolution: {integrity: sha512-cBuEZhxV8uyIRr+9oUZ4smQb+6p6ryH89+WdrGMTeKgKP3XkdlK9w+6veQAYOqgWAulTwmAxX+YU/zoPq2ne7w==} + engines: {node: 16.* || >= 18.*} dependencies: '@babel/core': 7.21.8 '@babel/plugin-transform-block-scoping': 7.21.0(@babel/core@7.21.8) '@babel/runtime': 7.21.5 - '@ember-data/canary-features': 4.11.3 '@ember/edition-utils': 1.2.0 '@embroider/macros': 1.10.0 babel-import-util: 1.3.0 @@ -3764,10 +3812,8 @@ packages: ember-cli-string-utils: 1.1.0 ember-cli-version-checker: 5.1.2 git-repo-info: 2.1.1 - glob: 8.1.0 + glob: 9.3.5 npm-git-info: 1.0.3 - rimraf: 3.0.2 - rsvp: 4.8.5 semver: 7.3.8 silent-error: 1.1.1 transitivePeerDependencies: @@ -3825,24 +3871,6 @@ packages: - supports-color dev: true - /@ember-data/record-data@4.11.3(@ember-data/store@4.11.3)(webpack@5.78.0): - resolution: {integrity: sha512-8NmeEZJ7or354NLZJgibJ1FuhWL70H6G24tGSEIzM8IV7wr6TreIyaWODaW372QwamWYgFIpfnFwWt5MTlY/gw==} - engines: {node: ^14.8.0 || 16.* || >= 18.*} - peerDependencies: - '@ember-data/store': 4.11.3 - dependencies: - '@ember-data/canary-features': 4.11.3 - '@ember-data/private-build-infra': 4.11.3 - '@ember-data/store': 4.11.3(@babel/core@7.19.6)(@ember-data/model@4.11.3)(@ember-data/record-data@4.11.3)(@ember-data/tracking@4.11.3)(@ember/string@3.0.1)(@glimmer/tracking@1.1.2)(ember-source@5.1.0-beta.1)(webpack@5.78.0) - '@ember/edition-utils': 1.2.0 - '@embroider/macros': 1.10.0 - ember-auto-import: 2.6.3(webpack@5.78.0) - ember-cli-babel: 7.26.11 - transitivePeerDependencies: - - supports-color - - webpack - dev: true - /@ember-data/record-data@4.4.0(@babel/core@7.19.6)(webpack@5.78.0): resolution: {integrity: sha512-QBP8EMih2tvXMHUv8j/jWzOkW/d/OGfO22maOi9mh/MoCaX+EphupgdCFJL2SjQZYFPVioPosLQmQX4l5ChkDw==} engines: {node: 12.* || >= 14.*} @@ -3861,6 +3889,18 @@ packages: - webpack dev: true + /@ember-data/request@4.12.0: + resolution: {integrity: sha512-n08NaFwJPq8TUj0F5M5Y88hZ8OhuzaeHjygnaumZtAnCbM9vRrJvrGCcTkfPp2XL3jfKOzeTHNzWzX8XY+efzQ==} + engines: {node: 16.* || >= 18} + dependencies: + '@ember-data/private-build-infra': 4.12.0 + '@ember/test-waiters': 3.0.2 + '@embroider/macros': 1.10.0 + ember-cli-babel: 7.26.11 + transitivePeerDependencies: + - supports-color + dev: true + /@ember-data/rfc395-data@0.0.4: resolution: {integrity: sha512-tGRdvgC9/QMQSuSuJV45xoyhI0Pzjm7A9o/MVVA3HakXIImJbbzx/k/6dO9CUEQXIyS2y0fW6C1XaYOG7rY0FQ==} @@ -3878,25 +3918,23 @@ packages: - supports-color dev: true - /@ember-data/serializer@4.11.3(@ember-data/store@4.11.3)(@ember/string@3.0.1)(ember-inflector@4.0.2)(webpack@5.78.0): - resolution: {integrity: sha512-Qnzrowinz14/onQfwd4TPwNG0sMTAwTWE0RajYo2fysF3CKyAua0nIzmFtXKx0CogD7TYd0C5xf6nMjFesT09Q==} - engines: {node: ^14.8.0 || 16.* || >= 18.*} + /@ember-data/serializer@4.12.0(@ember-data/store@4.12.0)(@ember/string@3.0.1)(ember-inflector@4.0.2): + resolution: {integrity: sha512-q6TJKrS95eFKm9fNm9UkwTQBJw5G+oj37lBPtsnLs6Sm05RCR8fvUX+WbkKi6CoqfKrn2zlZU8Z8mKg7DXc5nA==} + engines: {node: 16.* || >= 18.*} peerDependencies: - '@ember-data/store': 4.11.3 + '@ember-data/store': 4.12.0 '@ember/string': ^3.0.1 ember-inflector: ^4.0.2 dependencies: - '@ember-data/private-build-infra': 4.11.3 - '@ember-data/store': 4.11.3(@babel/core@7.19.6)(@ember-data/model@4.11.3)(@ember-data/record-data@4.11.3)(@ember-data/tracking@4.11.3)(@ember/string@3.0.1)(@glimmer/tracking@1.1.2)(ember-source@5.1.0-beta.1)(webpack@5.78.0) + '@ember-data/private-build-infra': 4.12.0 + '@ember-data/store': 4.12.0(@babel/core@7.19.6)(@ember-data/graph@4.12.0)(@ember-data/json-api@4.12.0)(@ember-data/legacy-compat@4.12.0)(@ember-data/model@4.12.0)(@ember-data/tracking@4.12.0)(@ember/string@3.0.1)(@glimmer/tracking@1.1.2)(ember-source@5.1.0-beta.1) '@ember/string': 3.0.1 '@embroider/macros': 1.10.0 - ember-auto-import: 2.6.3(webpack@5.78.0) ember-cli-babel: 7.26.11 ember-cli-test-info: 1.0.0 ember-inflector: 4.0.2 transitivePeerDependencies: - supports-color - - webpack dev: true /@ember-data/serializer@4.4.0(@babel/core@7.19.6)(webpack@5.78.0): @@ -3931,37 +3969,42 @@ packages: - supports-color dev: true - /@ember-data/store@4.11.3(@babel/core@7.19.6)(@ember-data/model@4.11.3)(@ember-data/record-data@4.11.3)(@ember-data/tracking@4.11.3)(@ember/string@3.0.1)(@glimmer/tracking@1.1.2)(ember-source@5.1.0-beta.1)(webpack@5.78.0): - resolution: {integrity: sha512-ogwWy+VqMpkCGs4n30pzuB2vqv/dJRL6wdV3fdNKpXrDugffjuMPpLBQYF937qztDUZKxmnbWAZe5PbQOz8b1Q==} - engines: {node: ^14.8.0 || 16.* || >= 18.*} + /@ember-data/store@4.12.0(@babel/core@7.19.6)(@ember-data/graph@4.12.0)(@ember-data/json-api@4.12.0)(@ember-data/legacy-compat@4.12.0)(@ember-data/model@4.12.0)(@ember-data/tracking@4.12.0)(@ember/string@3.0.1)(@glimmer/tracking@1.1.2)(ember-source@5.1.0-beta.1): + resolution: {integrity: sha512-7zOxg363f8raqmJcQYiH6JAWWyBDLRQTWLZeyeJD3kgFV+MqWlHLjEvOFCDW2SnfIrVAyFH7oh7x7POxClw9mA==} + engines: {node: 16.* || >= 18.*} peerDependencies: - '@ember-data/model': 4.11.3 - '@ember-data/record-data': 4.11.3 - '@ember-data/tracking': 4.11.3 + '@ember-data/graph': 4.12.0 + '@ember-data/json-api': 4.12.0 + '@ember-data/legacy-compat': 4.12.0 + '@ember-data/model': 4.12.0 + '@ember-data/tracking': 4.12.0 '@ember/string': ^3.0.1 '@glimmer/tracking': ^1.1.2 peerDependenciesMeta: - '@ember-data/model': + '@ember-data/graph': + optional: true + '@ember-data/json-api': optional: true - '@ember-data/record-data': + '@ember-data/legacy-compat': + optional: true + '@ember-data/model': optional: true dependencies: - '@ember-data/canary-features': 4.11.3 - '@ember-data/model': 4.11.3(@babel/core@7.19.6)(@ember-data/record-data@4.11.3)(@ember-data/store@4.11.3)(@ember-data/tracking@4.11.3)(@ember/string@3.0.1)(ember-inflector@4.0.2)(ember-source@5.1.0-beta.1)(webpack@5.78.0) - '@ember-data/private-build-infra': 4.11.3 - '@ember-data/record-data': 4.11.3(@ember-data/store@4.11.3)(webpack@5.78.0) - '@ember-data/tracking': 4.11.3 + '@ember-data/graph': 4.12.0(@ember-data/store@4.12.0) + '@ember-data/json-api': 4.12.0(@ember-data/graph@4.12.0)(@ember-data/store@4.12.0) + '@ember-data/legacy-compat': 4.12.0(@ember-data/graph@4.12.0)(@ember-data/json-api@4.12.0) + '@ember-data/model': 4.12.0(@babel/core@7.19.6)(@ember-data/debug@4.12.0)(@ember-data/graph@4.12.0)(@ember-data/json-api@4.12.0)(@ember-data/legacy-compat@4.12.0)(@ember-data/store@4.12.0)(@ember-data/tracking@4.12.0)(@ember/string@3.0.1)(ember-inflector@4.0.2)(ember-source@5.1.0-beta.1) + '@ember-data/private-build-infra': 4.12.0 + '@ember-data/tracking': 4.12.0 '@ember/string': 3.0.1 '@embroider/macros': 1.10.0 '@glimmer/tracking': 1.1.2 - ember-auto-import: 2.6.3(webpack@5.78.0) ember-cached-decorator-polyfill: 1.0.1(@babel/core@7.19.6)(ember-source@5.1.0-beta.1) ember-cli-babel: 7.26.11 transitivePeerDependencies: - '@babel/core' - ember-source - supports-color - - webpack dev: true /@ember-data/store@4.4.0(@babel/core@7.19.6)(webpack@5.78.0): @@ -3983,9 +4026,9 @@ packages: - webpack dev: true - /@ember-data/tracking@4.11.3: - resolution: {integrity: sha512-YZxFTMe2TBL8H8/GrnrvP7Wc/uuAijoSyiP2g6TMNRsL1e/3BWDT0EIl+B/5Wji+dchofY8iuMWfpY7VDvPIzA==} - engines: {node: 14.* || 16.* || >= 18} + /@ember-data/tracking@4.12.0: + resolution: {integrity: sha512-Jgg6ayR70HLdMqIuXgh/5bdD93Qxop4evSA/f0ltDyilTQ63Olw6GkaYBpjOf6rZbRxdAOwLOOITyoE04zVq+g==} + engines: {node: 16.* || >= 18} dependencies: ember-cli-babel: 7.26.11 transitivePeerDependencies: @@ -7394,7 +7437,7 @@ packages: wordwrap: 0.0.3 /bower-endpoint-parser@0.2.2: - resolution: {integrity: sha512-YWZHhWkPdXtIfH3VRu3QIV95sa75O9vrQWBOHjexWCLBCTy5qJvRr36LXTqFwTchSXVlzy5piYJOjzHr7qhsNg==} + resolution: {integrity: sha1-ALVlrb+rby01rd3pd+l5Yqy8s/Y=} engines: {node: '>=0.8.0'} /brace-expansion@1.1.11: @@ -11051,20 +11094,23 @@ packages: - supports-color dev: true - /ember-data@4.11.3(@babel/core@7.19.6)(@ember/string@3.0.1)(@glimmer/tracking@1.1.2)(ember-source@5.1.0-beta.1)(webpack@5.78.0): - resolution: {integrity: sha512-7vir6Re3M3M6yJoCHy6UxEg3oSY1JEnsuTByY3lJquWPaUamn7qbPQvNr16Tqh8EKrt+e/+X26czFm4kRGhpVg==} - engines: {node: ^14.8.0 || 16.* || >= 18.*} + /ember-data@4.12.0(@babel/core@7.19.6)(@ember/string@3.0.1)(@glimmer/tracking@1.1.2)(ember-source@5.1.0-beta.1)(webpack@5.78.0): + resolution: {integrity: sha512-E1A94HOurihoaFzJmArhtXfp56WsLlbTyhnqWfZKgqWZz1qKF4GVbDuOsGIsy6u345LdUCp2jtodRO2s43k88Q==} + engines: {node: 16.* || >= 18.*} peerDependencies: '@ember/string': ^3.0.1 dependencies: - '@ember-data/adapter': 4.11.3(@ember-data/store@4.11.3)(@ember/string@3.0.1)(ember-inflector@4.0.2)(webpack@5.78.0) - '@ember-data/debug': 4.11.3(@ember/string@3.0.1)(webpack@5.78.0) - '@ember-data/model': 4.11.3(@babel/core@7.19.6)(@ember-data/record-data@4.11.3)(@ember-data/store@4.11.3)(@ember-data/tracking@4.11.3)(@ember/string@3.0.1)(ember-inflector@4.0.2)(ember-source@5.1.0-beta.1)(webpack@5.78.0) - '@ember-data/private-build-infra': 4.11.3 - '@ember-data/record-data': 4.11.3(@ember-data/store@4.11.3)(webpack@5.78.0) - '@ember-data/serializer': 4.11.3(@ember-data/store@4.11.3)(@ember/string@3.0.1)(ember-inflector@4.0.2)(webpack@5.78.0) - '@ember-data/store': 4.11.3(@babel/core@7.19.6)(@ember-data/model@4.11.3)(@ember-data/record-data@4.11.3)(@ember-data/tracking@4.11.3)(@ember/string@3.0.1)(@glimmer/tracking@1.1.2)(ember-source@5.1.0-beta.1)(webpack@5.78.0) - '@ember-data/tracking': 4.11.3 + '@ember-data/adapter': 4.12.0(@ember-data/store@4.12.0)(@ember/string@3.0.1)(ember-inflector@4.0.2) + '@ember-data/debug': 4.12.0(@ember/string@3.0.1)(webpack@5.78.0) + '@ember-data/graph': 4.12.0(@ember-data/store@4.12.0) + '@ember-data/json-api': 4.12.0(@ember-data/graph@4.12.0)(@ember-data/store@4.12.0) + '@ember-data/legacy-compat': 4.12.0(@ember-data/graph@4.12.0)(@ember-data/json-api@4.12.0) + '@ember-data/model': 4.12.0(@babel/core@7.19.6)(@ember-data/debug@4.12.0)(@ember-data/graph@4.12.0)(@ember-data/json-api@4.12.0)(@ember-data/legacy-compat@4.12.0)(@ember-data/store@4.12.0)(@ember-data/tracking@4.12.0)(@ember/string@3.0.1)(ember-inflector@4.0.2)(ember-source@5.1.0-beta.1) + '@ember-data/private-build-infra': 4.12.0 + '@ember-data/request': 4.12.0 + '@ember-data/serializer': 4.12.0(@ember-data/store@4.12.0)(@ember/string@3.0.1)(ember-inflector@4.0.2) + '@ember-data/store': 4.12.0(@babel/core@7.19.6)(@ember-data/graph@4.12.0)(@ember-data/json-api@4.12.0)(@ember-data/legacy-compat@4.12.0)(@ember-data/model@4.12.0)(@ember-data/tracking@4.12.0)(@ember/string@3.0.1)(@glimmer/tracking@1.1.2)(ember-source@5.1.0-beta.1) + '@ember-data/tracking': 4.12.0 '@ember/edition-utils': 1.2.0 '@ember/string': 3.0.1 '@embroider/macros': 1.10.0 @@ -13886,6 +13932,16 @@ packages: once: 1.4.0 dev: true + /glob@9.3.5: + resolution: {integrity: sha512-e1LleDykUz2Iu+MTYdkSsuWX8lvAjAcs0Xef0lNIu0S2wOAzuTxCJtcd9S3cijlwYF18EsU3rzb8jPVobxDh9Q==} + engines: {node: '>=16 || 14 >=14.17'} + dependencies: + fs.realpath: 1.0.0 + minimatch: 8.0.4 + minipass: 4.2.8 + path-scurry: 1.9.2 + dev: true + /global-modules@1.0.0: resolution: {integrity: sha512-sKzpEkf11GpOFuw0Zzjzmt4B4UZwjOcG757PPvrfhxcLFbq0wpsgpOqxpxtxFiCG4DtG93M6XRVbF2oGdev7bg==} engines: {node: '>=0.10.0'} @@ -15777,7 +15833,7 @@ packages: dev: true /leek@0.0.24: - resolution: {integrity: sha512-6PVFIYXxlYF0o6hrAsHtGpTmi06otkwNrMcmQ0K96SeSRHPREPa9J3nJZ1frliVH7XT0XFswoJFQoXsDukzGNQ==} + resolution: {integrity: sha1-5ADlfw5g2O8r1NBo3EKKVDRdvNo=} dependencies: debug: 2.6.9(supports-color@8.1.0) lodash.assign: 3.2.0 @@ -16093,6 +16149,11 @@ packages: engines: {node: '>=12'} dev: true + /lru-cache@9.1.2: + resolution: {integrity: sha512-ERJq3FOzJTxBbFjZ7iDs+NiK4VI9Wz+RdrrAB8dio1oV+YvdPzUEE4QNiT2VD51DkIbCYRUUzCRkssXCHqSnKQ==} + engines: {node: 14 || >=16.14} + dev: true + /magic-string@0.24.1: resolution: {integrity: sha512-YBfNxbJiixMzxW40XqJEIldzHyh5f7CZKalo1uZffevyrPEX8Qgo9s0dmcORLHdV47UyvJg8/zD+6hQG3qvJrA==} dependencies: @@ -16461,6 +16522,13 @@ packages: brace-expansion: 2.0.1 dev: true + /minimatch@8.0.4: + resolution: {integrity: sha512-W0Wvr9HyFXZRGIDgCicunpQ299OKXs9RgZfaukz4qAW/pJhcpUfupc9c+OObPOFueNy8VSrZgEmDtk6Kh4WzDA==} + engines: {node: '>=16 || 14 >=14.17'} + dependencies: + brace-expansion: 2.0.1 + dev: true + /minimist-options@4.1.0: resolution: {integrity: sha512-Q4r8ghd80yhO/0j1O3B2BjweX3fiHg9cdOwjJd2J76Q135c+NDxGCqdYKQ1SKBuFfgWbAUzBfvYjPUEeNgqN1A==} engines: {node: '>= 6'} @@ -16482,6 +16550,16 @@ packages: safe-buffer: 5.2.1 yallist: 3.1.1 + /minipass@4.2.8: + resolution: {integrity: sha512-fNzuVyifolSLFL4NzpF+wEF4qrgqaaKX0haXPQEdQ7NKAN+WecoKMHV09YcuL/DHxrUsYQOK3MiuDf7Ip2OXfQ==} + engines: {node: '>=8'} + dev: true + + /minipass@6.0.2: + resolution: {integrity: sha512-MzWSV5nYVT7mVyWCwn2o7JH13w2TBRmmSqSRCKzTw+lmft9X4z+3wjvs06Tzijo5z4W/kahUCDpRXTF+ZrmF/w==} + engines: {node: '>=16 || 14 >=14.17'} + dev: true + /mississippi@3.0.0: resolution: {integrity: sha512-x471SsVjUtBRtcvd4BzKE9kFC+/2TeWgKCgw0bZcw1b9l2X3QX5vCWgF+KaZaYm87Ss//rHnWryupDrgLvmSkA==} engines: {node: '>=4.0.0'} @@ -17295,8 +17373,16 @@ packages: dependencies: path-root-regex: 0.1.2 + /path-scurry@1.9.2: + resolution: {integrity: sha512-qSDLy2aGFPm8i4rsbHd4MNyTcrzHFsLQykrtbuGRknZZCBBVXSv2tSCDN2Cg6Rt/GFRw8GoW9y9Ecw5rIPG1sg==} + engines: {node: '>=16 || 14 >=14.17'} + dependencies: + lru-cache: 9.1.2 + minipass: 6.0.2 + dev: true + /path-to-regexp@0.1.7: - resolution: {integrity: sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=} + resolution: {integrity: sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ==} /path-type@3.0.0: resolution: {integrity: sha512-T2ZUsdZFHgA3u4e5PfPbjd7HDDpxPnQb5jN0SrDsjNSuVXHJqtwTnWqG0B1jZrgmJ/7lj1EmVIByWt1gxGkWvg==} @@ -17865,7 +17951,7 @@ packages: /regenerator-transform@0.15.1: resolution: {integrity: sha512-knzmNAcuyxV+gQCufkYcvOqX/qIIfHLv0u5x79kRxuGojfYVky1f15TzZEu2Avte8QGepvUNTnLskf8E6X6Vyg==} dependencies: - '@babel/runtime': 7.18.6 + '@babel/runtime': 7.21.5 /regex-not@1.0.2: resolution: {integrity: sha512-J6SDjUgDxQj5NusnOtdFxDwN/+HWykR8GELwctJ7mdqhcyy1xEc4SRFHUXvxTp661YaVKAjfRLZ9cCqS6tn32A==} From a618c479251eedf87e7add22a6007564d6fec9e9 Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Tue, 6 Jun 2023 15:26:18 -0400 Subject: [PATCH 13/72] followups to the class merge --- packages/compat/src/compat-app.ts | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/packages/compat/src/compat-app.ts b/packages/compat/src/compat-app.ts index 14abc1590..fbe7d6d78 100644 --- a/packages/compat/src/compat-app.ts +++ b/packages/compat/src/compat-app.ts @@ -1517,7 +1517,7 @@ export default class CompatApp { return this.legacyEmberAppInstance.project.pkg.keywords?.includes('ember-addon') ?? false; } - get name(): string { + private get name(): string { if (this.isDummy) { // here we accept the ember-cli behavior return this.legacyEmberAppInstance.name; @@ -1602,7 +1602,7 @@ export default class CompatApp { } @Memoize() - get config(): V1Config { + private get config(): V1Config { return new V1Config(this.configTree, this.legacyEmberAppInstance.env); } @@ -1639,7 +1639,7 @@ export default class CompatApp { }); } - get htmlTree() { + private get htmlTree() { if (this.legacyEmberAppInstance.tests) { return mergeTrees([this.indexTree, this.testIndexTree]); } else { @@ -1876,7 +1876,7 @@ export default class CompatApp { return mergeTrees([tree, ...concatentations], { overwrite: true }); } - addOtherAssets() { + private addOtherAssets() { for (let asset of this.legacyEmberAppInstance.otherAssetPaths) { this._publicAssets[`${asset.src}/${asset.file}`] = `${asset.dest}/${asset.file}`; } @@ -2082,7 +2082,7 @@ export default class CompatApp { } } - get vendorTree(): BroccoliNode | undefined { + private get vendorTree(): BroccoliNode | undefined { return this.ensureTree(this.legacyEmberAppInstance.trees.vendor); } @@ -2108,11 +2108,11 @@ export default class CompatApp { return this.requireFromEmberCLI('ember-cli-preprocess-registry/preprocessors'); } - get publicTree(): BroccoliNode | undefined { + private get publicTree(): BroccoliNode | undefined { return this.ensureTree(this.legacyEmberAppInstance.trees.public); } - processAppJS(): { appJS: BroccoliNode } { + private processAppJS(): { appJS: BroccoliNode } { let appTree = this.appTree; let testsTree = this.testsTree; let lintTree = this.lintTree; @@ -2234,22 +2234,19 @@ export default class CompatApp { this.movablePackageCache.seed(new DummyPackage(this.root, this.owningAddon, this.movablePackageCache)); } - let { appJS } = this.processAppJS(); - let htmlTree = this.htmlTree; let publicTree = this.publicTree; let configTree = this.config; - let appBootTree = this.appBoot; if (options.extraPublicTrees.length > 0) { publicTree = mergeTrees([publicTree, ...options.extraPublicTrees].filter(Boolean) as BroccoliNode[]); } let inTrees = { - appJS, - htmlTree, + appJS: this.processAppJS().appJS, + htmlTree: this.htmlTree, publicTree, configTree, - appBootTree, + appBootTree: this.appBoot, }; let instantiate = async (root: string, appSrcDir: string, packageCache: PackageCache) => { From f9b179ed7738ab2a7756e4e2da9ad979983ffc7d Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Tue, 6 Jun 2023 15:35:20 -0400 Subject: [PATCH 14/72] more cleanup --- packages/compat/src/compat-app.ts | 69 +++++++++++++------------------ 1 file changed, 28 insertions(+), 41 deletions(-) diff --git a/packages/compat/src/compat-app.ts b/packages/compat/src/compat-app.ts index fbe7d6d78..faed4e308 100644 --- a/packages/compat/src/compat-app.ts +++ b/packages/compat/src/compat-app.ts @@ -1489,20 +1489,14 @@ class CompatAppBuilder { } } -interface ExtraTree { - __prevStageTree: BroccoliNode; -} - // This runs at broccoli-pipeline-construction time, whereas our actual // CompatAppBuilder instance only becomes available during tree-building time. export default class CompatApp { - private inTrees: TreeNames; private annotation = '@embroider/compat/app'; - private instantiate: (root: string, appSrcDir: string, packageCache: PackageCache) => Promise; - private active: CompatAppBuilder | undefined; private outputPath: string | undefined; private packageCache: PackageCache | undefined; + private options: Required; // used to signal that this is a dummy app owned by a particular addon owningAddon: Package | undefined; @@ -2224,7 +2218,7 @@ export default class CompatApp { } constructor(private legacyEmberAppInstance: EmberAppInstance, _options?: Options) { - let options = optionsWithDefaults(_options); + this.options = optionsWithDefaults(_options); this.movablePackageCache = new MovablePackageCache(MacrosConfig.for(legacyEmberAppInstance, this.root), this.root); @@ -2233,54 +2227,56 @@ export default class CompatApp { this.movablePackageCache.seed(this.owningAddon); this.movablePackageCache.seed(new DummyPackage(this.root, this.owningAddon, this.movablePackageCache)); } + } + private inTrees(prevStageTree: BroccoliNode) { let publicTree = this.publicTree; let configTree = this.config; - if (options.extraPublicTrees.length > 0) { - publicTree = mergeTrees([publicTree, ...options.extraPublicTrees].filter(Boolean) as BroccoliNode[]); + if (this.options.extraPublicTrees.length > 0) { + publicTree = mergeTrees([publicTree, ...this.options.extraPublicTrees].filter(Boolean) as BroccoliNode[]); } - let inTrees = { + return { appJS: this.processAppJS().appJS, htmlTree: this.htmlTree, publicTree, configTree, appBootTree: this.appBoot, + prevStageTree, }; + } - let instantiate = async (root: string, appSrcDir: string, packageCache: PackageCache) => { - let appPackage = packageCache.get(appSrcDir); - let macrosConfig = MacrosConfig.for(legacyEmberAppInstance, appSrcDir); - - return new CompatAppBuilder( - root, - appPackage, - options, - this, - configTree, - packageCache.get(join(root, 'node_modules', '@embroider', 'synthesized-vendor')), - packageCache.get(join(root, 'node_modules', '@embroider', 'synthesized-styles')), - macrosConfig - ); - }; + private async instantiate(root: string, appSrcDir: string, packageCache: PackageCache, configTree: V1Config) { + let appPackage = packageCache.get(appSrcDir); + let macrosConfig = MacrosConfig.for(this.legacyEmberAppInstance, appSrcDir); - this.inTrees = inTrees; - this.instantiate = instantiate; + return new CompatAppBuilder( + root, + appPackage, + this.options, + this, + configTree, + packageCache.get(join(root, 'node_modules', '@embroider', 'synthesized-vendor')), + packageCache.get(join(root, 'node_modules', '@embroider', 'synthesized-styles')), + macrosConfig + ); } asStage(prevStage: Stage): Stage { - let tree = () => - new WaitForTrees(this.augment(this.inTrees, prevStage.tree), this.annotation, async treePaths => { + let tree = () => { + let inTrees = this.inTrees(prevStage.tree); + return new WaitForTrees(inTrees, this.annotation, async treePaths => { if (!this.active) { let { outputPath, packageCache } = await prevStage.ready(); this.outputPath = outputPath; this.packageCache = packageCache; - this.active = await this.instantiate(outputPath, prevStage.inputPath, packageCache); + this.active = await this.instantiate(outputPath, prevStage.inputPath, packageCache, inTrees.configTree); } - await this.active.build(this.deAugment(treePaths)); + await this.active.build(treePaths); this.deferReady.resolve(); }); + }; return { get inputPath() { @@ -2305,15 +2301,6 @@ export default class CompatApp { let promise: Promise = new Promise(r => (resolve = r)); return { resolve: resolve!, promise }; } - - private augment(inTrees: TreeNames, prevStageTree: BroccoliNode): TreeNames & ExtraTree { - return Object.assign({ __prevStageTree: prevStageTree }, inTrees); - } - - private deAugment(treePaths: OutputPaths): OutputPaths { - delete (treePaths as any).__prevStageTree; - return treePaths; - } } function maybeReplace(dom: JSDOM, element: Element | undefined): Node | undefined { From e9aaad8c3a957e8b7a2dc236528dfb60d8cf833f Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Tue, 6 Jun 2023 15:39:03 -0400 Subject: [PATCH 15/72] dropping PrebuiltAddon This is going to come back, but in a different implementation --- packages/compat/src/default-pipeline.ts | 22 ++++-------- packages/compat/src/index.ts | 1 - packages/compat/src/prebuilt-addons.ts | 43 ------------------------ packages/compat/src/v1-instance-cache.ts | 6 ++-- 4 files changed, 10 insertions(+), 62 deletions(-) delete mode 100644 packages/compat/src/prebuilt-addons.ts diff --git a/packages/compat/src/default-pipeline.ts b/packages/compat/src/default-pipeline.ts index ec976d1cc..d7d17e89f 100644 --- a/packages/compat/src/default-pipeline.ts +++ b/packages/compat/src/default-pipeline.ts @@ -1,4 +1,4 @@ -import { App, Addons as CompatAddons, Options, PrebuiltAddons } from '.'; +import { App, Addons as CompatAddons, Options } from '.'; import { toBroccoliPlugin, PackagerConstructor, Variant, EmberAppInstance } from '@embroider/core'; import { tmpdir } from '@embroider/core'; import { Node } from 'broccoli-node-api'; @@ -29,21 +29,13 @@ export default function defaultPipeline( let outputPath: string; let addons; - if (process.env.REUSE_WORKSPACE) { - addons = new PrebuiltAddons(emberApp, options, process.env.REUSE_WORKSPACE); - } else { - if (process.env.SAVE_WORKSPACE) { - options.workspaceDir = process.env.SAVE_WORKSPACE; - } else { - options.workspaceDir = stableWorkspaceDir(emberApp.project.root, emberApp.env); - } + options.workspaceDir = stableWorkspaceDir(emberApp.project.root, emberApp.env); - emberApp.project.ui.write(`Building into ${options.workspaceDir}\n`); - addons = new CompatAddons(emberApp, options); - addons.ready().then(result => { - outputPath = result.outputPath; - }); - } + emberApp.project.ui.write(`Building into ${options.workspaceDir}\n`); + addons = new CompatAddons(emberApp, options); + addons.ready().then(result => { + outputPath = result.outputPath; + }); if (process.env.STAGE1_ONLY) { return mergeTrees([addons.tree, writeFile('.stage1-output', () => outputPath)]); diff --git a/packages/compat/src/index.ts b/packages/compat/src/index.ts index faac2ce13..3383f6f93 100644 --- a/packages/compat/src/index.ts +++ b/packages/compat/src/index.ts @@ -1,6 +1,5 @@ export { default as App } from './compat-app'; export { default as Addons } from './compat-addons'; -export { default as PrebuiltAddons } from './prebuilt-addons'; export { default as Options, recommendedOptions } from './options'; export { default as V1Addon } from './v1-addon'; export { default as compatBuild, PipelineOptions } from './default-pipeline'; diff --git a/packages/compat/src/prebuilt-addons.ts b/packages/compat/src/prebuilt-addons.ts deleted file mode 100644 index 8437b5fbc..000000000 --- a/packages/compat/src/prebuilt-addons.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { Stage, Package, PackageCache } from '@embroider/core'; -import { realpathSync, readJSONSync } from 'fs-extra'; -import { UnwatchedDir } from 'broccoli-source'; -import { Node } from 'broccoli-node-api'; -import { join } from 'path'; -import Options, { optionsWithDefaults } from './options'; -import V1InstanceCache from './v1-instance-cache'; - -export default class PrebuiltAddons implements Stage { - private packageCache: PackageCache; - private appDestDir: string; - readonly inputPath: string; - readonly tree: Node; - - constructor(legacyEmberAppInstance: object, maybeOptions: Options | undefined, workspaceDir: string) { - let options = optionsWithDefaults(maybeOptions); - let v1Cache = V1InstanceCache.forApp(legacyEmberAppInstance, options); - this.inputPath = realpathSync(v1Cache.app.root); - let { appDestDir } = readJSONSync(join(workspaceDir, '.embroider-reuse.json')); - this.appDestDir = realpathSync(join(workspaceDir, appDestDir)); - this.packageCache = new RehomedPackageCache(this.inputPath, this.appDestDir); - this.tree = new UnwatchedDir(this.inputPath); - } - - async ready(): Promise<{ packageCache: PackageCache; outputPath: string }> { - return { - packageCache: this.packageCache, - outputPath: this.appDestDir, - }; - } -} - -class RehomedPackageCache extends PackageCache { - constructor(private appSrcDir: string, appDestDir: string) { - super(appDestDir); - } - basedir(pkg: Package): string { - if (pkg.root === this.appSrcDir) { - return this.appRoot; - } - return super.basedir(pkg); - } -} diff --git a/packages/compat/src/v1-instance-cache.ts b/packages/compat/src/v1-instance-cache.ts index ab0b05e1d..8ac8a9d0d 100644 --- a/packages/compat/src/v1-instance-cache.ts +++ b/packages/compat/src/v1-instance-cache.ts @@ -4,7 +4,7 @@ import V1Addon, { V1AddonConstructor } from './v1-addon'; import { pathExistsSync } from 'fs-extra'; -import { AddonInstance, getOrCreate } from '@embroider/core'; +import { AddonInstance, EmberAppInstance, getOrCreate } from '@embroider/core'; import Options from './options'; import isEqual from 'lodash/isEqual'; import CompatApp from './compat-app'; @@ -12,7 +12,7 @@ import CompatApp from './compat-app'; export default class V1InstanceCache { static caches: WeakMap = new WeakMap(); - static forApp(emberApp: object, options: Required): V1InstanceCache { + static forApp(emberApp: EmberAppInstance, options: Required): V1InstanceCache { let instance = getOrCreate(this.caches, emberApp, () => new this(emberApp, options)); if (!isEqual(instance.options, options)) { throw new Error(`attempted double set of app Options`); @@ -28,7 +28,7 @@ export default class V1InstanceCache { app: CompatApp; orderIdx: number; - private constructor(oldApp: any, private options: Required) { + private constructor(oldApp: EmberAppInstance, private options: Required) { this.app = new CompatApp(oldApp, options); this.orderIdx = 0; From 20ce08f04f334a28f180c8d4ba8276931570415a Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Tue, 6 Jun 2023 16:02:10 -0400 Subject: [PATCH 16/72] more cleanup --- packages/compat/src/compat-addons.ts | 20 ++++++--------- packages/compat/src/compat-app.ts | 4 +-- packages/compat/src/default-pipeline.ts | 12 ++++----- packages/compat/src/v1-instance-cache.ts | 32 ++++++++---------------- 4 files changed, 26 insertions(+), 42 deletions(-) diff --git a/packages/compat/src/compat-addons.ts b/packages/compat/src/compat-addons.ts index 867cfc433..36d6a289b 100644 --- a/packages/compat/src/compat-addons.ts +++ b/packages/compat/src/compat-addons.ts @@ -1,12 +1,11 @@ import { Node } from 'broccoli-node-api'; import { join, relative, dirname, isAbsolute, sep } from 'path'; import { emptyDirSync, ensureSymlinkSync, ensureDirSync, realpathSync, copySync, writeJSONSync } from 'fs-extra'; -import { Stage, Package, PackageCache, WaitForTrees, mangledEngineRoot, EmberAppInstance } from '@embroider/core'; +import { Stage, Package, PackageCache, WaitForTrees, mangledEngineRoot } from '@embroider/core'; import V1InstanceCache from './v1-instance-cache'; import { MovedPackageCache } from './moved-package-cache'; import { Memoize } from 'typescript-memoize'; import buildCompatAddon from './build-compat-addon'; -import Options, { optionsWithDefaults } from './options'; import TreeSync from 'tree-sync'; import { WatchedDir } from 'broccoli-source'; import CompatApp from './compat-app'; @@ -30,20 +29,17 @@ export default class CompatAddons implements Stage { private v1Cache: V1InstanceCache; readonly inputPath: string; - constructor(legacyEmberAppInstance: EmberAppInstance, maybeOptions?: Options) { - let options = optionsWithDefaults(maybeOptions); - let v1Cache = V1InstanceCache.forApp(legacyEmberAppInstance, options); - + constructor(private compatApp: CompatApp) { // we want this to be stable across builds, because it becomes part of the // path to all of the files that the stage3 packager sees, and we want to // benefit from on-disk caching in stage3 packagers. - ensureDirSync(options.workspaceDir!); - this.destDir = realpathSync(options.workspaceDir!); + ensureDirSync(compatApp.options.workspaceDir!); + this.destDir = realpathSync(compatApp.options.workspaceDir!); - this.packageCache = v1Cache.app.movablePackageCache.moveAddons(this.destDir); - this.inputPath = v1Cache.app.root; + this.packageCache = compatApp.movablePackageCache.moveAddons(this.destDir); + this.inputPath = compatApp.root; this.treeSyncMap = new WeakMap(); - this.v1Cache = v1Cache; + this.v1Cache = new V1InstanceCache(compatApp); } get tree(): Node { @@ -57,7 +53,7 @@ export default class CompatAddons implements Stage { .filter(pkg => pkg.mayRebuild) .map(pkg => new WatchedDir(pkg.root)); - let { synthVendor, synthStyles } = this.getSyntheticPackages(this.v1Cache.app, movedAddons); + let { synthVendor, synthStyles } = this.getSyntheticPackages(this.compatApp, movedAddons); return new WaitForTrees( { movedAddons, synthVendor, synthStyles, watchedUnmovedAddons }, '@embroider/compat/addons', diff --git a/packages/compat/src/compat-app.ts b/packages/compat/src/compat-app.ts index faed4e308..6d2f2c7f6 100644 --- a/packages/compat/src/compat-app.ts +++ b/packages/compat/src/compat-app.ts @@ -1496,7 +1496,7 @@ export default class CompatApp { private active: CompatAppBuilder | undefined; private outputPath: string | undefined; private packageCache: PackageCache | undefined; - private options: Required; + readonly options: Required; // used to signal that this is a dummy app owned by a particular addon owningAddon: Package | undefined; @@ -2217,7 +2217,7 @@ export default class CompatApp { ); } - constructor(private legacyEmberAppInstance: EmberAppInstance, _options?: Options) { + constructor(readonly legacyEmberAppInstance: EmberAppInstance, _options?: Options) { this.options = optionsWithDefaults(_options); this.movablePackageCache = new MovablePackageCache(MacrosConfig.for(legacyEmberAppInstance, this.root), this.root); diff --git a/packages/compat/src/default-pipeline.ts b/packages/compat/src/default-pipeline.ts index d7d17e89f..4f3b50b41 100644 --- a/packages/compat/src/default-pipeline.ts +++ b/packages/compat/src/default-pipeline.ts @@ -32,7 +32,10 @@ export default function defaultPipeline( options.workspaceDir = stableWorkspaceDir(emberApp.project.root, emberApp.env); emberApp.project.ui.write(`Building into ${options.workspaceDir}\n`); - addons = new CompatAddons(emberApp, options); + + let embroiderApp = new App(emberApp, options); + + addons = new CompatAddons(embroiderApp); addons.ready().then(result => { outputPath = result.outputPath; }); @@ -41,16 +44,13 @@ export default function defaultPipeline( return mergeTrees([addons.tree, writeFile('.stage1-output', () => outputPath)]); } - let embroiderApp = new App(emberApp, options); - let appStage = embroiderApp.asStage(addons); - if (process.env.STAGE2_ONLY || !packager) { - return mergeTrees([appStage.tree, writeFile('.stage2-output', () => outputPath)]); + return mergeTrees([embroiderApp.asStage(addons).tree, writeFile('.stage2-output', () => outputPath)]); } let BroccoliPackager = toBroccoliPlugin(packager); let variants = (options && options.variants) || defaultVariants(emberApp); - return new BroccoliPackager(appStage, variants, options && options.packagerOptions); + return new BroccoliPackager(embroiderApp.asStage(addons), variants, options && options.packagerOptions); } function hasFastboot(emberApp: EmberAppInstance | EmberAppInstance) { diff --git a/packages/compat/src/v1-instance-cache.ts b/packages/compat/src/v1-instance-cache.ts index 8ac8a9d0d..cb245a874 100644 --- a/packages/compat/src/v1-instance-cache.ts +++ b/packages/compat/src/v1-instance-cache.ts @@ -4,37 +4,25 @@ import V1Addon, { V1AddonConstructor } from './v1-addon'; import { pathExistsSync } from 'fs-extra'; -import { AddonInstance, EmberAppInstance, getOrCreate } from '@embroider/core'; -import Options from './options'; -import isEqual from 'lodash/isEqual'; +import { AddonInstance, getOrCreate } from '@embroider/core'; import CompatApp from './compat-app'; export default class V1InstanceCache { - static caches: WeakMap = new WeakMap(); - - static forApp(emberApp: EmberAppInstance, options: Required): V1InstanceCache { - let instance = getOrCreate(this.caches, emberApp, () => new this(emberApp, options)); - if (!isEqual(instance.options, options)) { - throw new Error(`attempted double set of app Options`); - } - return instance; - } - // maps from package root directories to known V1 instances of that packages. // There can be many because a single copy of an addon may be consumed by many // other packages and each gets an instance. private addons: Map = new Map(); - app: CompatApp; - orderIdx: number; + private app: CompatApp; + private orderIdx: number; - private constructor(oldApp: EmberAppInstance, private options: Required) { - this.app = new CompatApp(oldApp, options); + constructor(app: CompatApp) { + this.app = app; this.orderIdx = 0; - // no reason to do this on demand because oldApp already eagerly loaded - // all descendants - (oldApp.project.addons as AddonInstance[]).forEach(addon => { + // no reason to do this on demand because the legacy ember app instance + // already loaded all descendants + (app.legacyEmberAppInstance.project.addons as AddonInstance[]).forEach(addon => { this.addAddon(addon); }); } @@ -43,7 +31,7 @@ export default class V1InstanceCache { let packageName = addonInstance.pkg.name; // if the user registered something (including "null", which allows // disabling the built-in adapters), that takes precedence. - let AdapterClass = this.options.compatAdapters.get(packageName); + let AdapterClass = this.app.options.compatAdapters.get(packageName); if (AdapterClass === null) { return V1Addon; @@ -75,7 +63,7 @@ export default class V1InstanceCache { this.orderIdx += 1; let Klass = this.adapterClass(addonInstance); - let v1Addon = new Klass(addonInstance, this.options, this.app, this.app.movablePackageCache, this.orderIdx); + let v1Addon = new Klass(addonInstance, this.app.options, this.app, this.app.movablePackageCache, this.orderIdx); let pkgs = getOrCreate(this.addons, v1Addon.root, () => []); pkgs.push(v1Addon); } From 064a4cf00fd12129865b00da089e05d1a8323ab2 Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Tue, 6 Jun 2023 16:17:47 -0400 Subject: [PATCH 17/72] simplifying macrosConfig handling --- packages/compat/src/compat-app.ts | 74 +++++++++++++------------------ 1 file changed, 30 insertions(+), 44 deletions(-) diff --git a/packages/compat/src/compat-app.ts b/packages/compat/src/compat-app.ts index 6d2f2c7f6..3c2326c84 100644 --- a/packages/compat/src/compat-app.ts +++ b/packages/compat/src/compat-app.ts @@ -155,19 +155,8 @@ class CompatAppBuilder { private compatApp: CompatApp, private configTree: V1Config, private synthVendor: Package, - private synthStyles: Package, - private macrosConfig: MacrosConfig - ) { - // this uses globalConfig because it's a way for packages to ask "is - // Embroider doing this build?". So it's necessarily global, not scoped to - // any subgraph of dependencies. - macrosConfig.setGlobalConfig(__filename, `@embroider/core`, { - // this is hard-coded to true because it literally means "embroider is - // building this Ember app". You can see non-true when using the Embroider - // macros in a classic build. - active: true, - }); - } + private synthStyles: Package + ) {} private appJSSrcDir(treePaths: OutputPaths) { return treePaths.appJS; @@ -235,13 +224,6 @@ class CompatAppBuilder { } } - private developingAddons(): string[] { - if (this.compatApp.owningAddon) { - return [this.compatApp.owningAddon.root]; - } - return []; - } - private activeAddonChildren(pkg: Package = this.appPackage): AddonPackage[] { let result = (pkg.dependencies.filter(this.isActiveAddon) as AddonPackage[]).filter( // When looking for child addons, we want to ignore 'peerDependencies' of @@ -543,7 +525,7 @@ class CompatAppBuilder { babel.plugins.push([require.resolve('babel-plugin-ember-template-compilation'), this.etcOptions(resolverConfig)]); // this is @embroider/macros configured for full stage3 resolution - babel.plugins.push(...this.macrosConfig.babelPluginConfig()); + babel.plugins.push(...this.compatApp.macrosConfig.babelPluginConfig()); let colocationOptions: TemplateColocationPluginOptions = { appRoot: this.root, @@ -1025,17 +1007,9 @@ class CompatAppBuilder { } async build(inputPaths: OutputPaths) { - if (this.compatApp.env !== 'production') { - this.macrosConfig.enablePackageDevelopment(this.root); - this.macrosConfig.enableRuntimeMode(); - } - for (let pkgRoot of this.developingAddons()) { - this.macrosConfig.enablePackageDevelopment(pkgRoot); - } - // on the first build, we lock down the macros config. on subsequent builds, // this doesn't do anything anyway because it's idempotent. - this.macrosConfig.finalize(); + this.compatApp.macrosConfig.finalize(); let appFiles = this.updateAppJS(inputPaths); let emberENV = this.configTree.readConfig().EmberENV; @@ -1100,7 +1074,7 @@ class CompatAppBuilder { let transforms = this.compatApp.htmlbarsPlugins; let { plugins: macroPlugins, setConfig } = MacrosConfig.transforms(); - setConfig(this.macrosConfig); + setConfig(this.compatApp.macrosConfig); for (let macroPlugin of macroPlugins) { transforms.push(macroPlugin as any); } @@ -1498,9 +1472,6 @@ export default class CompatApp { private packageCache: PackageCache | undefined; readonly options: Required; - // used to signal that this is a dummy app owned by a particular addon - owningAddon: Package | undefined; - private _publicAssets: { [filePath: string]: string } = Object.create(null); private _implicitScripts: string[] = []; private _implicitStyles: string[] = []; @@ -2217,15 +2188,34 @@ export default class CompatApp { ); } + readonly macrosConfig: MacrosConfig; + constructor(readonly legacyEmberAppInstance: EmberAppInstance, _options?: Options) { this.options = optionsWithDefaults(_options); - this.movablePackageCache = new MovablePackageCache(MacrosConfig.for(legacyEmberAppInstance, this.root), this.root); + this.macrosConfig = MacrosConfig.for(legacyEmberAppInstance, this.root); + if (this.env !== 'production') { + this.macrosConfig.enablePackageDevelopment(this.root); + this.macrosConfig.enableRuntimeMode(); + } + + // this uses globalConfig because it's a way for packages to ask "is + // Embroider doing this build?". So it's necessarily global, not scoped to + // any subgraph of dependencies. + this.macrosConfig.setGlobalConfig(__filename, `@embroider/core`, { + // this is hard-coded to true because it literally means "embroider is + // building this Ember app". You can see non-true when using the Embroider + // macros in a classic build. + active: true, + }); + + this.movablePackageCache = new MovablePackageCache(this.macrosConfig, this.root); if (this.isDummy) { - this.owningAddon = new OwningAddon(legacyEmberAppInstance.project.root, this.movablePackageCache); - this.movablePackageCache.seed(this.owningAddon); - this.movablePackageCache.seed(new DummyPackage(this.root, this.owningAddon, this.movablePackageCache)); + let owningAddon = new OwningAddon(legacyEmberAppInstance.project.root, this.movablePackageCache); + this.movablePackageCache.seed(owningAddon); + this.movablePackageCache.seed(new DummyPackage(this.root, owningAddon, this.movablePackageCache)); + this.macrosConfig.enablePackageDevelopment(owningAddon.root); } } @@ -2248,18 +2238,14 @@ export default class CompatApp { } private async instantiate(root: string, appSrcDir: string, packageCache: PackageCache, configTree: V1Config) { - let appPackage = packageCache.get(appSrcDir); - let macrosConfig = MacrosConfig.for(this.legacyEmberAppInstance, appSrcDir); - return new CompatAppBuilder( root, - appPackage, + packageCache.get(appSrcDir), this.options, this, configTree, packageCache.get(join(root, 'node_modules', '@embroider', 'synthesized-vendor')), - packageCache.get(join(root, 'node_modules', '@embroider', 'synthesized-styles')), - macrosConfig + packageCache.get(join(root, 'node_modules', '@embroider', 'synthesized-styles')) ); } From ac58e4351a1e38b48e57f6314a0d2eac7d7c0aab Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Tue, 6 Jun 2023 16:35:24 -0400 Subject: [PATCH 18/72] more cleanup --- packages/compat/src/compat-app.ts | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/packages/compat/src/compat-app.ts b/packages/compat/src/compat-app.ts index 3c2326c84..a698318f5 100644 --- a/packages/compat/src/compat-app.ts +++ b/packages/compat/src/compat-app.ts @@ -158,12 +158,8 @@ class CompatAppBuilder { private synthStyles: Package ) {} - private appJSSrcDir(treePaths: OutputPaths) { - return treePaths.appJS; - } - @Memoize() - private fastbootJSSrcDir(_treePaths: OutputPaths) { + private fastbootJSSrcDir() { let target = join(this.root, 'fastboot'); if (pathExistsSync(target)) { return target; @@ -801,7 +797,7 @@ class CompatAppBuilder { private appDiffers: { differ: AppDiffer; engine: EngineSummary }[] | undefined; private updateAppJS(inputPaths: OutputPaths): Engine[] { - let appJSPath = this.appJSSrcDir(inputPaths); + let appJSPath = inputPaths.appJS; if (!this.appDiffers) { let engines = this.partitionEngines(appJSPath); this.appDiffers = engines.map(engine => { @@ -812,7 +808,7 @@ class CompatAppBuilder { engine.sourcePath, [...engine.addons], true, - this.fastbootJSSrcDir(inputPaths), + this.fastbootJSSrcDir(), this.babelParserConfig() ); } else { From 0915c4a0febbe77079d16daa0d6706007b2f2930 Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Tue, 6 Jun 2023 16:41:22 -0400 Subject: [PATCH 19/72] fix fastboot regression --- packages/compat/src/compat-app.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/compat/src/compat-app.ts b/packages/compat/src/compat-app.ts index a698318f5..f52f521f5 100644 --- a/packages/compat/src/compat-app.ts +++ b/packages/compat/src/compat-app.ts @@ -160,7 +160,7 @@ class CompatAppBuilder { @Memoize() private fastbootJSSrcDir() { - let target = join(this.root, 'fastboot'); + let target = join(this.compatApp.root, 'fastboot'); if (pathExistsSync(target)) { return target; } From ee54e0a7fde493f6be42b878cb2e0fd57113bb91 Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Tue, 6 Jun 2023 16:47:48 -0400 Subject: [PATCH 20/72] splitting CompatAppBuilder to its own module --- packages/compat/src/compat-app-builder.ts | 1627 ++++++++++++++++++++ packages/compat/src/compat-app.ts | 1633 +-------------------- 2 files changed, 1634 insertions(+), 1626 deletions(-) create mode 100644 packages/compat/src/compat-app-builder.ts diff --git a/packages/compat/src/compat-app-builder.ts b/packages/compat/src/compat-app-builder.ts new file mode 100644 index 000000000..8834018bc --- /dev/null +++ b/packages/compat/src/compat-app-builder.ts @@ -0,0 +1,1627 @@ +import { Node as BroccoliNode } from 'broccoli-node-api'; +import { + OutputPaths, + Asset, + EmberAsset, + AddonPackage, + Engine, + AppMeta, + explicitRelative, + extensionsPattern, + TemplateColocationPluginOptions, + debug, + warn, + jsHandlebarsCompile, + templateColocationPluginPath, + cacheBustingPluginVersion, + cacheBustingPluginPath, +} from '@embroider/core'; +import walkSync from 'walk-sync'; +import { resolve as resolvePath, posix } from 'path'; +import { JSDOM } from 'jsdom'; +import Options from './options'; +import { CompatResolverOptions } from './resolver-transform'; +import { activePackageRules, PackageRules } from './dependency-rules'; +import flatMap from 'lodash/flatMap'; +import sortBy from 'lodash/sortBy'; +import flatten from 'lodash/flatten'; +import partition from 'lodash/partition'; +import mergeWith from 'lodash/mergeWith'; +import cloneDeep from 'lodash/cloneDeep'; +import { sync as resolveSync } from 'resolve'; +import bind from 'bind-decorator'; +import { outputJSONSync, readJSONSync, statSync, unlinkSync, writeFileSync } from 'fs-extra'; +import type { Options as EtcOptions } from 'babel-plugin-ember-template-compilation'; +import type { Options as ResolverTransformOptions } from './resolver-transform'; +import type { Options as AdjustImportsOptions } from './babel-plugin-adjust-imports'; +import { PreparedEmberHTML } from '@embroider/core/src/ember-html'; +import { InMemoryAsset, OnDiskAsset, ImplicitAssetPaths } from '@embroider/core/src/asset'; +import { makePortable } from '@embroider/core/src/portable-babel-config'; +import { AppFiles, EngineSummary, RouteFiles } from '@embroider/core/src/app-files'; +import { mangledEngineRoot } from '@embroider/core/src/engine-mangler'; +import { PortableHint, maybeNodeModuleVersion } from '@embroider/core/src/portable'; +import AppDiffer from '@embroider/core/src/app-differ'; +import assertNever from 'assert-never'; +import { Memoize } from 'typescript-memoize'; +import { join, dirname, sep } from 'path'; +import resolve from 'resolve'; +import { V1Config } from './v1-config'; +import { AddonMeta, Package, PackageInfo } from '@embroider/core'; +import { ensureDirSync, copySync, readdirSync, pathExistsSync } from 'fs-extra'; +import { TransformOptions } from '@babel/core'; +import { MacrosConfig } from '@embroider/macros/src/node'; +import SourceMapConcat from 'fast-sourcemap-concat'; +import escapeRegExp from 'escape-string-regexp'; + +import type CompatApp from './compat-app'; + +// This exists during the actual broccoli build step. As opposed to CompatApp, +// which also exists during pipeline-construction time. + +export class CompatAppBuilder { + // for each relativePath, an Asset we have already emitted + private assets: Map = new Map(); + + constructor( + private root: string, + private appPackage: Package, + private options: Required, + private compatApp: CompatApp, + private configTree: V1Config, + private synthVendor: Package, + private synthStyles: Package + ) {} + + @Memoize() + private fastbootJSSrcDir() { + let target = join(this.compatApp.root, 'fastboot'); + if (pathExistsSync(target)) { + return target; + } + } + + private extractAssets(treePaths: OutputPaths): Asset[] { + let assets: Asset[] = []; + + // Everything in our traditional public tree is an on-disk asset + if (treePaths.publicTree) { + walkSync + .entries(treePaths.publicTree, { + directories: false, + }) + .forEach(entry => { + assets.push({ + kind: 'on-disk', + relativePath: entry.relativePath, + sourcePath: entry.fullPath, + mtime: entry.mtime as unknown as number, // https://github.com/joliss/node-walk-sync/pull/38 + size: entry.size, + }); + }); + } + + // ember-cli traditionally outputs a dummy testem.js file to prevent + // spurious errors when running tests under "ember s". + if (this.compatApp.shouldBuildTests) { + let testemAsset = this.findTestemAsset(); + if (testemAsset) { + assets.push(testemAsset); + } + } + + for (let asset of this.emberEntrypoints(treePaths.htmlTree)) { + assets.push(asset); + } + + return assets; + } + + @Memoize() + private findTestemAsset(): Asset | undefined { + let sourcePath; + try { + sourcePath = resolveSync('ember-cli/lib/broccoli/testem.js', { basedir: this.root }); + } catch (err) {} + if (sourcePath) { + let stat = statSync(sourcePath); + return { + kind: 'on-disk', + relativePath: 'testem.js', + sourcePath, + mtime: stat.mtime.getTime(), + size: stat.size, + }; + } + } + + private activeAddonChildren(pkg: Package = this.appPackage): AddonPackage[] { + let result = (pkg.dependencies.filter(this.isActiveAddon) as AddonPackage[]).filter( + // When looking for child addons, we want to ignore 'peerDependencies' of + // a given package, to align with how ember-cli resolves addons. So here + // we only include dependencies that definitely appear in one of the other + // sections. + addon => pkg.packageJSON.dependencies?.[addon.name] || pkg.packageJSON.devDependencies?.[addon.name] + ); + if (pkg === this.appPackage) { + let extras = [this.synthVendor, this.synthStyles].filter(this.isActiveAddon) as AddonPackage[]; + result = [...result, ...extras]; + } + return result.sort(this.orderAddons); + } + + @Memoize() + private get allActiveAddons(): AddonPackage[] { + let result = this.appPackage.findDescendants(this.isActiveAddon) as AddonPackage[]; + let extras = [this.synthVendor, this.synthStyles].filter(this.isActiveAddon) as AddonPackage[]; + let extraDescendants = flatMap(extras, dep => dep.findDescendants(this.isActiveAddon)) as AddonPackage[]; + result = [...result, ...extras, ...extraDescendants]; + return result.sort(this.orderAddons); + } + + @bind + private isActiveAddon(pkg: Package): boolean { + // todo: filter by addon-provided hook + return pkg.isEmberPackage(); + } + + @bind + private orderAddons(depA: Package, depB: Package): number { + let depAIdx = 0; + let depBIdx = 0; + + if (depA && depA.meta && depA.isV2Addon()) { + depAIdx = depA.meta['order-index'] || 0; + } + if (depB && depB.meta && depB.isV2Addon()) { + depBIdx = depB.meta['order-index'] || 0; + } + + return depAIdx - depBIdx; + } + + private resolvableExtensions(): string[] { + // webpack's default is ['.wasm', '.mjs', '.js', '.json']. Keeping that + // subset in that order is sensible, since many third-party libraries will + // expect it to work that way. + // + // For TS, we defer to ember-cli-babel, and the setting for + // "enableTypescriptTransform" can be set with and without + // ember-cli-typescript + return ['.wasm', '.mjs', '.js', '.json', '.ts', '.hbs', '.hbs.js']; + } + + private *emberEntrypoints(htmlTreePath: string): IterableIterator { + let classicEntrypoints = [ + { entrypoint: 'index.html', includeTests: false }, + { entrypoint: 'tests/index.html', includeTests: true }, + ]; + if (!this.compatApp.shouldBuildTests) { + classicEntrypoints.pop(); + } + for (let { entrypoint, includeTests } of classicEntrypoints) { + let sourcePath = join(htmlTreePath, entrypoint); + let stats = statSync(sourcePath); + let asset: EmberAsset = { + kind: 'ember', + relativePath: entrypoint, + includeTests, + sourcePath, + mtime: stats.mtime.getTime(), + size: stats.size, + rootURL: this.rootURL(), + prepare: (dom: JSDOM) => { + let scripts = [...dom.window.document.querySelectorAll('script')]; + let styles = [...dom.window.document.querySelectorAll('link[rel="stylesheet"]')] as HTMLLinkElement[]; + + return { + javascript: definitelyReplace(dom, this.compatApp.findAppScript(scripts, entrypoint)), + styles: definitelyReplace(dom, this.compatApp.findAppStyles(styles, entrypoint)), + implicitScripts: definitelyReplace(dom, this.compatApp.findVendorScript(scripts, entrypoint)), + implicitStyles: definitelyReplace(dom, this.compatApp.findVendorStyles(styles, entrypoint)), + testJavascript: maybeReplace(dom, this.compatApp.findTestScript(scripts)), + implicitTestScripts: maybeReplace(dom, this.compatApp.findTestSupportScript(scripts)), + implicitTestStyles: maybeReplace(dom, this.compatApp.findTestSupportStyles(styles)), + }; + }, + }; + yield asset; + } + } + + private modulePrefix(): string { + return this.configTree.readConfig().modulePrefix; + } + + private podModulePrefix(): string | undefined { + return this.configTree.readConfig().podModulePrefix; + } + + private rootURL(): string { + return this.configTree.readConfig().rootURL; + } + + private templateCompilerPath(): string { + return 'ember-source/vendor/ember/ember-template-compiler'; + } + + @Memoize() + private activeRules() { + return activePackageRules(this.options.packageRules.concat(defaultAddonPackageRules()), [ + { name: this.appPackage.name, version: this.appPackage.version, root: this.root }, + ...this.allActiveAddons.filter(p => p.meta['auto-upgraded']), + ]); + } + + private resolverConfig(engines: Engine[]): CompatResolverOptions { + let renamePackages = Object.assign({}, ...this.allActiveAddons.map(dep => dep.meta['renamed-packages'])); + let renameModules = Object.assign({}, ...this.allActiveAddons.map(dep => dep.meta['renamed-modules'])); + + let activeAddons: CompatResolverOptions['activeAddons'] = {}; + for (let addon of this.allActiveAddons) { + activeAddons[addon.name] = addon.root; + } + + let config: CompatResolverOptions = { + // this part is the base ModuleResolverOptions as required by @embroider/core + activeAddons, + renameModules, + renamePackages, + resolvableExtensions: this.resolvableExtensions(), + appRoot: this.root, + engines: engines.map((engine, index) => ({ + packageName: engine.package.name, + root: index === 0 ? this.root : engine.package.root, // first engine is the app, which has been relocated to this.roto + activeAddons: [...engine.addons] + .map(a => ({ + name: a.name, + root: a.root, + })) + // the traditional order is the order in which addons will run, such + // that the last one wins. Our resolver's order is the order to + // search, so first one wins. + .reverse(), + })), + + // this is the additional stufff that @embroider/compat adds on top to do + // global template resolving + modulePrefix: this.modulePrefix(), + podModulePrefix: this.podModulePrefix(), + options: this.options, + activePackageRules: this.activeRules(), + }; + + return config; + } + + private scriptPriority(pkg: Package) { + switch (pkg.name) { + case 'loader.js': + return 0; + case 'ember-source': + return 10; + default: + return 1000; + } + } + + @Memoize() + private get resolvableExtensionsPattern(): RegExp { + return extensionsPattern(this.resolvableExtensions()); + } + + private impliedAssets( + type: keyof ImplicitAssetPaths, + engine: Engine, + emberENV?: EmberENV + ): (OnDiskAsset | InMemoryAsset)[] { + let result: (OnDiskAsset | InMemoryAsset)[] = this.impliedAddonAssets(type, engine).map( + (sourcePath: string): OnDiskAsset => { + let stats = statSync(sourcePath); + return { + kind: 'on-disk', + relativePath: explicitRelative(this.root, sourcePath), + sourcePath, + mtime: stats.mtimeMs, + size: stats.size, + }; + } + ); + + if (type === 'implicit-scripts') { + result.unshift({ + kind: 'in-memory', + relativePath: '_testing_prefix_.js', + source: `var runningTests=false;`, + }); + + result.unshift({ + kind: 'in-memory', + relativePath: '_ember_env_.js', + source: `window.EmberENV={ ...(window.EmberENV || {}), ...${JSON.stringify(emberENV, null, 2)} };`, + }); + + result.push({ + kind: 'in-memory', + relativePath: '_loader_.js', + source: `loader.makeDefaultExport=false;`, + }); + } + + if (type === 'implicit-test-scripts') { + // this is the traditional test-support-suffix.js + result.push({ + kind: 'in-memory', + relativePath: '_testing_suffix_.js', + source: ` + var runningTests=true; + if (typeof Testem !== 'undefined' && (typeof QUnit !== 'undefined' || typeof Mocha !== 'undefined')) { + Testem.hookIntoTestFramework(); + }`, + }); + + // whether or not anybody was actually using @embroider/macros + // explicitly as an addon, we ensure its test-support file is always + // present. + if (!result.find(s => s.kind === 'on-disk' && s.sourcePath.endsWith('embroider-macros-test-support.js'))) { + result.unshift({ + kind: 'on-disk', + sourcePath: require.resolve('@embroider/macros/src/vendor/embroider-macros-test-support'), + mtime: 0, + size: 0, + relativePath: 'embroider-macros-test-support.js', + }); + } + } + + return result; + } + + private impliedAddonAssets(type: keyof ImplicitAssetPaths, engine: Engine): string[] { + let result: Array = []; + for (let addon of sortBy(Array.from(engine.addons), this.scriptPriority.bind(this))) { + let implicitScripts = addon.meta[type]; + if (implicitScripts) { + let styles = []; + let options = { basedir: addon.root }; + for (let mod of implicitScripts) { + if (type === 'implicit-styles') { + // exclude engines because they will handle their own css importation + if (!addon.isLazyEngine()) { + styles.push(resolve.sync(mod, options)); + } + } else { + result.push(resolve.sync(mod, options)); + } + } + if (styles.length) { + result = [...styles, ...result]; + } + } + } + return result; + } + + // unlike our full config, this one just needs to know how to parse all the + // syntax our app can contain. + @Memoize() + private babelParserConfig(): TransformOptions { + let babel = cloneDeep(this.compatApp.babelConfig()); + + if (!babel.plugins) { + babel.plugins = []; + } + + // Our stage3 code is always allowed to use dynamic import. We may emit it + // ourself when splitting routes. + babel.plugins.push(require.resolve('@babel/plugin-syntax-dynamic-import')); + return babel; + } + + @Memoize() + private babelConfig(resolverConfig: CompatResolverOptions) { + let babel = cloneDeep(this.compatApp.babelConfig()); + + if (!babel.plugins) { + babel.plugins = []; + } + + // Our stage3 code is always allowed to use dynamic import. We may emit it + // ourself when splitting routes. + babel.plugins.push(require.resolve('@babel/plugin-syntax-dynamic-import')); + + // https://github.com/webpack/webpack/issues/12154 + babel.plugins.push(require.resolve('./rename-require-plugin')); + + babel.plugins.push([require.resolve('babel-plugin-ember-template-compilation'), this.etcOptions(resolverConfig)]); + + // this is @embroider/macros configured for full stage3 resolution + babel.plugins.push(...this.compatApp.macrosConfig.babelPluginConfig()); + + let colocationOptions: TemplateColocationPluginOptions = { + appRoot: this.root, + + // This extra weirdness is a compromise in favor of build performance. + // + // 1. When auto-upgrading an addon from v1 to v2, we definitely want to + // run any custom AST transforms in stage1. + // + // 2. In general case, AST transforms are allowed to manipulate Javascript + // scope. This means that running transforms -- even when we're doing + // source-to-source compilation that emits handlebars and not wire + // format -- implies changing .hbs files into .js files. + // + // 3. So stage1 may need to rewrite .hbs to .hbs.js (to avoid colliding + // with an existing co-located .js file). + // + // 4. But stage1 doesn't necessarily want to run babel over the + // corresponding JS file. Most of the time, that's just an + // unnecessarily expensive second parse. (We only run it in stage1 to + // eliminate an addon's custom babel plugins, and many addons don't + // have any.) + // + // 5. Therefore, the work of template-colocation gets defered until here, + // and it may see co-located templates named `.hbs.js` instead of the + // usual `.hbs. + templateExtensions: ['.hbs', '.hbs.js'], + + // All of the above only applies to auto-upgraded packages that were + // authored in v1. V2 packages don't get any of this complexity, they're + // supposed to take care of colocating their own templates explicitly. + packageGuard: true, + }; + babel.plugins.push([templateColocationPluginPath, colocationOptions]); + + babel.plugins.push([ + require.resolve('./babel-plugin-adjust-imports'), + (() => { + let pluginConfig: AdjustImportsOptions = { + appRoot: resolverConfig.appRoot, + }; + return pluginConfig; + })(), + ]); + + // we can use globally shared babel runtime by default + babel.plugins.push([ + require.resolve('@babel/plugin-transform-runtime'), + { absoluteRuntime: __dirname, useESModules: true, regenerator: false }, + ]); + + const portable = makePortable(babel, { basedir: this.root }, this.portableHints); + addCachablePlugin(portable.config); + return portable; + } + + private insertEmberApp( + asset: ParsedEmberAsset, + appFiles: Engine[], + prepared: Map, + emberENV: EmberENV + ) { + let html = asset.html; + + if (this.fastbootConfig) { + // ignore scripts like ember-cli-livereload.js which are not really associated with + // "the app". + let ignoreScripts = html.dom.window.document.querySelectorAll('script'); + ignoreScripts.forEach(script => { + script.setAttribute('data-fastboot-ignore', ''); + }); + } + + // our tests entrypoint already includes a correct module dependency on the + // app, so we only insert the app when we're not inserting tests + if (!asset.fileAsset.includeTests) { + let appJS = this.topAppJSAsset(appFiles, prepared); + html.insertScriptTag(html.javascript, appJS.relativePath, { type: 'module' }); + } + + if (this.fastbootConfig) { + // any extra fastboot app files get inserted into our html.javascript + // section, after the app has been inserted. + for (let script of this.fastbootConfig.extraAppFiles) { + html.insertScriptTag(html.javascript, script, { tag: 'fastboot-script' }); + } + } + + html.insertStyleLink(html.styles, `assets/${this.appPackage.name}.css`); + + const parentEngine = appFiles.find(e => !e.parent) as Engine; + let vendorJS = this.implicitScriptsAsset(prepared, parentEngine, emberENV); + if (vendorJS) { + html.insertScriptTag(html.implicitScripts, vendorJS.relativePath); + } + + if (this.fastbootConfig) { + // any extra fastboot vendor files get inserted into our + // html.implicitScripts section, after the regular implicit script + // (vendor.js) have been inserted. + for (let script of this.fastbootConfig.extraVendorFiles) { + html.insertScriptTag(html.implicitScripts, script, { tag: 'fastboot-script' }); + } + } + + let implicitStyles = this.implicitStylesAsset(prepared, parentEngine); + if (implicitStyles) { + html.insertStyleLink(html.implicitStyles, implicitStyles.relativePath); + } + + if (!asset.fileAsset.includeTests) { + return; + } + + // Test-related assets happen below this point + + let testJS = this.testJSEntrypoint(appFiles, prepared); + html.insertScriptTag(html.testJavascript, testJS.relativePath, { type: 'module' }); + + let implicitTestScriptsAsset = this.implicitTestScriptsAsset(prepared, parentEngine); + if (implicitTestScriptsAsset) { + html.insertScriptTag(html.implicitTestScripts, implicitTestScriptsAsset.relativePath); + } + + let implicitTestStylesAsset = this.implicitTestStylesAsset(prepared, parentEngine); + if (implicitTestStylesAsset) { + html.insertStyleLink(html.implicitTestStyles, implicitTestStylesAsset.relativePath); + } + } + + private implicitScriptsAsset( + prepared: Map, + application: Engine, + emberENV: EmberENV + ): InternalAsset | undefined { + let asset = prepared.get('assets/vendor.js'); + if (!asset) { + let implicitScripts = this.impliedAssets('implicit-scripts', application, emberENV); + if (implicitScripts.length > 0) { + asset = new ConcatenatedAsset('assets/vendor.js', implicitScripts, this.resolvableExtensionsPattern); + prepared.set(asset.relativePath, asset); + } + } + return asset; + } + + private implicitStylesAsset(prepared: Map, application: Engine): InternalAsset | undefined { + let asset = prepared.get('assets/vendor.css'); + if (!asset) { + let implicitStyles = this.impliedAssets('implicit-styles', application); + if (implicitStyles.length > 0) { + // we reverse because we want the synthetic vendor style at the top + asset = new ConcatenatedAsset('assets/vendor.css', implicitStyles.reverse(), this.resolvableExtensionsPattern); + prepared.set(asset.relativePath, asset); + } + } + return asset; + } + + private implicitTestScriptsAsset( + prepared: Map, + application: Engine + ): InternalAsset | undefined { + let testSupportJS = prepared.get('assets/test-support.js'); + if (!testSupportJS) { + let implicitTestScripts = this.impliedAssets('implicit-test-scripts', application); + if (implicitTestScripts.length > 0) { + testSupportJS = new ConcatenatedAsset( + 'assets/test-support.js', + implicitTestScripts, + this.resolvableExtensionsPattern + ); + prepared.set(testSupportJS.relativePath, testSupportJS); + } + } + return testSupportJS; + } + + private implicitTestStylesAsset( + prepared: Map, + application: Engine + ): InternalAsset | undefined { + let asset = prepared.get('assets/test-support.css'); + if (!asset) { + let implicitTestStyles = this.impliedAssets('implicit-test-styles', application); + if (implicitTestStyles.length > 0) { + asset = new ConcatenatedAsset('assets/test-support.css', implicitTestStyles, this.resolvableExtensionsPattern); + prepared.set(asset.relativePath, asset); + } + } + return asset; + } + + // recurse to find all active addons that don't cross an engine boundary. + // Inner engines themselves will be returned, but not those engines' children. + // The output set's insertion order is the proper ember-cli compatible + // ordering of the addons. + private findActiveAddons(pkg: Package, engine: EngineSummary, isChild = false): void { + for (let child of this.activeAddonChildren(pkg)) { + if (!child.isEngine()) { + this.findActiveAddons(child, engine, true); + } + engine.addons.add(child); + } + // ensure addons are applied in the correct order, if set (via @embroider/compat/v1-addon) + if (!isChild) { + engine.addons = new Set( + [...engine.addons].sort((a, b) => { + return (a.meta['order-index'] || 0) - (b.meta['order-index'] || 0); + }) + ); + } + } + + private partitionEngines(appJSPath: string): EngineSummary[] { + let queue: EngineSummary[] = [ + { + package: this.appPackage, + addons: new Set(), + parent: undefined, + sourcePath: appJSPath, + destPath: this.root, + modulePrefix: this.modulePrefix(), + appRelativePath: '.', + }, + ]; + let done: EngineSummary[] = []; + let seenEngines: Set = new Set(); + while (true) { + let current = queue.shift(); + if (!current) { + break; + } + this.findActiveAddons(current.package, current); + for (let addon of current.addons) { + if (addon.isEngine() && !seenEngines.has(addon)) { + seenEngines.add(addon); + queue.push({ + package: addon, + addons: new Set(), + parent: current, + sourcePath: mangledEngineRoot(addon), + destPath: addon.root, + modulePrefix: addon.name, + appRelativePath: explicitRelative(this.root, addon.root), + }); + } + } + done.push(current); + } + return done; + } + + @Memoize() + private get activeFastboot() { + return this.activeAddonChildren(this.appPackage).find(a => a.name === 'ember-cli-fastboot'); + } + + @Memoize() + private get fastbootConfig(): + | { packageJSON: PackageInfo; extraAppFiles: string[]; extraVendorFiles: string[] } + | undefined { + if (this.activeFastboot) { + // this is relying on work done in stage1 by @embroider/compat/src/compat-adapters/ember-cli-fastboot.ts + let packageJSON = readJSONSync(join(this.activeFastboot.root, '_fastboot_', 'package.json')); + let { extraAppFiles, extraVendorFiles } = packageJSON['embroider-fastboot']; + delete packageJSON['embroider-fastboot']; + extraVendorFiles.push('assets/embroider_macros_fastboot_init.js'); + return { packageJSON, extraAppFiles, extraVendorFiles }; + } + } + + private appDiffers: { differ: AppDiffer; engine: EngineSummary }[] | undefined; + + private updateAppJS(inputPaths: OutputPaths): Engine[] { + let appJSPath = inputPaths.appJS; + if (!this.appDiffers) { + let engines = this.partitionEngines(appJSPath); + this.appDiffers = engines.map(engine => { + let differ: AppDiffer; + if (this.activeFastboot) { + differ = new AppDiffer( + engine.destPath, + engine.sourcePath, + [...engine.addons], + true, + this.fastbootJSSrcDir(), + this.babelParserConfig() + ); + } else { + differ = new AppDiffer(engine.destPath, engine.sourcePath, [...engine.addons]); + } + return { + differ, + engine, + }; + }); + } + // this is in reverse order because we need deeper engines to update before + // their parents, because they aren't really valid packages until they + // update, and their parents will go looking for their own `app-js` content. + this.appDiffers + .slice() + .reverse() + .forEach(a => a.differ.update()); + return this.appDiffers.map(a => { + return { + ...a.engine, + appFiles: new AppFiles(a.differ, this.resolvableExtensionsPattern, this.podModulePrefix()), + }; + }); + } + + private prepareAsset(asset: Asset, appFiles: Engine[], prepared: Map, emberENV: EmberENV) { + if (asset.kind === 'ember') { + let prior = this.assets.get(asset.relativePath); + let parsed: ParsedEmberAsset; + if (prior && prior.kind === 'built-ember' && prior.parsedAsset.validFor(asset)) { + // we can reuse the parsed html + parsed = prior.parsedAsset; + parsed.html.clear(); + } else { + parsed = new ParsedEmberAsset(asset); + } + this.insertEmberApp(parsed, appFiles, prepared, emberENV); + prepared.set(asset.relativePath, new BuiltEmberAsset(parsed)); + } else { + prepared.set(asset.relativePath, asset); + } + } + + private prepareAssets(requestedAssets: Asset[], appFiles: Engine[], emberENV: EmberENV): Map { + let prepared: Map = new Map(); + for (let asset of requestedAssets) { + this.prepareAsset(asset, appFiles, prepared, emberENV); + } + return prepared; + } + + private assetIsValid(asset: InternalAsset, prior: InternalAsset | undefined): boolean { + if (!prior) { + return false; + } + switch (asset.kind) { + case 'on-disk': + return prior.kind === 'on-disk' && prior.size === asset.size && prior.mtime === asset.mtime; + case 'in-memory': + return prior.kind === 'in-memory' && stringOrBufferEqual(prior.source, asset.source); + case 'built-ember': + return prior.kind === 'built-ember' && prior.source === asset.source; + case 'concatenated-asset': + return ( + prior.kind === 'concatenated-asset' && + prior.sources.length === asset.sources.length && + prior.sources.every((priorFile, index) => { + let newFile = asset.sources[index]; + return this.assetIsValid(newFile, priorFile); + }) + ); + } + } + + private updateOnDiskAsset(asset: OnDiskAsset) { + let destination = join(this.root, asset.relativePath); + ensureDirSync(dirname(destination)); + copySync(asset.sourcePath, destination, { dereference: true }); + } + + private updateInMemoryAsset(asset: InMemoryAsset) { + let destination = join(this.root, asset.relativePath); + ensureDirSync(dirname(destination)); + writeFileSync(destination, asset.source, 'utf8'); + } + + private updateBuiltEmberAsset(asset: BuiltEmberAsset) { + let destination = join(this.root, asset.relativePath); + ensureDirSync(dirname(destination)); + writeFileSync(destination, asset.source, 'utf8'); + } + + private async updateConcatenatedAsset(asset: ConcatenatedAsset) { + let concat = new SourceMapConcat({ + outputFile: join(this.root, asset.relativePath), + mapCommentType: asset.relativePath.endsWith('.js') ? 'line' : 'block', + baseDir: this.root, + }); + if (process.env.EMBROIDER_CONCAT_STATS) { + let MeasureConcat = (await import('@embroider/core/src/measure-concat')).default; + concat = new MeasureConcat(asset.relativePath, concat, this.root); + } + for (let source of asset.sources) { + switch (source.kind) { + case 'on-disk': + concat.addFile(explicitRelative(this.root, source.sourcePath)); + break; + case 'in-memory': + if (typeof source.source !== 'string') { + throw new Error(`attempted to concatenated a Buffer-backed in-memory asset`); + } + concat.addSpace(source.source); + break; + default: + assertNever(source); + } + } + await concat.end(); + } + + private async updateAssets(requestedAssets: Asset[], appFiles: Engine[], emberENV: EmberENV) { + let assets = this.prepareAssets(requestedAssets, appFiles, emberENV); + for (let asset of assets.values()) { + if (this.assetIsValid(asset, this.assets.get(asset.relativePath))) { + continue; + } + debug('rebuilding %s', asset.relativePath); + switch (asset.kind) { + case 'on-disk': + this.updateOnDiskAsset(asset); + break; + case 'in-memory': + this.updateInMemoryAsset(asset); + break; + case 'built-ember': + this.updateBuiltEmberAsset(asset); + break; + case 'concatenated-asset': + await this.updateConcatenatedAsset(asset); + break; + default: + assertNever(asset); + } + } + for (let oldAsset of this.assets.values()) { + if (!assets.has(oldAsset.relativePath)) { + unlinkSync(join(this.root, oldAsset.relativePath)); + } + } + this.assets = assets; + return [...assets.values()]; + } + + private gatherAssets(inputPaths: OutputPaths): Asset[] { + // first gather all the assets out of addons + let assets: Asset[] = []; + for (let pkg of this.allActiveAddons) { + if (pkg.meta['public-assets']) { + for (let [filename, appRelativeURL] of Object.entries(pkg.meta['public-assets'] || {})) { + let sourcePath = resolvePath(pkg.root, filename); + let stats = statSync(sourcePath); + assets.push({ + kind: 'on-disk', + sourcePath, + relativePath: appRelativeURL, + mtime: stats.mtimeMs, + size: stats.size, + }); + } + } + } + + if (this.activeFastboot) { + const source = ` + (function(){ + var key = '_embroider_macros_runtime_config'; + if (!window[key]){ window[key] = [];} + window[key].push(function(m) { + m.setGlobalConfig('fastboot', Object.assign({}, m.getGlobalConfig().fastboot, { isRunning: true })); + }); + }())`; + assets.push({ + kind: 'in-memory', + source, + relativePath: 'assets/embroider_macros_fastboot_init.js', + }); + } + + // and finally tack on the ones from our app itself + return assets.concat(this.extractAssets(inputPaths)); + } + + async build(inputPaths: OutputPaths) { + // on the first build, we lock down the macros config. on subsequent builds, + // this doesn't do anything anyway because it's idempotent. + this.compatApp.macrosConfig.finalize(); + + let appFiles = this.updateAppJS(inputPaths); + let emberENV = this.configTree.readConfig().EmberENV; + let assets = this.gatherAssets(inputPaths); + + let finalAssets = await this.updateAssets(assets, appFiles, emberENV); + + let assetPaths = assets.map(asset => asset.relativePath); + + if (this.activeFastboot) { + // when using fastboot, our own package.json needs to be in the output so fastboot can read it. + assetPaths.push('package.json'); + } + + for (let asset of finalAssets) { + // our concatenated assets all have map files that ride along. Here we're + // telling the final stage packager to be sure and serve the map files + // too. + if (asset.kind === 'concatenated-asset') { + assetPaths.push(asset.sourcemapPath); + } + } + + let meta: AppMeta = { + type: 'app', + version: 2, + assets: assetPaths, + babel: { + filename: '_babel_config_.js', + isParallelSafe: true, // TODO + majorVersion: this.compatApp.babelMajorVersion(), + fileFilter: '_babel_filter_.js', + }, + 'root-url': this.rootURL(), + }; + + // all compat apps are auto-upgraded, there's no v2 app format here + meta['auto-upgraded'] = true; + + let pkg = this.combinePackageJSON(meta); + writeFileSync(join(this.root, 'package.json'), JSON.stringify(pkg, null, 2), 'utf8'); + + let resolverConfig = this.resolverConfig(appFiles); + this.addResolverConfig(resolverConfig); + let babelConfig = this.babelConfig(resolverConfig); + this.addBabelConfig(babelConfig); + } + + private combinePackageJSON(meta: AppMeta): object { + let pkgLayers: any[] = [this.appPackage.packageJSON]; + let fastbootConfig = this.fastbootConfig; + if (fastbootConfig) { + // fastboot-specific package.json output is allowed to add to our original package.json + pkgLayers.push(fastbootConfig.packageJSON); + } + // but our own new v2 app metadata takes precedence over both + pkgLayers.push({ keywords: ['ember-addon'], 'ember-addon': meta }); + return combinePackageJSON(...pkgLayers); + } + + private etcOptions(resolverConfig: CompatResolverOptions): EtcOptions { + let transforms = this.compatApp.htmlbarsPlugins; + + let { plugins: macroPlugins, setConfig } = MacrosConfig.transforms(); + setConfig(this.compatApp.macrosConfig); + for (let macroPlugin of macroPlugins) { + transforms.push(macroPlugin as any); + } + + if ( + this.options.staticComponents || + this.options.staticHelpers || + this.options.staticModifiers || + (globalThis as any).embroider_audit + ) { + let opts: ResolverTransformOptions = { + appRoot: resolverConfig.appRoot, + }; + transforms.push([require.resolve('./resolver-transform'), opts]); + } + + return { + transforms, + compilerPath: resolve.sync(this.templateCompilerPath(), { basedir: this.root }), + enableLegacyModules: ['ember-cli-htmlbars', 'ember-cli-htmlbars-inline-precompile', 'htmlbars-inline-precompile'], + }; + } + + @Memoize() + private get portableHints(): PortableHint[] { + return this.options.pluginHints.map(hint => { + let cursor = join(this.appPackage.root, 'package.json'); + for (let i = 0; i < hint.resolve.length; i++) { + let target = hint.resolve[i]; + if (i < hint.resolve.length - 1) { + target = join(target, 'package.json'); + } + cursor = resolve.sync(target, { basedir: dirname(cursor) }); + } + + return { + requireFile: cursor, + useMethod: hint.useMethod, + packageVersion: maybeNodeModuleVersion(cursor), + }; + }); + } + + private addBabelConfig(pconfig: { config: TransformOptions; isParallelSafe: boolean }) { + if (!pconfig.isParallelSafe) { + warn('Your build is slower because some babel plugins are non-serializable'); + } + writeFileSync( + join(this.root, '_babel_config_.js'), + `module.exports = ${JSON.stringify(pconfig.config, null, 2)}`, + 'utf8' + ); + writeFileSync( + join(this.root, '_babel_filter_.js'), + babelFilterTemplate({ skipBabel: this.options.skipBabel, appRoot: this.root }), + 'utf8' + ); + } + + private addResolverConfig(config: CompatResolverOptions) { + outputJSONSync(join(this.root, '.embroider', 'resolver.json'), config); + } + + private shouldSplitRoute(routeName: string) { + return ( + !this.options.splitAtRoutes || + this.options.splitAtRoutes.find(pattern => { + if (typeof pattern === 'string') { + return pattern === routeName; + } else { + return pattern.test(routeName); + } + }) + ); + } + + private splitRoute( + routeName: string, + files: RouteFiles, + addToParent: (routeName: string, filename: string) => void, + addLazyBundle: (routeNames: string[], files: string[]) => void + ) { + let shouldSplit = routeName && this.shouldSplitRoute(routeName); + let ownFiles = []; + let ownNames = new Set() as Set; + + if (files.template) { + if (shouldSplit) { + ownFiles.push(files.template); + ownNames.add(routeName); + } else { + addToParent(routeName, files.template); + } + } + + if (files.controller) { + if (shouldSplit) { + ownFiles.push(files.controller); + ownNames.add(routeName); + } else { + addToParent(routeName, files.controller); + } + } + + if (files.route) { + if (shouldSplit) { + ownFiles.push(files.route); + ownNames.add(routeName); + } else { + addToParent(routeName, files.route); + } + } + + for (let [childName, childFiles] of files.children) { + this.splitRoute( + `${routeName}.${childName}`, + childFiles, + + (childRouteName: string, childFile: string) => { + // this is our child calling "addToParent" + if (shouldSplit) { + ownFiles.push(childFile); + ownNames.add(childRouteName); + } else { + addToParent(childRouteName, childFile); + } + }, + (routeNames: string[], files: string[]) => { + addLazyBundle(routeNames, files); + } + ); + } + + if (ownFiles.length > 0) { + addLazyBundle([...ownNames], ownFiles); + } + } + + private topAppJSAsset(engines: Engine[], prepared: Map): InternalAsset { + let [app, ...childEngines] = engines; + let relativePath = `assets/${this.appPackage.name}.js`; + return this.appJSAsset(relativePath, app, childEngines, prepared, { + autoRun: this.compatApp.autoRun, + appBoot: !this.compatApp.autoRun ? this.compatApp.appBoot.readAppBoot() : '', + mainModule: explicitRelative(dirname(relativePath), 'app'), + appConfig: this.configTree.readConfig().APP, + }); + } + + @Memoize() + private get staticAppPathsPattern(): RegExp | undefined { + if (this.options.staticAppPaths.length > 0) { + return new RegExp( + '^(?:' + this.options.staticAppPaths.map(staticAppPath => escapeRegExp(staticAppPath)).join('|') + ')(?:$|/)' + ); + } + } + + private requiredOtherFiles(appFiles: AppFiles): readonly string[] { + let pattern = this.staticAppPathsPattern; + if (pattern) { + return appFiles.otherAppFiles.filter(f => { + return !pattern!.test(f); + }); + } else { + return appFiles.otherAppFiles; + } + } + + private appJSAsset( + relativePath: string, + engine: Engine, + childEngines: Engine[], + prepared: Map, + entryParams?: Partial[0]> + ): InternalAsset { + let { appFiles } = engine; + let cached = prepared.get(relativePath); + if (cached) { + return cached; + } + + let eagerModules = []; + + let requiredAppFiles = [this.requiredOtherFiles(appFiles)]; + if (!this.options.staticComponents) { + requiredAppFiles.push(appFiles.components); + } + if (!this.options.staticHelpers) { + requiredAppFiles.push(appFiles.helpers); + } + if (!this.options.staticModifiers) { + requiredAppFiles.push(appFiles.modifiers); + } + + let styles = []; + // only import styles from engines with a parent (this excludeds the parent application) as their styles + // will be inserted via a direct tag. + if (engine.parent && engine.package.isLazyEngine()) { + let implicitStyles = this.impliedAssets('implicit-styles', engine); + for (let style of implicitStyles) { + styles.push({ + path: explicitRelative('assets/_engine_', style.relativePath), + }); + } + + let engineMeta = engine.package.meta as AddonMeta; + if (engineMeta && engineMeta['implicit-styles']) { + for (let style of engineMeta['implicit-styles']) { + styles.push({ + path: explicitRelative(dirname(relativePath), join(engine.appRelativePath, style)), + }); + } + } + } + + let lazyEngines: { names: string[]; path: string }[] = []; + for (let childEngine of childEngines) { + let asset = this.appJSAsset( + `assets/_engine_/${encodeURIComponent(childEngine.package.name)}.js`, + childEngine, + [], + prepared + ); + if (childEngine.package.isLazyEngine()) { + lazyEngines.push({ + names: [childEngine.package.name], + path: explicitRelative(dirname(relativePath), asset.relativePath), + }); + } else { + eagerModules.push(explicitRelative(dirname(relativePath), asset.relativePath)); + } + } + let lazyRoutes: { names: string[]; path: string }[] = []; + for (let [routeName, routeFiles] of appFiles.routeFiles.children) { + this.splitRoute( + routeName, + routeFiles, + (_: string, filename: string) => { + requiredAppFiles.push([filename]); + }, + (routeNames: string[], files: string[]) => { + let routeEntrypoint = `assets/_route_/${encodeURIComponent(routeNames[0])}.js`; + if (!prepared.has(routeEntrypoint)) { + prepared.set(routeEntrypoint, this.routeEntrypoint(engine, routeEntrypoint, files)); + } + lazyRoutes.push({ + names: routeNames, + path: this.importPaths(engine, routeEntrypoint).buildtime, + }); + } + ); + } + + let [fastboot, nonFastboot] = partition(excludeDotFiles(flatten(requiredAppFiles)), file => + appFiles.isFastbootOnly.get(file) + ); + let amdModules = nonFastboot.map(file => this.importPaths(engine, file)); + let fastbootOnlyAmdModules = fastboot.map(file => this.importPaths(engine, file)); + + // this is a backward-compatibility feature: addons can force inclusion of + // modules. + this.gatherImplicitModules('implicit-modules', engine, amdModules); + + let params = { amdModules, fastbootOnlyAmdModules, lazyRoutes, lazyEngines, eagerModules, styles }; + if (entryParams) { + Object.assign(params, entryParams); + } + + let source = entryTemplate(params); + + let asset: InternalAsset = { + kind: 'in-memory', + source, + relativePath, + }; + prepared.set(relativePath, asset); + return asset; + } + + private importPaths(engine: Engine, engineRelativePath: string) { + let noHBS = engineRelativePath.replace(this.resolvableExtensionsPattern, '').replace(/\.hbs$/, ''); + return { + runtime: `${engine.modulePrefix}/${noHBS}`, + buildtime: posix.join(engine.package.name, engineRelativePath), + }; + } + + private routeEntrypoint(engine: Engine, relativePath: string, files: string[]) { + let [fastboot, nonFastboot] = partition(files, file => engine.appFiles.isFastbootOnly.get(file)); + + let asset: InternalAsset = { + kind: 'in-memory', + source: routeEntryTemplate({ + files: nonFastboot.map(f => this.importPaths(engine, f)), + fastbootOnlyFiles: fastboot.map(f => this.importPaths(engine, f)), + }), + relativePath, + }; + return asset; + } + + private testJSEntrypoint(engines: Engine[], prepared: Map): InternalAsset { + let asset = prepared.get(`assets/test.js`); + if (asset) { + return asset; + } + + // We're only building tests from the first engine (the app). This is the + // normal thing to do -- tests from engines don't automatically roll up into + // the app. + let engine = engines[0]; + + const myName = 'assets/test.js'; + + // tests necessarily also include the app. This is where we account for + // that. The classic solution was to always include the app's separate + // script tag in the tests HTML, but that isn't as easy for final stage + // packagers to understand. It's better to express it here as a direct + // module dependency. + let eagerModules: string[] = [ + explicitRelative(dirname(myName), this.topAppJSAsset(engines, prepared).relativePath), + ]; + + let amdModules: { runtime: string; buildtime: string }[] = []; + // this is a backward-compatibility feature: addons can force inclusion of + // test support modules. + this.gatherImplicitModules('implicit-test-modules', engine, amdModules); + + let { appFiles } = engine; + for (let relativePath of appFiles.tests) { + amdModules.push(this.importPaths(engine, relativePath)); + } + + let source = entryTemplate({ + amdModules, + eagerModules, + testSuffix: true, + }); + + asset = { + kind: 'in-memory', + source, + relativePath: myName, + }; + prepared.set(asset.relativePath, asset); + return asset; + } + + private gatherImplicitModules( + section: 'implicit-modules' | 'implicit-test-modules', + engine: Engine, + lazyModules: { runtime: string; buildtime: string }[] + ) { + for (let addon of engine.addons) { + let implicitModules = addon.meta[section]; + if (implicitModules) { + let renamedModules = inverseRenamedModules(addon.meta, this.resolvableExtensionsPattern); + for (let name of implicitModules) { + let packageName = addon.name; + + if (addon.isV2Addon()) { + let renamedMeta = addon.meta['renamed-packages']; + if (renamedMeta) { + Object.entries(renamedMeta).forEach(([key, value]) => { + if (value === addon!.name) { + packageName = key; + } + }); + } + } + + let runtime = join(packageName, name).replace(this.resolvableExtensionsPattern, ''); + let runtimeRenameLookup = runtime.split('\\').join('/'); + if (renamedModules && renamedModules[runtimeRenameLookup]) { + runtime = renamedModules[runtimeRenameLookup]; + } + runtime = runtime.split(sep).join('/'); + lazyModules.push({ + runtime, + buildtime: posix.join(packageName, name), + }); + } + } + } + } +} + +function maybeReplace(dom: JSDOM, element: Element | undefined): Node | undefined { + if (element) { + return definitelyReplace(dom, element); + } +} + +function definitelyReplace(dom: JSDOM, element: Element): Node { + let placeholder = dom.window.document.createTextNode(''); + element.replaceWith(placeholder); + return placeholder; +} + +function defaultAddonPackageRules(): PackageRules[] { + return readdirSync(join(__dirname, 'addon-dependency-rules')) + .map(filename => { + if (filename.endsWith('.js')) { + // eslint-disable-next-line @typescript-eslint/no-require-imports + return require(join(__dirname, 'addon-dependency-rules', filename)).default; + } + }) + .filter(Boolean) + .reduce((a, b) => a.concat(b), []); +} + +const entryTemplate = jsHandlebarsCompile(` +import { importSync as i, macroCondition, getGlobalConfig } from '@embroider/macros'; +let w = window; +let d = w.define; + +{{#if styles}} + if (macroCondition(!getGlobalConfig().fastboot?.isRunning)) { + {{#each styles as |stylePath| ~}} + i("{{js-string-escape stylePath.path}}"); + {{/each}} + } +{{/if}} + +{{#each amdModules as |amdModule| ~}} + d("{{js-string-escape amdModule.runtime}}", function(){ return i("{{js-string-escape amdModule.buildtime}}");}); +{{/each}} + +{{#if fastbootOnlyAmdModules}} + if (macroCondition(getGlobalConfig().fastboot?.isRunning)) { + {{#each fastbootOnlyAmdModules as |amdModule| ~}} + d("{{js-string-escape amdModule.runtime}}", function(){ return i("{{js-string-escape amdModule.buildtime}}");}); + {{/each}} + } +{{/if}} + +{{#each eagerModules as |eagerModule| ~}} + i("{{js-string-escape eagerModule}}"); +{{/each}} + +{{#if lazyRoutes}} +w._embroiderRouteBundles_ = [ + {{#each lazyRoutes as |route|}} + { + names: {{{json-stringify route.names}}}, + load: function() { + return import("{{js-string-escape route.path}}"); + } + }, + {{/each}} +] +{{/if}} + +{{#if lazyEngines}} +w._embroiderEngineBundles_ = [ + {{#each lazyEngines as |engine|}} + { + names: {{{json-stringify engine.names}}}, + load: function() { + return import("{{js-string-escape engine.path}}"); + } + }, + {{/each}} +] +{{/if}} + +{{#if autoRun ~}} +if (!runningTests) { + i("{{js-string-escape mainModule}}").default.create({{{json-stringify appConfig}}}); +} +{{else if appBoot ~}} + {{{ appBoot }}} +{{/if}} + +{{#if testSuffix ~}} + {{!- TODO: both of these suffixes should get dynamically generated so they incorporate + any content-for added by addons. -}} + + + {{!- this is the traditional tests-suffix.js -}} + i('../tests/test-helper'); + EmberENV.TESTS_FILE_LOADED = true; +{{/if}} +`) as (params: { + amdModules: { runtime: string; buildtime: string }[]; + fastbootOnlyAmdModules?: { runtime: string; buildtime: string }[]; + eagerModules?: string[]; + autoRun?: boolean; + appBoot?: string; + mainModule?: string; + appConfig?: unknown; + testSuffix?: boolean; + lazyRoutes?: { names: string[]; path: string }[]; + lazyEngines?: { names: string[]; path: string }[]; + styles?: { path: string }[]; +}) => string; + +const routeEntryTemplate = jsHandlebarsCompile(` +import { importSync as i } from '@embroider/macros'; +let d = window.define; +{{#each files as |amdModule| ~}} +d("{{js-string-escape amdModule.runtime}}", function(){ return i("{{js-string-escape amdModule.buildtime}}");}); +{{/each}} +{{#if fastbootOnlyFiles}} + import { macroCondition, getGlobalConfig } from '@embroider/macros'; + if (macroCondition(getGlobalConfig().fastboot?.isRunning)) { + {{#each fastbootOnlyFiles as |amdModule| ~}} + d("{{js-string-escape amdModule.runtime}}", function(){ return i("{{js-string-escape amdModule.buildtime}}");}); + {{/each}} + } +{{/if}} +`) as (params: { + files: { runtime: string; buildtime: string }[]; + fastbootOnlyFiles: { runtime: string; buildtime: string }[]; +}) => string; + +function stringOrBufferEqual(a: string | Buffer, b: string | Buffer): boolean { + if (typeof a === 'string' && typeof b === 'string') { + return a === b; + } + if (a instanceof Buffer && b instanceof Buffer) { + return Buffer.compare(a, b) === 0; + } + return false; +} + +const babelFilterTemplate = jsHandlebarsCompile(` +const { babelFilter } = require(${JSON.stringify(require.resolve('@embroider/core'))}); +module.exports = babelFilter({{{json-stringify skipBabel}}}, "{{{js-string-escape appRoot}}}"); +`) as (params: { skipBabel: Options['skipBabel']; appRoot: string }) => string; + +// meta['renamed-modules'] has mapping from classic filename to real filename. +// This takes that and converts it to the inverst mapping from real import path +// to classic import path. +function inverseRenamedModules(meta: AddonPackage['meta'], extensions: RegExp) { + let renamed = meta['renamed-modules']; + if (renamed) { + let inverted = {} as { [name: string]: string }; + for (let [classic, real] of Object.entries(renamed)) { + inverted[real.replace(extensions, '')] = classic.replace(extensions, ''); + } + return inverted; + } +} + +function combinePackageJSON(...layers: object[]) { + function custom(objValue: any, srcValue: any, key: string, _object: any, _source: any, stack: { size: number }) { + if (key === 'keywords' && stack.size === 0) { + if (Array.isArray(objValue)) { + return objValue.concat(srcValue); + } + } + } + return mergeWith({}, ...layers, custom); +} + +function addCachablePlugin(babelConfig: TransformOptions) { + if (Array.isArray(babelConfig.plugins) && babelConfig.plugins.length > 0) { + const plugins = Object.create(null); + plugins[cacheBustingPluginPath] = cacheBustingPluginVersion; + + for (const plugin of babelConfig.plugins) { + let absolutePathToPlugin: string; + if (Array.isArray(plugin) && typeof plugin[0] === 'string') { + absolutePathToPlugin = plugin[0] as string; + } else if (typeof plugin === 'string') { + absolutePathToPlugin = plugin; + } else { + throw new Error(`[Embroider] a babel plugin without an absolute path was from: ${plugin}`); + } + + plugins[absolutePathToPlugin] = maybeNodeModuleVersion(absolutePathToPlugin); + } + + babelConfig.plugins.push([ + cacheBustingPluginPath, + { + plugins, + }, + ]); + } +} + +function excludeDotFiles(files: string[]) { + return files.filter(file => !file.startsWith('.') && !file.includes('/.')); +} + +interface TreeNames { + appJS: BroccoliNode; + htmlTree: BroccoliNode; + publicTree: BroccoliNode | undefined; + configTree: BroccoliNode; +} + +type EmberENV = unknown; + +type InternalAsset = OnDiskAsset | InMemoryAsset | BuiltEmberAsset | ConcatenatedAsset; + +class ParsedEmberAsset { + kind: 'parsed-ember' = 'parsed-ember'; + relativePath: string; + fileAsset: EmberAsset; + html: PreparedEmberHTML; + + constructor(asset: EmberAsset) { + this.fileAsset = asset; + this.html = new PreparedEmberHTML(asset); + this.relativePath = asset.relativePath; + } + + validFor(other: EmberAsset) { + return this.fileAsset.mtime === other.mtime && this.fileAsset.size === other.size; + } +} + +class BuiltEmberAsset { + kind: 'built-ember' = 'built-ember'; + relativePath: string; + parsedAsset: ParsedEmberAsset; + source: string; + + constructor(asset: ParsedEmberAsset) { + this.parsedAsset = asset; + this.source = asset.html.dom.serialize(); + this.relativePath = asset.relativePath; + } +} + +class ConcatenatedAsset { + kind: 'concatenated-asset' = 'concatenated-asset'; + constructor( + public relativePath: string, + public sources: (OnDiskAsset | InMemoryAsset)[], + private resolvableExtensions: RegExp + ) {} + get sourcemapPath() { + return this.relativePath.replace(this.resolvableExtensions, '') + '.map'; + } +} diff --git a/packages/compat/src/compat-app.ts b/packages/compat/src/compat-app.ts index f52f521f5..2028d370c 100644 --- a/packages/compat/src/compat-app.ts +++ b/packages/compat/src/compat-app.ts @@ -1,50 +1,6 @@ import { Node as BroccoliNode } from 'broccoli-node-api'; -import { - PackageCache, - OutputPaths, - Asset, - EmberAsset, - AddonPackage, - Engine, - WaitForTrees, - AppMeta, - explicitRelative, - extensionsPattern, - TemplateColocationPluginOptions, - debug, - warn, - jsHandlebarsCompile, - templateColocationPluginPath, - cacheBustingPluginVersion, - cacheBustingPluginPath, - Stage, -} from '@embroider/core'; -import walkSync from 'walk-sync'; -import { resolve as resolvePath, posix } from 'path'; -import { JSDOM } from 'jsdom'; +import { PackageCache, WaitForTrees, Stage } from '@embroider/core'; import Options, { optionsWithDefaults } from './options'; -import { CompatResolverOptions } from './resolver-transform'; -import { activePackageRules, PackageRules } from './dependency-rules'; -import flatMap from 'lodash/flatMap'; -import sortBy from 'lodash/sortBy'; -import flatten from 'lodash/flatten'; -import partition from 'lodash/partition'; -import mergeWith from 'lodash/mergeWith'; -import cloneDeep from 'lodash/cloneDeep'; -import { sync as resolveSync } from 'resolve'; -import bind from 'bind-decorator'; -import { outputJSONSync, readJSONSync, statSync, unlinkSync, writeFileSync } from 'fs-extra'; -import type { Options as EtcOptions } from 'babel-plugin-ember-template-compilation'; -import type { Options as ResolverTransformOptions } from './resolver-transform'; -import type { Options as AdjustImportsOptions } from './babel-plugin-adjust-imports'; -import { PreparedEmberHTML } from '@embroider/core/src/ember-html'; -import { InMemoryAsset, OnDiskAsset, ImplicitAssetPaths } from '@embroider/core/src/asset'; -import { makePortable } from '@embroider/core/src/portable-babel-config'; -import { AppFiles, EngineSummary, RouteFiles } from '@embroider/core/src/app-files'; -import { mangledEngineRoot } from '@embroider/core/src/engine-mangler'; -import { PortableHint, maybeNodeModuleVersion } from '@embroider/core/src/portable'; -import AppDiffer from '@embroider/core/src/app-differ'; -import assertNever from 'assert-never'; import { Memoize } from 'typescript-memoize'; import { sync as pkgUpSync } from 'pkg-up'; import { join, dirname, isAbsolute, sep } from 'path'; @@ -54,14 +10,7 @@ import { WatchedDir } from 'broccoli-source'; import resolve from 'resolve'; import { V1Config, WriteV1Config } from './v1-config'; import { WriteV1AppBoot, ReadV1AppBoot } from './v1-appboot'; -import { - AddonMeta, - Package, - EmberAppInstance, - OutputFileToInputFileMap, - PackageInfo, - AddonInstance, -} from '@embroider/core'; +import { AddonMeta, EmberAppInstance, OutputFileToInputFileMap, PackageInfo, AddonInstance } from '@embroider/core'; import { writeJSONSync, ensureDirSync, copySync, readdirSync, pathExistsSync, existsSync } from 'fs-extra'; import AddToTree from './add-to-tree'; import DummyPackage, { OwningAddon } from './dummy-package'; @@ -78,8 +27,7 @@ import type { Options as HTMLBarsOptions } from 'ember-cli-htmlbars'; import semver from 'semver'; import { MovablePackageCache } from './moved-package-cache'; import type { Transform } from 'babel-plugin-ember-template-compilation'; -import SourceMapConcat from 'fast-sourcemap-concat'; -import escapeRegExp from 'escape-string-regexp'; +import { CompatAppBuilder } from './compat-app-builder'; type EmberCliHTMLBarsAddon = AddonInstance & { htmlbarsOptions(): HTMLBarsOptions; @@ -91,1375 +39,7 @@ interface Group { vendorOutputPath: 'string'; } -interface TreeNames { - appJS: BroccoliNode; - htmlTree: BroccoliNode; - publicTree: BroccoliNode | undefined; - configTree: BroccoliNode; -} - -class ParsedEmberAsset { - kind: 'parsed-ember' = 'parsed-ember'; - relativePath: string; - fileAsset: EmberAsset; - html: PreparedEmberHTML; - - constructor(asset: EmberAsset) { - this.fileAsset = asset; - this.html = new PreparedEmberHTML(asset); - this.relativePath = asset.relativePath; - } - - validFor(other: EmberAsset) { - return this.fileAsset.mtime === other.mtime && this.fileAsset.size === other.size; - } -} - -type EmberENV = unknown; - -class BuiltEmberAsset { - kind: 'built-ember' = 'built-ember'; - relativePath: string; - parsedAsset: ParsedEmberAsset; - source: string; - - constructor(asset: ParsedEmberAsset) { - this.parsedAsset = asset; - this.source = asset.html.dom.serialize(); - this.relativePath = asset.relativePath; - } -} - -class ConcatenatedAsset { - kind: 'concatenated-asset' = 'concatenated-asset'; - constructor( - public relativePath: string, - public sources: (OnDiskAsset | InMemoryAsset)[], - private resolvableExtensions: RegExp - ) {} - get sourcemapPath() { - return this.relativePath.replace(this.resolvableExtensions, '') + '.map'; - } -} - -type InternalAsset = OnDiskAsset | InMemoryAsset | BuiltEmberAsset | ConcatenatedAsset; - -class CompatAppBuilder { - // for each relativePath, an Asset we have already emitted - private assets: Map = new Map(); - - constructor( - private root: string, - private appPackage: Package, - private options: Required, - private compatApp: CompatApp, - private configTree: V1Config, - private synthVendor: Package, - private synthStyles: Package - ) {} - - @Memoize() - private fastbootJSSrcDir() { - let target = join(this.compatApp.root, 'fastboot'); - if (pathExistsSync(target)) { - return target; - } - } - - private extractAssets(treePaths: OutputPaths): Asset[] { - let assets: Asset[] = []; - - // Everything in our traditional public tree is an on-disk asset - if (treePaths.publicTree) { - walkSync - .entries(treePaths.publicTree, { - directories: false, - }) - .forEach(entry => { - assets.push({ - kind: 'on-disk', - relativePath: entry.relativePath, - sourcePath: entry.fullPath, - mtime: entry.mtime as unknown as number, // https://github.com/joliss/node-walk-sync/pull/38 - size: entry.size, - }); - }); - } - - // ember-cli traditionally outputs a dummy testem.js file to prevent - // spurious errors when running tests under "ember s". - if (this.compatApp.shouldBuildTests) { - let testemAsset = this.findTestemAsset(); - if (testemAsset) { - assets.push(testemAsset); - } - } - - for (let asset of this.emberEntrypoints(treePaths.htmlTree)) { - assets.push(asset); - } - - return assets; - } - - @Memoize() - private findTestemAsset(): Asset | undefined { - let sourcePath; - try { - sourcePath = resolveSync('ember-cli/lib/broccoli/testem.js', { basedir: this.root }); - } catch (err) {} - if (sourcePath) { - let stat = statSync(sourcePath); - return { - kind: 'on-disk', - relativePath: 'testem.js', - sourcePath, - mtime: stat.mtime.getTime(), - size: stat.size, - }; - } - } - - private activeAddonChildren(pkg: Package = this.appPackage): AddonPackage[] { - let result = (pkg.dependencies.filter(this.isActiveAddon) as AddonPackage[]).filter( - // When looking for child addons, we want to ignore 'peerDependencies' of - // a given package, to align with how ember-cli resolves addons. So here - // we only include dependencies that definitely appear in one of the other - // sections. - addon => pkg.packageJSON.dependencies?.[addon.name] || pkg.packageJSON.devDependencies?.[addon.name] - ); - if (pkg === this.appPackage) { - let extras = [this.synthVendor, this.synthStyles].filter(this.isActiveAddon) as AddonPackage[]; - result = [...result, ...extras]; - } - return result.sort(this.orderAddons); - } - - @Memoize() - private get allActiveAddons(): AddonPackage[] { - let result = this.appPackage.findDescendants(this.isActiveAddon) as AddonPackage[]; - let extras = [this.synthVendor, this.synthStyles].filter(this.isActiveAddon) as AddonPackage[]; - let extraDescendants = flatMap(extras, dep => dep.findDescendants(this.isActiveAddon)) as AddonPackage[]; - result = [...result, ...extras, ...extraDescendants]; - return result.sort(this.orderAddons); - } - - @bind - private isActiveAddon(pkg: Package): boolean { - // todo: filter by addon-provided hook - return pkg.isEmberPackage(); - } - - @bind - private orderAddons(depA: Package, depB: Package): number { - let depAIdx = 0; - let depBIdx = 0; - - if (depA && depA.meta && depA.isV2Addon()) { - depAIdx = depA.meta['order-index'] || 0; - } - if (depB && depB.meta && depB.isV2Addon()) { - depBIdx = depB.meta['order-index'] || 0; - } - - return depAIdx - depBIdx; - } - - private resolvableExtensions(): string[] { - // webpack's default is ['.wasm', '.mjs', '.js', '.json']. Keeping that - // subset in that order is sensible, since many third-party libraries will - // expect it to work that way. - // - // For TS, we defer to ember-cli-babel, and the setting for - // "enableTypescriptTransform" can be set with and without - // ember-cli-typescript - return ['.wasm', '.mjs', '.js', '.json', '.ts', '.hbs', '.hbs.js']; - } - - private *emberEntrypoints(htmlTreePath: string): IterableIterator { - let classicEntrypoints = [ - { entrypoint: 'index.html', includeTests: false }, - { entrypoint: 'tests/index.html', includeTests: true }, - ]; - if (!this.compatApp.shouldBuildTests) { - classicEntrypoints.pop(); - } - for (let { entrypoint, includeTests } of classicEntrypoints) { - let sourcePath = join(htmlTreePath, entrypoint); - let stats = statSync(sourcePath); - let asset: EmberAsset = { - kind: 'ember', - relativePath: entrypoint, - includeTests, - sourcePath, - mtime: stats.mtime.getTime(), - size: stats.size, - rootURL: this.rootURL(), - prepare: (dom: JSDOM) => { - let scripts = [...dom.window.document.querySelectorAll('script')]; - let styles = [...dom.window.document.querySelectorAll('link[rel="stylesheet"]')] as HTMLLinkElement[]; - - return { - javascript: definitelyReplace(dom, this.compatApp.findAppScript(scripts, entrypoint)), - styles: definitelyReplace(dom, this.compatApp.findAppStyles(styles, entrypoint)), - implicitScripts: definitelyReplace(dom, this.compatApp.findVendorScript(scripts, entrypoint)), - implicitStyles: definitelyReplace(dom, this.compatApp.findVendorStyles(styles, entrypoint)), - testJavascript: maybeReplace(dom, this.compatApp.findTestScript(scripts)), - implicitTestScripts: maybeReplace(dom, this.compatApp.findTestSupportScript(scripts)), - implicitTestStyles: maybeReplace(dom, this.compatApp.findTestSupportStyles(styles)), - }; - }, - }; - yield asset; - } - } - - private modulePrefix(): string { - return this.configTree.readConfig().modulePrefix; - } - - private podModulePrefix(): string | undefined { - return this.configTree.readConfig().podModulePrefix; - } - - private rootURL(): string { - return this.configTree.readConfig().rootURL; - } - - private templateCompilerPath(): string { - return 'ember-source/vendor/ember/ember-template-compiler'; - } - - @Memoize() - private activeRules() { - return activePackageRules(this.options.packageRules.concat(defaultAddonPackageRules()), [ - { name: this.appPackage.name, version: this.appPackage.version, root: this.root }, - ...this.allActiveAddons.filter(p => p.meta['auto-upgraded']), - ]); - } - - private resolverConfig(engines: Engine[]): CompatResolverOptions { - let renamePackages = Object.assign({}, ...this.allActiveAddons.map(dep => dep.meta['renamed-packages'])); - let renameModules = Object.assign({}, ...this.allActiveAddons.map(dep => dep.meta['renamed-modules'])); - - let activeAddons: CompatResolverOptions['activeAddons'] = {}; - for (let addon of this.allActiveAddons) { - activeAddons[addon.name] = addon.root; - } - - let config: CompatResolverOptions = { - // this part is the base ModuleResolverOptions as required by @embroider/core - activeAddons, - renameModules, - renamePackages, - resolvableExtensions: this.resolvableExtensions(), - appRoot: this.root, - engines: engines.map((engine, index) => ({ - packageName: engine.package.name, - root: index === 0 ? this.root : engine.package.root, // first engine is the app, which has been relocated to this.roto - activeAddons: [...engine.addons] - .map(a => ({ - name: a.name, - root: a.root, - })) - // the traditional order is the order in which addons will run, such - // that the last one wins. Our resolver's order is the order to - // search, so first one wins. - .reverse(), - })), - - // this is the additional stufff that @embroider/compat adds on top to do - // global template resolving - modulePrefix: this.modulePrefix(), - podModulePrefix: this.podModulePrefix(), - options: this.options, - activePackageRules: this.activeRules(), - }; - - return config; - } - - private scriptPriority(pkg: Package) { - switch (pkg.name) { - case 'loader.js': - return 0; - case 'ember-source': - return 10; - default: - return 1000; - } - } - - @Memoize() - private get resolvableExtensionsPattern(): RegExp { - return extensionsPattern(this.resolvableExtensions()); - } - - private impliedAssets( - type: keyof ImplicitAssetPaths, - engine: Engine, - emberENV?: EmberENV - ): (OnDiskAsset | InMemoryAsset)[] { - let result: (OnDiskAsset | InMemoryAsset)[] = this.impliedAddonAssets(type, engine).map( - (sourcePath: string): OnDiskAsset => { - let stats = statSync(sourcePath); - return { - kind: 'on-disk', - relativePath: explicitRelative(this.root, sourcePath), - sourcePath, - mtime: stats.mtimeMs, - size: stats.size, - }; - } - ); - - if (type === 'implicit-scripts') { - result.unshift({ - kind: 'in-memory', - relativePath: '_testing_prefix_.js', - source: `var runningTests=false;`, - }); - - result.unshift({ - kind: 'in-memory', - relativePath: '_ember_env_.js', - source: `window.EmberENV={ ...(window.EmberENV || {}), ...${JSON.stringify(emberENV, null, 2)} };`, - }); - - result.push({ - kind: 'in-memory', - relativePath: '_loader_.js', - source: `loader.makeDefaultExport=false;`, - }); - } - - if (type === 'implicit-test-scripts') { - // this is the traditional test-support-suffix.js - result.push({ - kind: 'in-memory', - relativePath: '_testing_suffix_.js', - source: ` - var runningTests=true; - if (typeof Testem !== 'undefined' && (typeof QUnit !== 'undefined' || typeof Mocha !== 'undefined')) { - Testem.hookIntoTestFramework(); - }`, - }); - - // whether or not anybody was actually using @embroider/macros - // explicitly as an addon, we ensure its test-support file is always - // present. - if (!result.find(s => s.kind === 'on-disk' && s.sourcePath.endsWith('embroider-macros-test-support.js'))) { - result.unshift({ - kind: 'on-disk', - sourcePath: require.resolve('@embroider/macros/src/vendor/embroider-macros-test-support'), - mtime: 0, - size: 0, - relativePath: 'embroider-macros-test-support.js', - }); - } - } - - return result; - } - - private impliedAddonAssets(type: keyof ImplicitAssetPaths, engine: Engine): string[] { - let result: Array = []; - for (let addon of sortBy(Array.from(engine.addons), this.scriptPriority.bind(this))) { - let implicitScripts = addon.meta[type]; - if (implicitScripts) { - let styles = []; - let options = { basedir: addon.root }; - for (let mod of implicitScripts) { - if (type === 'implicit-styles') { - // exclude engines because they will handle their own css importation - if (!addon.isLazyEngine()) { - styles.push(resolve.sync(mod, options)); - } - } else { - result.push(resolve.sync(mod, options)); - } - } - if (styles.length) { - result = [...styles, ...result]; - } - } - } - return result; - } - - // unlike our full config, this one just needs to know how to parse all the - // syntax our app can contain. - @Memoize() - private babelParserConfig(): TransformOptions { - let babel = cloneDeep(this.compatApp.babelConfig()); - - if (!babel.plugins) { - babel.plugins = []; - } - - // Our stage3 code is always allowed to use dynamic import. We may emit it - // ourself when splitting routes. - babel.plugins.push(require.resolve('@babel/plugin-syntax-dynamic-import')); - return babel; - } - - @Memoize() - private babelConfig(resolverConfig: CompatResolverOptions) { - let babel = cloneDeep(this.compatApp.babelConfig()); - - if (!babel.plugins) { - babel.plugins = []; - } - - // Our stage3 code is always allowed to use dynamic import. We may emit it - // ourself when splitting routes. - babel.plugins.push(require.resolve('@babel/plugin-syntax-dynamic-import')); - - // https://github.com/webpack/webpack/issues/12154 - babel.plugins.push(require.resolve('./rename-require-plugin')); - - babel.plugins.push([require.resolve('babel-plugin-ember-template-compilation'), this.etcOptions(resolverConfig)]); - - // this is @embroider/macros configured for full stage3 resolution - babel.plugins.push(...this.compatApp.macrosConfig.babelPluginConfig()); - - let colocationOptions: TemplateColocationPluginOptions = { - appRoot: this.root, - - // This extra weirdness is a compromise in favor of build performance. - // - // 1. When auto-upgrading an addon from v1 to v2, we definitely want to - // run any custom AST transforms in stage1. - // - // 2. In general case, AST transforms are allowed to manipulate Javascript - // scope. This means that running transforms -- even when we're doing - // source-to-source compilation that emits handlebars and not wire - // format -- implies changing .hbs files into .js files. - // - // 3. So stage1 may need to rewrite .hbs to .hbs.js (to avoid colliding - // with an existing co-located .js file). - // - // 4. But stage1 doesn't necessarily want to run babel over the - // corresponding JS file. Most of the time, that's just an - // unnecessarily expensive second parse. (We only run it in stage1 to - // eliminate an addon's custom babel plugins, and many addons don't - // have any.) - // - // 5. Therefore, the work of template-colocation gets defered until here, - // and it may see co-located templates named `.hbs.js` instead of the - // usual `.hbs. - templateExtensions: ['.hbs', '.hbs.js'], - - // All of the above only applies to auto-upgraded packages that were - // authored in v1. V2 packages don't get any of this complexity, they're - // supposed to take care of colocating their own templates explicitly. - packageGuard: true, - }; - babel.plugins.push([templateColocationPluginPath, colocationOptions]); - - babel.plugins.push([ - require.resolve('./babel-plugin-adjust-imports'), - (() => { - let pluginConfig: AdjustImportsOptions = { - appRoot: resolverConfig.appRoot, - }; - return pluginConfig; - })(), - ]); - - // we can use globally shared babel runtime by default - babel.plugins.push([ - require.resolve('@babel/plugin-transform-runtime'), - { absoluteRuntime: __dirname, useESModules: true, regenerator: false }, - ]); - - const portable = makePortable(babel, { basedir: this.root }, this.portableHints); - addCachablePlugin(portable.config); - return portable; - } - - private insertEmberApp( - asset: ParsedEmberAsset, - appFiles: Engine[], - prepared: Map, - emberENV: EmberENV - ) { - let html = asset.html; - - if (this.fastbootConfig) { - // ignore scripts like ember-cli-livereload.js which are not really associated with - // "the app". - let ignoreScripts = html.dom.window.document.querySelectorAll('script'); - ignoreScripts.forEach(script => { - script.setAttribute('data-fastboot-ignore', ''); - }); - } - - // our tests entrypoint already includes a correct module dependency on the - // app, so we only insert the app when we're not inserting tests - if (!asset.fileAsset.includeTests) { - let appJS = this.topAppJSAsset(appFiles, prepared); - html.insertScriptTag(html.javascript, appJS.relativePath, { type: 'module' }); - } - - if (this.fastbootConfig) { - // any extra fastboot app files get inserted into our html.javascript - // section, after the app has been inserted. - for (let script of this.fastbootConfig.extraAppFiles) { - html.insertScriptTag(html.javascript, script, { tag: 'fastboot-script' }); - } - } - - html.insertStyleLink(html.styles, `assets/${this.appPackage.name}.css`); - - const parentEngine = appFiles.find(e => !e.parent) as Engine; - let vendorJS = this.implicitScriptsAsset(prepared, parentEngine, emberENV); - if (vendorJS) { - html.insertScriptTag(html.implicitScripts, vendorJS.relativePath); - } - - if (this.fastbootConfig) { - // any extra fastboot vendor files get inserted into our - // html.implicitScripts section, after the regular implicit script - // (vendor.js) have been inserted. - for (let script of this.fastbootConfig.extraVendorFiles) { - html.insertScriptTag(html.implicitScripts, script, { tag: 'fastboot-script' }); - } - } - - let implicitStyles = this.implicitStylesAsset(prepared, parentEngine); - if (implicitStyles) { - html.insertStyleLink(html.implicitStyles, implicitStyles.relativePath); - } - - if (!asset.fileAsset.includeTests) { - return; - } - - // Test-related assets happen below this point - - let testJS = this.testJSEntrypoint(appFiles, prepared); - html.insertScriptTag(html.testJavascript, testJS.relativePath, { type: 'module' }); - - let implicitTestScriptsAsset = this.implicitTestScriptsAsset(prepared, parentEngine); - if (implicitTestScriptsAsset) { - html.insertScriptTag(html.implicitTestScripts, implicitTestScriptsAsset.relativePath); - } - - let implicitTestStylesAsset = this.implicitTestStylesAsset(prepared, parentEngine); - if (implicitTestStylesAsset) { - html.insertStyleLink(html.implicitTestStyles, implicitTestStylesAsset.relativePath); - } - } - - private implicitScriptsAsset( - prepared: Map, - application: Engine, - emberENV: EmberENV - ): InternalAsset | undefined { - let asset = prepared.get('assets/vendor.js'); - if (!asset) { - let implicitScripts = this.impliedAssets('implicit-scripts', application, emberENV); - if (implicitScripts.length > 0) { - asset = new ConcatenatedAsset('assets/vendor.js', implicitScripts, this.resolvableExtensionsPattern); - prepared.set(asset.relativePath, asset); - } - } - return asset; - } - - private implicitStylesAsset(prepared: Map, application: Engine): InternalAsset | undefined { - let asset = prepared.get('assets/vendor.css'); - if (!asset) { - let implicitStyles = this.impliedAssets('implicit-styles', application); - if (implicitStyles.length > 0) { - // we reverse because we want the synthetic vendor style at the top - asset = new ConcatenatedAsset('assets/vendor.css', implicitStyles.reverse(), this.resolvableExtensionsPattern); - prepared.set(asset.relativePath, asset); - } - } - return asset; - } - - private implicitTestScriptsAsset( - prepared: Map, - application: Engine - ): InternalAsset | undefined { - let testSupportJS = prepared.get('assets/test-support.js'); - if (!testSupportJS) { - let implicitTestScripts = this.impliedAssets('implicit-test-scripts', application); - if (implicitTestScripts.length > 0) { - testSupportJS = new ConcatenatedAsset( - 'assets/test-support.js', - implicitTestScripts, - this.resolvableExtensionsPattern - ); - prepared.set(testSupportJS.relativePath, testSupportJS); - } - } - return testSupportJS; - } - - private implicitTestStylesAsset( - prepared: Map, - application: Engine - ): InternalAsset | undefined { - let asset = prepared.get('assets/test-support.css'); - if (!asset) { - let implicitTestStyles = this.impliedAssets('implicit-test-styles', application); - if (implicitTestStyles.length > 0) { - asset = new ConcatenatedAsset('assets/test-support.css', implicitTestStyles, this.resolvableExtensionsPattern); - prepared.set(asset.relativePath, asset); - } - } - return asset; - } - - // recurse to find all active addons that don't cross an engine boundary. - // Inner engines themselves will be returned, but not those engines' children. - // The output set's insertion order is the proper ember-cli compatible - // ordering of the addons. - private findActiveAddons(pkg: Package, engine: EngineSummary, isChild = false): void { - for (let child of this.activeAddonChildren(pkg)) { - if (!child.isEngine()) { - this.findActiveAddons(child, engine, true); - } - engine.addons.add(child); - } - // ensure addons are applied in the correct order, if set (via @embroider/compat/v1-addon) - if (!isChild) { - engine.addons = new Set( - [...engine.addons].sort((a, b) => { - return (a.meta['order-index'] || 0) - (b.meta['order-index'] || 0); - }) - ); - } - } - - private partitionEngines(appJSPath: string): EngineSummary[] { - let queue: EngineSummary[] = [ - { - package: this.appPackage, - addons: new Set(), - parent: undefined, - sourcePath: appJSPath, - destPath: this.root, - modulePrefix: this.modulePrefix(), - appRelativePath: '.', - }, - ]; - let done: EngineSummary[] = []; - let seenEngines: Set = new Set(); - while (true) { - let current = queue.shift(); - if (!current) { - break; - } - this.findActiveAddons(current.package, current); - for (let addon of current.addons) { - if (addon.isEngine() && !seenEngines.has(addon)) { - seenEngines.add(addon); - queue.push({ - package: addon, - addons: new Set(), - parent: current, - sourcePath: mangledEngineRoot(addon), - destPath: addon.root, - modulePrefix: addon.name, - appRelativePath: explicitRelative(this.root, addon.root), - }); - } - } - done.push(current); - } - return done; - } - - @Memoize() - private get activeFastboot() { - return this.activeAddonChildren(this.appPackage).find(a => a.name === 'ember-cli-fastboot'); - } - - @Memoize() - private get fastbootConfig(): - | { packageJSON: PackageInfo; extraAppFiles: string[]; extraVendorFiles: string[] } - | undefined { - if (this.activeFastboot) { - // this is relying on work done in stage1 by @embroider/compat/src/compat-adapters/ember-cli-fastboot.ts - let packageJSON = readJSONSync(join(this.activeFastboot.root, '_fastboot_', 'package.json')); - let { extraAppFiles, extraVendorFiles } = packageJSON['embroider-fastboot']; - delete packageJSON['embroider-fastboot']; - extraVendorFiles.push('assets/embroider_macros_fastboot_init.js'); - return { packageJSON, extraAppFiles, extraVendorFiles }; - } - } - - private appDiffers: { differ: AppDiffer; engine: EngineSummary }[] | undefined; - - private updateAppJS(inputPaths: OutputPaths): Engine[] { - let appJSPath = inputPaths.appJS; - if (!this.appDiffers) { - let engines = this.partitionEngines(appJSPath); - this.appDiffers = engines.map(engine => { - let differ: AppDiffer; - if (this.activeFastboot) { - differ = new AppDiffer( - engine.destPath, - engine.sourcePath, - [...engine.addons], - true, - this.fastbootJSSrcDir(), - this.babelParserConfig() - ); - } else { - differ = new AppDiffer(engine.destPath, engine.sourcePath, [...engine.addons]); - } - return { - differ, - engine, - }; - }); - } - // this is in reverse order because we need deeper engines to update before - // their parents, because they aren't really valid packages until they - // update, and their parents will go looking for their own `app-js` content. - this.appDiffers - .slice() - .reverse() - .forEach(a => a.differ.update()); - return this.appDiffers.map(a => { - return { - ...a.engine, - appFiles: new AppFiles(a.differ, this.resolvableExtensionsPattern, this.podModulePrefix()), - }; - }); - } - - private prepareAsset(asset: Asset, appFiles: Engine[], prepared: Map, emberENV: EmberENV) { - if (asset.kind === 'ember') { - let prior = this.assets.get(asset.relativePath); - let parsed: ParsedEmberAsset; - if (prior && prior.kind === 'built-ember' && prior.parsedAsset.validFor(asset)) { - // we can reuse the parsed html - parsed = prior.parsedAsset; - parsed.html.clear(); - } else { - parsed = new ParsedEmberAsset(asset); - } - this.insertEmberApp(parsed, appFiles, prepared, emberENV); - prepared.set(asset.relativePath, new BuiltEmberAsset(parsed)); - } else { - prepared.set(asset.relativePath, asset); - } - } - - private prepareAssets(requestedAssets: Asset[], appFiles: Engine[], emberENV: EmberENV): Map { - let prepared: Map = new Map(); - for (let asset of requestedAssets) { - this.prepareAsset(asset, appFiles, prepared, emberENV); - } - return prepared; - } - - private assetIsValid(asset: InternalAsset, prior: InternalAsset | undefined): boolean { - if (!prior) { - return false; - } - switch (asset.kind) { - case 'on-disk': - return prior.kind === 'on-disk' && prior.size === asset.size && prior.mtime === asset.mtime; - case 'in-memory': - return prior.kind === 'in-memory' && stringOrBufferEqual(prior.source, asset.source); - case 'built-ember': - return prior.kind === 'built-ember' && prior.source === asset.source; - case 'concatenated-asset': - return ( - prior.kind === 'concatenated-asset' && - prior.sources.length === asset.sources.length && - prior.sources.every((priorFile, index) => { - let newFile = asset.sources[index]; - return this.assetIsValid(newFile, priorFile); - }) - ); - } - } - - private updateOnDiskAsset(asset: OnDiskAsset) { - let destination = join(this.root, asset.relativePath); - ensureDirSync(dirname(destination)); - copySync(asset.sourcePath, destination, { dereference: true }); - } - - private updateInMemoryAsset(asset: InMemoryAsset) { - let destination = join(this.root, asset.relativePath); - ensureDirSync(dirname(destination)); - writeFileSync(destination, asset.source, 'utf8'); - } - - private updateBuiltEmberAsset(asset: BuiltEmberAsset) { - let destination = join(this.root, asset.relativePath); - ensureDirSync(dirname(destination)); - writeFileSync(destination, asset.source, 'utf8'); - } - - private async updateConcatenatedAsset(asset: ConcatenatedAsset) { - let concat = new SourceMapConcat({ - outputFile: join(this.root, asset.relativePath), - mapCommentType: asset.relativePath.endsWith('.js') ? 'line' : 'block', - baseDir: this.root, - }); - if (process.env.EMBROIDER_CONCAT_STATS) { - let MeasureConcat = (await import('@embroider/core/src/measure-concat')).default; - concat = new MeasureConcat(asset.relativePath, concat, this.root); - } - for (let source of asset.sources) { - switch (source.kind) { - case 'on-disk': - concat.addFile(explicitRelative(this.root, source.sourcePath)); - break; - case 'in-memory': - if (typeof source.source !== 'string') { - throw new Error(`attempted to concatenated a Buffer-backed in-memory asset`); - } - concat.addSpace(source.source); - break; - default: - assertNever(source); - } - } - await concat.end(); - } - - private async updateAssets(requestedAssets: Asset[], appFiles: Engine[], emberENV: EmberENV) { - let assets = this.prepareAssets(requestedAssets, appFiles, emberENV); - for (let asset of assets.values()) { - if (this.assetIsValid(asset, this.assets.get(asset.relativePath))) { - continue; - } - debug('rebuilding %s', asset.relativePath); - switch (asset.kind) { - case 'on-disk': - this.updateOnDiskAsset(asset); - break; - case 'in-memory': - this.updateInMemoryAsset(asset); - break; - case 'built-ember': - this.updateBuiltEmberAsset(asset); - break; - case 'concatenated-asset': - await this.updateConcatenatedAsset(asset); - break; - default: - assertNever(asset); - } - } - for (let oldAsset of this.assets.values()) { - if (!assets.has(oldAsset.relativePath)) { - unlinkSync(join(this.root, oldAsset.relativePath)); - } - } - this.assets = assets; - return [...assets.values()]; - } - - private gatherAssets(inputPaths: OutputPaths): Asset[] { - // first gather all the assets out of addons - let assets: Asset[] = []; - for (let pkg of this.allActiveAddons) { - if (pkg.meta['public-assets']) { - for (let [filename, appRelativeURL] of Object.entries(pkg.meta['public-assets'] || {})) { - let sourcePath = resolvePath(pkg.root, filename); - let stats = statSync(sourcePath); - assets.push({ - kind: 'on-disk', - sourcePath, - relativePath: appRelativeURL, - mtime: stats.mtimeMs, - size: stats.size, - }); - } - } - } - - if (this.activeFastboot) { - const source = ` - (function(){ - var key = '_embroider_macros_runtime_config'; - if (!window[key]){ window[key] = [];} - window[key].push(function(m) { - m.setGlobalConfig('fastboot', Object.assign({}, m.getGlobalConfig().fastboot, { isRunning: true })); - }); - }())`; - assets.push({ - kind: 'in-memory', - source, - relativePath: 'assets/embroider_macros_fastboot_init.js', - }); - } - - // and finally tack on the ones from our app itself - return assets.concat(this.extractAssets(inputPaths)); - } - - async build(inputPaths: OutputPaths) { - // on the first build, we lock down the macros config. on subsequent builds, - // this doesn't do anything anyway because it's idempotent. - this.compatApp.macrosConfig.finalize(); - - let appFiles = this.updateAppJS(inputPaths); - let emberENV = this.configTree.readConfig().EmberENV; - let assets = this.gatherAssets(inputPaths); - - let finalAssets = await this.updateAssets(assets, appFiles, emberENV); - - let assetPaths = assets.map(asset => asset.relativePath); - - if (this.activeFastboot) { - // when using fastboot, our own package.json needs to be in the output so fastboot can read it. - assetPaths.push('package.json'); - } - - for (let asset of finalAssets) { - // our concatenated assets all have map files that ride along. Here we're - // telling the final stage packager to be sure and serve the map files - // too. - if (asset.kind === 'concatenated-asset') { - assetPaths.push(asset.sourcemapPath); - } - } - - let meta: AppMeta = { - type: 'app', - version: 2, - assets: assetPaths, - babel: { - filename: '_babel_config_.js', - isParallelSafe: true, // TODO - majorVersion: this.compatApp.babelMajorVersion(), - fileFilter: '_babel_filter_.js', - }, - 'root-url': this.rootURL(), - }; - - // all compat apps are auto-upgraded, there's no v2 app format here - meta['auto-upgraded'] = true; - - let pkg = this.combinePackageJSON(meta); - writeFileSync(join(this.root, 'package.json'), JSON.stringify(pkg, null, 2), 'utf8'); - - let resolverConfig = this.resolverConfig(appFiles); - this.addResolverConfig(resolverConfig); - let babelConfig = this.babelConfig(resolverConfig); - this.addBabelConfig(babelConfig); - } - - private combinePackageJSON(meta: AppMeta): object { - let pkgLayers: any[] = [this.appPackage.packageJSON]; - let fastbootConfig = this.fastbootConfig; - if (fastbootConfig) { - // fastboot-specific package.json output is allowed to add to our original package.json - pkgLayers.push(fastbootConfig.packageJSON); - } - // but our own new v2 app metadata takes precedence over both - pkgLayers.push({ keywords: ['ember-addon'], 'ember-addon': meta }); - return combinePackageJSON(...pkgLayers); - } - - private etcOptions(resolverConfig: CompatResolverOptions): EtcOptions { - let transforms = this.compatApp.htmlbarsPlugins; - - let { plugins: macroPlugins, setConfig } = MacrosConfig.transforms(); - setConfig(this.compatApp.macrosConfig); - for (let macroPlugin of macroPlugins) { - transforms.push(macroPlugin as any); - } - - if ( - this.options.staticComponents || - this.options.staticHelpers || - this.options.staticModifiers || - (globalThis as any).embroider_audit - ) { - let opts: ResolverTransformOptions = { - appRoot: resolverConfig.appRoot, - }; - transforms.push([require.resolve('./resolver-transform'), opts]); - } - - return { - transforms, - compilerPath: resolve.sync(this.templateCompilerPath(), { basedir: this.root }), - enableLegacyModules: ['ember-cli-htmlbars', 'ember-cli-htmlbars-inline-precompile', 'htmlbars-inline-precompile'], - }; - } - - @Memoize() - private get portableHints(): PortableHint[] { - return this.options.pluginHints.map(hint => { - let cursor = join(this.appPackage.root, 'package.json'); - for (let i = 0; i < hint.resolve.length; i++) { - let target = hint.resolve[i]; - if (i < hint.resolve.length - 1) { - target = join(target, 'package.json'); - } - cursor = resolve.sync(target, { basedir: dirname(cursor) }); - } - - return { - requireFile: cursor, - useMethod: hint.useMethod, - packageVersion: maybeNodeModuleVersion(cursor), - }; - }); - } - - private addBabelConfig(pconfig: { config: TransformOptions; isParallelSafe: boolean }) { - if (!pconfig.isParallelSafe) { - warn('Your build is slower because some babel plugins are non-serializable'); - } - writeFileSync( - join(this.root, '_babel_config_.js'), - `module.exports = ${JSON.stringify(pconfig.config, null, 2)}`, - 'utf8' - ); - writeFileSync( - join(this.root, '_babel_filter_.js'), - babelFilterTemplate({ skipBabel: this.options.skipBabel, appRoot: this.root }), - 'utf8' - ); - } - - private addResolverConfig(config: CompatResolverOptions) { - outputJSONSync(join(this.root, '.embroider', 'resolver.json'), config); - } - - private shouldSplitRoute(routeName: string) { - return ( - !this.options.splitAtRoutes || - this.options.splitAtRoutes.find(pattern => { - if (typeof pattern === 'string') { - return pattern === routeName; - } else { - return pattern.test(routeName); - } - }) - ); - } - - private splitRoute( - routeName: string, - files: RouteFiles, - addToParent: (routeName: string, filename: string) => void, - addLazyBundle: (routeNames: string[], files: string[]) => void - ) { - let shouldSplit = routeName && this.shouldSplitRoute(routeName); - let ownFiles = []; - let ownNames = new Set() as Set; - - if (files.template) { - if (shouldSplit) { - ownFiles.push(files.template); - ownNames.add(routeName); - } else { - addToParent(routeName, files.template); - } - } - - if (files.controller) { - if (shouldSplit) { - ownFiles.push(files.controller); - ownNames.add(routeName); - } else { - addToParent(routeName, files.controller); - } - } - - if (files.route) { - if (shouldSplit) { - ownFiles.push(files.route); - ownNames.add(routeName); - } else { - addToParent(routeName, files.route); - } - } - - for (let [childName, childFiles] of files.children) { - this.splitRoute( - `${routeName}.${childName}`, - childFiles, - - (childRouteName: string, childFile: string) => { - // this is our child calling "addToParent" - if (shouldSplit) { - ownFiles.push(childFile); - ownNames.add(childRouteName); - } else { - addToParent(childRouteName, childFile); - } - }, - (routeNames: string[], files: string[]) => { - addLazyBundle(routeNames, files); - } - ); - } - - if (ownFiles.length > 0) { - addLazyBundle([...ownNames], ownFiles); - } - } - - private topAppJSAsset(engines: Engine[], prepared: Map): InternalAsset { - let [app, ...childEngines] = engines; - let relativePath = `assets/${this.appPackage.name}.js`; - return this.appJSAsset(relativePath, app, childEngines, prepared, { - autoRun: this.compatApp.autoRun, - appBoot: !this.compatApp.autoRun ? this.compatApp.appBoot.readAppBoot() : '', - mainModule: explicitRelative(dirname(relativePath), 'app'), - appConfig: this.configTree.readConfig().APP, - }); - } - - @Memoize() - private get staticAppPathsPattern(): RegExp | undefined { - if (this.options.staticAppPaths.length > 0) { - return new RegExp( - '^(?:' + this.options.staticAppPaths.map(staticAppPath => escapeRegExp(staticAppPath)).join('|') + ')(?:$|/)' - ); - } - } - - private requiredOtherFiles(appFiles: AppFiles): readonly string[] { - let pattern = this.staticAppPathsPattern; - if (pattern) { - return appFiles.otherAppFiles.filter(f => { - return !pattern!.test(f); - }); - } else { - return appFiles.otherAppFiles; - } - } - - private appJSAsset( - relativePath: string, - engine: Engine, - childEngines: Engine[], - prepared: Map, - entryParams?: Partial[0]> - ): InternalAsset { - let { appFiles } = engine; - let cached = prepared.get(relativePath); - if (cached) { - return cached; - } - - let eagerModules = []; - - let requiredAppFiles = [this.requiredOtherFiles(appFiles)]; - if (!this.options.staticComponents) { - requiredAppFiles.push(appFiles.components); - } - if (!this.options.staticHelpers) { - requiredAppFiles.push(appFiles.helpers); - } - if (!this.options.staticModifiers) { - requiredAppFiles.push(appFiles.modifiers); - } - - let styles = []; - // only import styles from engines with a parent (this excludeds the parent application) as their styles - // will be inserted via a direct tag. - if (engine.parent && engine.package.isLazyEngine()) { - let implicitStyles = this.impliedAssets('implicit-styles', engine); - for (let style of implicitStyles) { - styles.push({ - path: explicitRelative('assets/_engine_', style.relativePath), - }); - } - - let engineMeta = engine.package.meta as AddonMeta; - if (engineMeta && engineMeta['implicit-styles']) { - for (let style of engineMeta['implicit-styles']) { - styles.push({ - path: explicitRelative(dirname(relativePath), join(engine.appRelativePath, style)), - }); - } - } - } - - let lazyEngines: { names: string[]; path: string }[] = []; - for (let childEngine of childEngines) { - let asset = this.appJSAsset( - `assets/_engine_/${encodeURIComponent(childEngine.package.name)}.js`, - childEngine, - [], - prepared - ); - if (childEngine.package.isLazyEngine()) { - lazyEngines.push({ - names: [childEngine.package.name], - path: explicitRelative(dirname(relativePath), asset.relativePath), - }); - } else { - eagerModules.push(explicitRelative(dirname(relativePath), asset.relativePath)); - } - } - let lazyRoutes: { names: string[]; path: string }[] = []; - for (let [routeName, routeFiles] of appFiles.routeFiles.children) { - this.splitRoute( - routeName, - routeFiles, - (_: string, filename: string) => { - requiredAppFiles.push([filename]); - }, - (routeNames: string[], files: string[]) => { - let routeEntrypoint = `assets/_route_/${encodeURIComponent(routeNames[0])}.js`; - if (!prepared.has(routeEntrypoint)) { - prepared.set(routeEntrypoint, this.routeEntrypoint(engine, routeEntrypoint, files)); - } - lazyRoutes.push({ - names: routeNames, - path: this.importPaths(engine, routeEntrypoint).buildtime, - }); - } - ); - } - - let [fastboot, nonFastboot] = partition(excludeDotFiles(flatten(requiredAppFiles)), file => - appFiles.isFastbootOnly.get(file) - ); - let amdModules = nonFastboot.map(file => this.importPaths(engine, file)); - let fastbootOnlyAmdModules = fastboot.map(file => this.importPaths(engine, file)); - - // this is a backward-compatibility feature: addons can force inclusion of - // modules. - this.gatherImplicitModules('implicit-modules', engine, amdModules); - - let params = { amdModules, fastbootOnlyAmdModules, lazyRoutes, lazyEngines, eagerModules, styles }; - if (entryParams) { - Object.assign(params, entryParams); - } - - let source = entryTemplate(params); - - let asset: InternalAsset = { - kind: 'in-memory', - source, - relativePath, - }; - prepared.set(relativePath, asset); - return asset; - } - - private importPaths(engine: Engine, engineRelativePath: string) { - let noHBS = engineRelativePath.replace(this.resolvableExtensionsPattern, '').replace(/\.hbs$/, ''); - return { - runtime: `${engine.modulePrefix}/${noHBS}`, - buildtime: posix.join(engine.package.name, engineRelativePath), - }; - } - - private routeEntrypoint(engine: Engine, relativePath: string, files: string[]) { - let [fastboot, nonFastboot] = partition(files, file => engine.appFiles.isFastbootOnly.get(file)); - - let asset: InternalAsset = { - kind: 'in-memory', - source: routeEntryTemplate({ - files: nonFastboot.map(f => this.importPaths(engine, f)), - fastbootOnlyFiles: fastboot.map(f => this.importPaths(engine, f)), - }), - relativePath, - }; - return asset; - } - - private testJSEntrypoint(engines: Engine[], prepared: Map): InternalAsset { - let asset = prepared.get(`assets/test.js`); - if (asset) { - return asset; - } - - // We're only building tests from the first engine (the app). This is the - // normal thing to do -- tests from engines don't automatically roll up into - // the app. - let engine = engines[0]; - - const myName = 'assets/test.js'; - - // tests necessarily also include the app. This is where we account for - // that. The classic solution was to always include the app's separate - // script tag in the tests HTML, but that isn't as easy for final stage - // packagers to understand. It's better to express it here as a direct - // module dependency. - let eagerModules: string[] = [ - explicitRelative(dirname(myName), this.topAppJSAsset(engines, prepared).relativePath), - ]; - - let amdModules: { runtime: string; buildtime: string }[] = []; - // this is a backward-compatibility feature: addons can force inclusion of - // test support modules. - this.gatherImplicitModules('implicit-test-modules', engine, amdModules); - - let { appFiles } = engine; - for (let relativePath of appFiles.tests) { - amdModules.push(this.importPaths(engine, relativePath)); - } - - let source = entryTemplate({ - amdModules, - eagerModules, - testSuffix: true, - }); - - asset = { - kind: 'in-memory', - source, - relativePath: myName, - }; - prepared.set(asset.relativePath, asset); - return asset; - } - - private gatherImplicitModules( - section: 'implicit-modules' | 'implicit-test-modules', - engine: Engine, - lazyModules: { runtime: string; buildtime: string }[] - ) { - for (let addon of engine.addons) { - let implicitModules = addon.meta[section]; - if (implicitModules) { - let renamedModules = inverseRenamedModules(addon.meta, this.resolvableExtensionsPattern); - for (let name of implicitModules) { - let packageName = addon.name; - - if (addon.isV2Addon()) { - let renamedMeta = addon.meta['renamed-packages']; - if (renamedMeta) { - Object.entries(renamedMeta).forEach(([key, value]) => { - if (value === addon!.name) { - packageName = key; - } - }); - } - } - - let runtime = join(packageName, name).replace(this.resolvableExtensionsPattern, ''); - let runtimeRenameLookup = runtime.split('\\').join('/'); - if (renamedModules && renamedModules[runtimeRenameLookup]) { - runtime = renamedModules[runtimeRenameLookup]; - } - runtime = runtime.split(sep).join('/'); - lazyModules.push({ - runtime, - buildtime: posix.join(packageName, name), - }); - } - } - } - } -} - -// This runs at broccoli-pipeline-construction time, whereas our actual +// This runs at broccoli-pipeline-construction time, whereas the // CompatAppBuilder instance only becomes available during tree-building time. export default class CompatApp { private annotation = '@embroider/compat/app'; @@ -2285,205 +865,11 @@ export default class CompatApp { } } -function maybeReplace(dom: JSDOM, element: Element | undefined): Node | undefined { - if (element) { - return definitelyReplace(dom, element); - } -} - -function definitelyReplace(dom: JSDOM, element: Element): Node { - let placeholder = dom.window.document.createTextNode(''); - element.replaceWith(placeholder); - return placeholder; -} - -function defaultAddonPackageRules(): PackageRules[] { - return readdirSync(join(__dirname, 'addon-dependency-rules')) - .map(filename => { - if (filename.endsWith('.js')) { - // eslint-disable-next-line @typescript-eslint/no-require-imports - return require(join(__dirname, 'addon-dependency-rules', filename)).default; - } - }) - .filter(Boolean) - .reduce((a, b) => a.concat(b), []); -} - -const entryTemplate = jsHandlebarsCompile(` -import { importSync as i, macroCondition, getGlobalConfig } from '@embroider/macros'; -let w = window; -let d = w.define; - -{{#if styles}} - if (macroCondition(!getGlobalConfig().fastboot?.isRunning)) { - {{#each styles as |stylePath| ~}} - i("{{js-string-escape stylePath.path}}"); - {{/each}} - } -{{/if}} - -{{#each amdModules as |amdModule| ~}} - d("{{js-string-escape amdModule.runtime}}", function(){ return i("{{js-string-escape amdModule.buildtime}}");}); -{{/each}} - -{{#if fastbootOnlyAmdModules}} - if (macroCondition(getGlobalConfig().fastboot?.isRunning)) { - {{#each fastbootOnlyAmdModules as |amdModule| ~}} - d("{{js-string-escape amdModule.runtime}}", function(){ return i("{{js-string-escape amdModule.buildtime}}");}); - {{/each}} - } -{{/if}} - -{{#each eagerModules as |eagerModule| ~}} - i("{{js-string-escape eagerModule}}"); -{{/each}} - -{{#if lazyRoutes}} -w._embroiderRouteBundles_ = [ - {{#each lazyRoutes as |route|}} - { - names: {{{json-stringify route.names}}}, - load: function() { - return import("{{js-string-escape route.path}}"); - } - }, - {{/each}} -] -{{/if}} - -{{#if lazyEngines}} -w._embroiderEngineBundles_ = [ - {{#each lazyEngines as |engine|}} - { - names: {{{json-stringify engine.names}}}, - load: function() { - return import("{{js-string-escape engine.path}}"); - } - }, - {{/each}} -] -{{/if}} - -{{#if autoRun ~}} -if (!runningTests) { - i("{{js-string-escape mainModule}}").default.create({{{json-stringify appConfig}}}); -} -{{else if appBoot ~}} - {{{ appBoot }}} -{{/if}} - -{{#if testSuffix ~}} - {{!- TODO: both of these suffixes should get dynamically generated so they incorporate - any content-for added by addons. -}} - - - {{!- this is the traditional tests-suffix.js -}} - i('../tests/test-helper'); - EmberENV.TESTS_FILE_LOADED = true; -{{/if}} -`) as (params: { - amdModules: { runtime: string; buildtime: string }[]; - fastbootOnlyAmdModules?: { runtime: string; buildtime: string }[]; - eagerModules?: string[]; - autoRun?: boolean; - appBoot?: string; - mainModule?: string; - appConfig?: unknown; - testSuffix?: boolean; - lazyRoutes?: { names: string[]; path: string }[]; - lazyEngines?: { names: string[]; path: string }[]; - styles?: { path: string }[]; -}) => string; - -const routeEntryTemplate = jsHandlebarsCompile(` -import { importSync as i } from '@embroider/macros'; -let d = window.define; -{{#each files as |amdModule| ~}} -d("{{js-string-escape amdModule.runtime}}", function(){ return i("{{js-string-escape amdModule.buildtime}}");}); -{{/each}} -{{#if fastbootOnlyFiles}} - import { macroCondition, getGlobalConfig } from '@embroider/macros'; - if (macroCondition(getGlobalConfig().fastboot?.isRunning)) { - {{#each fastbootOnlyFiles as |amdModule| ~}} - d("{{js-string-escape amdModule.runtime}}", function(){ return i("{{js-string-escape amdModule.buildtime}}");}); - {{/each}} - } -{{/if}} -`) as (params: { - files: { runtime: string; buildtime: string }[]; - fastbootOnlyFiles: { runtime: string; buildtime: string }[]; -}) => string; - -function stringOrBufferEqual(a: string | Buffer, b: string | Buffer): boolean { - if (typeof a === 'string' && typeof b === 'string') { - return a === b; - } - if (a instanceof Buffer && b instanceof Buffer) { - return Buffer.compare(a, b) === 0; - } - return false; -} - -const babelFilterTemplate = jsHandlebarsCompile(` -const { babelFilter } = require(${JSON.stringify(require.resolve('@embroider/core'))}); -module.exports = babelFilter({{{json-stringify skipBabel}}}, "{{{js-string-escape appRoot}}}"); -`) as (params: { skipBabel: Options['skipBabel']; appRoot: string }) => string; - -// meta['renamed-modules'] has mapping from classic filename to real filename. -// This takes that and converts it to the inverst mapping from real import path -// to classic import path. -function inverseRenamedModules(meta: AddonPackage['meta'], extensions: RegExp) { - let renamed = meta['renamed-modules']; - if (renamed) { - let inverted = {} as { [name: string]: string }; - for (let [classic, real] of Object.entries(renamed)) { - inverted[real.replace(extensions, '')] = classic.replace(extensions, ''); - } - return inverted; - } -} - -function combinePackageJSON(...layers: object[]) { - function custom(objValue: any, srcValue: any, key: string, _object: any, _source: any, stack: { size: number }) { - if (key === 'keywords' && stack.size === 0) { - if (Array.isArray(objValue)) { - return objValue.concat(srcValue); - } - } - } - return mergeWith({}, ...layers, custom); -} - -function addCachablePlugin(babelConfig: TransformOptions) { - if (Array.isArray(babelConfig.plugins) && babelConfig.plugins.length > 0) { - const plugins = Object.create(null); - plugins[cacheBustingPluginPath] = cacheBustingPluginVersion; - - for (const plugin of babelConfig.plugins) { - let absolutePathToPlugin: string; - if (Array.isArray(plugin) && typeof plugin[0] === 'string') { - absolutePathToPlugin = plugin[0] as string; - } else if (typeof plugin === 'string') { - absolutePathToPlugin = plugin; - } else { - throw new Error(`[Embroider] a babel plugin without an absolute path was from: ${plugin}`); - } - - plugins[absolutePathToPlugin] = maybeNodeModuleVersion(absolutePathToPlugin); - } - - babelConfig.plugins.push([ - cacheBustingPluginPath, - { - plugins, - }, - ]); - } +interface Preprocessors { + preprocessJs(tree: BroccoliNode, a: string, b: string, options: object): BroccoliNode; + preprocessCss(tree: BroccoliNode, a: string, b: string, options: object): BroccoliNode; } -function excludeDotFiles(files: string[]) { - return files.filter(file => !file.startsWith('.') && !file.includes('/.')); -} function throwIfMissing( asset: T | undefined, needle: string, @@ -2503,8 +889,3 @@ function throwIfMissing( return asset; } - -interface Preprocessors { - preprocessJs(tree: BroccoliNode, a: string, b: string, options: object): BroccoliNode; - preprocessCss(tree: BroccoliNode, a: string, b: string, options: object): BroccoliNode; -} From 318ef3aba45749ae2a7c1488e1e9e9306e5ba057 Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Wed, 7 Jun 2023 09:50:05 -0400 Subject: [PATCH 21/72] less state on CompatApp --- packages/compat/src/compat-addons.ts | 5 ++-- packages/compat/src/compat-app.ts | 35 ++++++++---------------- packages/compat/src/v1-instance-cache.ts | 8 ++---- 3 files changed, 18 insertions(+), 30 deletions(-) diff --git a/packages/compat/src/compat-addons.ts b/packages/compat/src/compat-addons.ts index 36d6a289b..77722cde9 100644 --- a/packages/compat/src/compat-addons.ts +++ b/packages/compat/src/compat-addons.ts @@ -36,10 +36,11 @@ export default class CompatAddons implements Stage { ensureDirSync(compatApp.options.workspaceDir!); this.destDir = realpathSync(compatApp.options.workspaceDir!); - this.packageCache = compatApp.movablePackageCache.moveAddons(this.destDir); + let movablePackageCache = compatApp.makePackageCache(); + this.packageCache = movablePackageCache.moveAddons(this.destDir); this.inputPath = compatApp.root; this.treeSyncMap = new WeakMap(); - this.v1Cache = new V1InstanceCache(compatApp); + this.v1Cache = new V1InstanceCache(compatApp, movablePackageCache); } get tree(): Node { diff --git a/packages/compat/src/compat-app.ts b/packages/compat/src/compat-app.ts index 2028d370c..d7bc3355e 100644 --- a/packages/compat/src/compat-app.ts +++ b/packages/compat/src/compat-app.ts @@ -44,16 +44,12 @@ interface Group { export default class CompatApp { private annotation = '@embroider/compat/app'; private active: CompatAppBuilder | undefined; - private outputPath: string | undefined; - private packageCache: PackageCache | undefined; readonly options: Required; private _publicAssets: { [filePath: string]: string } = Object.create(null); private _implicitScripts: string[] = []; private _implicitStyles: string[] = []; - movablePackageCache: MovablePackageCache; - private get isDummy(): boolean { return this.legacyEmberAppInstance.project.pkg.keywords?.includes('ember-addon') ?? false; } @@ -784,15 +780,18 @@ export default class CompatApp { // macros in a classic build. active: true, }); + } - this.movablePackageCache = new MovablePackageCache(this.macrosConfig, this.root); + makePackageCache(): MovablePackageCache { + let movablePackageCache = new MovablePackageCache(this.macrosConfig, this.root); if (this.isDummy) { - let owningAddon = new OwningAddon(legacyEmberAppInstance.project.root, this.movablePackageCache); - this.movablePackageCache.seed(owningAddon); - this.movablePackageCache.seed(new DummyPackage(this.root, owningAddon, this.movablePackageCache)); + let owningAddon = new OwningAddon(this.legacyEmberAppInstance.project.root, movablePackageCache); + movablePackageCache.seed(owningAddon); + movablePackageCache.seed(new DummyPackage(this.root, owningAddon, movablePackageCache)); this.macrosConfig.enablePackageDevelopment(owningAddon.root); } + return movablePackageCache; } private inTrees(prevStageTree: BroccoliNode) { @@ -826,17 +825,18 @@ export default class CompatApp { } asStage(prevStage: Stage): Stage { + let resolve: (result: { packageCache: PackageCache; outputPath: string }) => void; + let promise: Promise<{ packageCache: PackageCache; outputPath: string }> = new Promise(r => (resolve = r)); + let tree = () => { let inTrees = this.inTrees(prevStage.tree); return new WaitForTrees(inTrees, this.annotation, async treePaths => { if (!this.active) { let { outputPath, packageCache } = await prevStage.ready(); - this.outputPath = outputPath; - this.packageCache = packageCache; this.active = await this.instantiate(outputPath, prevStage.inputPath, packageCache, inTrees.configTree); + resolve({ packageCache, outputPath }); } await this.active.build(treePaths); - this.deferReady.resolve(); }); }; @@ -845,24 +845,13 @@ export default class CompatApp { return prevStage.inputPath; }, ready: async () => { - await this.deferReady.promise; - return { - outputPath: this.outputPath!, - packageCache: this.packageCache!, - }; + return await promise; }, get tree() { return tree(); }, }; } - - @Memoize() - private get deferReady() { - let resolve: Function; - let promise: Promise = new Promise(r => (resolve = r)); - return { resolve: resolve!, promise }; - } } interface Preprocessors { diff --git a/packages/compat/src/v1-instance-cache.ts b/packages/compat/src/v1-instance-cache.ts index cb245a874..149708ff5 100644 --- a/packages/compat/src/v1-instance-cache.ts +++ b/packages/compat/src/v1-instance-cache.ts @@ -4,7 +4,7 @@ import V1Addon, { V1AddonConstructor } from './v1-addon'; import { pathExistsSync } from 'fs-extra'; -import { AddonInstance, getOrCreate } from '@embroider/core'; +import { AddonInstance, getOrCreate, PackageCache } from '@embroider/core'; import CompatApp from './compat-app'; export default class V1InstanceCache { @@ -12,11 +12,9 @@ export default class V1InstanceCache { // There can be many because a single copy of an addon may be consumed by many // other packages and each gets an instance. private addons: Map = new Map(); - - private app: CompatApp; private orderIdx: number; - constructor(app: CompatApp) { + constructor(private app: CompatApp, private packageCache: PackageCache) { this.app = app; this.orderIdx = 0; @@ -63,7 +61,7 @@ export default class V1InstanceCache { this.orderIdx += 1; let Klass = this.adapterClass(addonInstance); - let v1Addon = new Klass(addonInstance, this.app.options, this.app, this.app.movablePackageCache, this.orderIdx); + let v1Addon = new Klass(addonInstance, this.app.options, this.app, this.packageCache, this.orderIdx); let pkgs = getOrCreate(this.addons, v1Addon.root, () => []); pkgs.push(v1Addon); } From 09efde90ffd3dd6680e199180cbeaa42c0bae774 Mon Sep 17 00:00:00 2001 From: Chris Manson Date: Thu, 8 Jun 2023 14:16:24 +0100 Subject: [PATCH 22/72] remove OwningAddon custom package subclass --- packages/compat/src/compat-app.ts | 10 +++++----- packages/compat/src/dummy-package.ts | 19 ++++--------------- 2 files changed, 9 insertions(+), 20 deletions(-) diff --git a/packages/compat/src/compat-app.ts b/packages/compat/src/compat-app.ts index c424b4385..f4c11a21b 100644 --- a/packages/compat/src/compat-app.ts +++ b/packages/compat/src/compat-app.ts @@ -13,7 +13,7 @@ import { WriteV1AppBoot, ReadV1AppBoot } from './v1-appboot'; import { AddonMeta, EmberAppInstance, OutputFileToInputFileMap, PackageInfo, AddonInstance } from '@embroider/core'; import { writeJSONSync, ensureDirSync, copySync, readdirSync, pathExistsSync, existsSync } from 'fs-extra'; import AddToTree from './add-to-tree'; -import DummyPackage, { OwningAddon } from './dummy-package'; +import DummyPackage from './dummy-package'; import { TransformOptions } from '@babel/core'; import { isEmbroiderMacrosPlugin, MacrosConfig } from '@embroider/macros/src/node'; import resolvePackagePath from 'resolve-package-path'; @@ -793,10 +793,10 @@ export default class CompatApp { let movablePackageCache = new MovablePackageCache(this.macrosConfig, this.root); if (this.isDummy) { - let owningAddon = new OwningAddon(this.legacyEmberAppInstance.project.root, movablePackageCache); - movablePackageCache.seed(owningAddon); - movablePackageCache.seed(new DummyPackage(this.root, owningAddon, movablePackageCache)); - this.macrosConfig.enablePackageDevelopment(owningAddon.root); + movablePackageCache.seed( + new DummyPackage(this.root, this.legacyEmberAppInstance.project.root, movablePackageCache) + ); + this.macrosConfig.enablePackageDevelopment(this.legacyEmberAppInstance.project.root); } return movablePackageCache; } diff --git a/packages/compat/src/dummy-package.ts b/packages/compat/src/dummy-package.ts index 3449e51fb..f85ccdc66 100644 --- a/packages/compat/src/dummy-package.ts +++ b/packages/compat/src/dummy-package.ts @@ -5,13 +5,13 @@ import cloneDeep from 'lodash/cloneDeep'; // A specialized Package that represents a Dummy App (the app that comes along // with an addon for purposes of testing that addon). export default class DummyPackage extends Package { - constructor(root: string, private owningAddon: Package, packageCache: PackageCache) { + constructor(root: string, private owningAddonRoot: string, packageCache: PackageCache) { super(root, packageCache, true); } @Memoize() protected get internalPackageJSON() { - let pkg = cloneDeep(this.owningAddon.packageJSON); + let pkg = cloneDeep(this.packageCache.get(this.owningAddonRoot).packageJSON); pkg.name = 'dummy'; return pkg; } @@ -22,19 +22,8 @@ export default class DummyPackage extends Package { if (!deps) { deps = new Map(); } - deps.set(this.owningAddon.name, this.owningAddon); + const owningAddon = this.packageCache.get(this.owningAddonRoot); + deps.set(owningAddon.name, owningAddon); return deps; } } - -// A specialized Package that represents an Addon that owns the current Dummy -// App. It's special because it always supports rebuilds. -export class OwningAddon extends Package { - constructor(root: string, packageCache: PackageCache) { - super(root, packageCache, false); - } - - get mayRebuild(): boolean { - return true; - } -} From c9010b7c21c101cd37fc26cbe32eb6c11d16567e Mon Sep 17 00:00:00 2001 From: Chris Manson Date: Thu, 8 Jun 2023 14:39:03 +0100 Subject: [PATCH 23/72] remove seed from package-cache and set app explicitly on moveable-package-cache --- packages/compat/src/compat-app.ts | 4 +++- packages/compat/src/moved-package-cache.ts | 12 +++++++++++- packages/shared-internals/src/package-cache.ts | 7 ------- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/packages/compat/src/compat-app.ts b/packages/compat/src/compat-app.ts index f4c11a21b..8316c086f 100644 --- a/packages/compat/src/compat-app.ts +++ b/packages/compat/src/compat-app.ts @@ -793,10 +793,12 @@ export default class CompatApp { let movablePackageCache = new MovablePackageCache(this.macrosConfig, this.root); if (this.isDummy) { - movablePackageCache.seed( + movablePackageCache.setApp( new DummyPackage(this.root, this.legacyEmberAppInstance.project.root, movablePackageCache) ); this.macrosConfig.enablePackageDevelopment(this.legacyEmberAppInstance.project.root); + } else { + movablePackageCache.setApp(movablePackageCache.get(this.root)); } return movablePackageCache; } diff --git a/packages/compat/src/moved-package-cache.ts b/packages/compat/src/moved-package-cache.ts index b13dd57dd..e58354bec 100644 --- a/packages/compat/src/moved-package-cache.ts +++ b/packages/compat/src/moved-package-cache.ts @@ -13,13 +13,19 @@ function assertNoTildeExpansion(source: string, target: string) { } } export class MovablePackageCache extends PackageCache { + #appPackage: Package | undefined; + constructor(private macrosConfig: MacrosConfig, appRoot: string) { super(appRoot); } moveAddons(destDir: string): MovedPackageCache { // start with the plain old app package - let origApp = this.get(this.appRoot); + let origApp = this.#appPackage; + + if (!origApp) { + throw new Error('You must call setApp() on MovablePackageCache before calling moveAddons()'); + } // discover the set of all packages that will need to be moved into the // workspace @@ -27,6 +33,10 @@ export class MovablePackageCache extends PackageCache { return new MovedPackageCache(this.rootCache, this.resolutionCache, destDir, movedSet, origApp, this.macrosConfig); } + + setApp(pkg: Package) { + this.#appPackage = pkg; + } } export class MovedPackageCache extends PackageCache { diff --git a/packages/shared-internals/src/package-cache.ts b/packages/shared-internals/src/package-cache.ts index 1c2fb8e07..88ad4b24e 100644 --- a/packages/shared-internals/src/package-cache.ts +++ b/packages/shared-internals/src/package-cache.ts @@ -27,13 +27,6 @@ export default class PackageCache { return result; } - seed(pkg: Package) { - if (this.rootCache.has(pkg.root)) { - throw new Error(`bug: tried to seed package ${pkg.name} but it's already in packageCache`); - } - this.rootCache.set(pkg.root, pkg); - } - protected rootCache: Map = new Map(); protected resolutionCache: Map> = new Map(); From 4706d1e07a3715a654f2383bc804b6c974579fbd Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Mon, 12 Jun 2023 12:43:41 -0400 Subject: [PATCH 24/72] make stage2 distinguish both views into appPackage --- packages/compat/src/compat-app-builder.ts | 32 +++++++----- packages/compat/src/compat-app.ts | 24 ++++++--- packages/core/src/rewritten-package-cache.ts | 54 +++++++++++++------- packages/core/src/stage.ts | 5 -- packages/core/src/to-broccoli-plugin.ts | 7 +-- 5 files changed, 72 insertions(+), 50 deletions(-) diff --git a/packages/compat/src/compat-app-builder.ts b/packages/compat/src/compat-app-builder.ts index 8834018bc..04e21767d 100644 --- a/packages/compat/src/compat-app-builder.ts +++ b/packages/compat/src/compat-app-builder.ts @@ -64,7 +64,8 @@ export class CompatAppBuilder { constructor( private root: string, - private appPackage: Package, + private origAppPackage: Package, + private appPackageWithMovedDeps: Package, private options: Required, private compatApp: CompatApp, private configTree: V1Config, @@ -134,7 +135,7 @@ export class CompatAppBuilder { } } - private activeAddonChildren(pkg: Package = this.appPackage): AddonPackage[] { + private activeAddonChildren(pkg: Package): AddonPackage[] { let result = (pkg.dependencies.filter(this.isActiveAddon) as AddonPackage[]).filter( // When looking for child addons, we want to ignore 'peerDependencies' of // a given package, to align with how ember-cli resolves addons. So here @@ -142,7 +143,7 @@ export class CompatAppBuilder { // sections. addon => pkg.packageJSON.dependencies?.[addon.name] || pkg.packageJSON.devDependencies?.[addon.name] ); - if (pkg === this.appPackage) { + if (pkg === this.appPackageWithMovedDeps) { let extras = [this.synthVendor, this.synthStyles].filter(this.isActiveAddon) as AddonPackage[]; result = [...result, ...extras]; } @@ -151,7 +152,7 @@ export class CompatAppBuilder { @Memoize() private get allActiveAddons(): AddonPackage[] { - let result = this.appPackage.findDescendants(this.isActiveAddon) as AddonPackage[]; + let result = this.appPackageWithMovedDeps.findDescendants(this.isActiveAddon) as AddonPackage[]; let extras = [this.synthVendor, this.synthStyles].filter(this.isActiveAddon) as AddonPackage[]; let extraDescendants = flatMap(extras, dep => dep.findDescendants(this.isActiveAddon)) as AddonPackage[]; result = [...result, ...extras, ...extraDescendants]; @@ -160,8 +161,13 @@ export class CompatAppBuilder { @bind private isActiveAddon(pkg: Package): boolean { - // todo: filter by addon-provided hook - return pkg.isEmberPackage(); + // stage1 already took care of converting everything that's actually active + // into v2 addons. If it's not a v2 addon, we don't want it. + // + // We can encounter v1 addons here when there is inactive stuff floating + // around in the node_modules that accidentally satisfy something like an + // optional peer dep. + return pkg.isV2Addon(); } @bind @@ -247,7 +253,7 @@ export class CompatAppBuilder { @Memoize() private activeRules() { return activePackageRules(this.options.packageRules.concat(defaultAddonPackageRules()), [ - { name: this.appPackage.name, version: this.appPackage.version, root: this.root }, + { name: this.origAppPackage.name, version: this.origAppPackage.version, root: this.root }, ...this.allActiveAddons.filter(p => p.meta['auto-upgraded']), ]); } @@ -524,7 +530,7 @@ export class CompatAppBuilder { } } - html.insertStyleLink(html.styles, `assets/${this.appPackage.name}.css`); + html.insertStyleLink(html.styles, `assets/${this.origAppPackage.name}.css`); const parentEngine = appFiles.find(e => !e.parent) as Engine; let vendorJS = this.implicitScriptsAsset(prepared, parentEngine, emberENV); @@ -653,7 +659,7 @@ export class CompatAppBuilder { private partitionEngines(appJSPath: string): EngineSummary[] { let queue: EngineSummary[] = [ { - package: this.appPackage, + package: this.appPackageWithMovedDeps, addons: new Set(), parent: undefined, sourcePath: appJSPath, @@ -691,7 +697,7 @@ export class CompatAppBuilder { @Memoize() private get activeFastboot() { - return this.activeAddonChildren(this.appPackage).find(a => a.name === 'ember-cli-fastboot'); + return this.activeAddonChildren(this.appPackageWithMovedDeps).find(a => a.name === 'ember-cli-fastboot'); } @Memoize() @@ -969,7 +975,7 @@ export class CompatAppBuilder { } private combinePackageJSON(meta: AppMeta): object { - let pkgLayers: any[] = [this.appPackage.packageJSON]; + let pkgLayers: any[] = [this.origAppPackage.packageJSON]; let fastbootConfig = this.fastbootConfig; if (fastbootConfig) { // fastboot-specific package.json output is allowed to add to our original package.json @@ -1011,7 +1017,7 @@ export class CompatAppBuilder { @Memoize() private get portableHints(): PortableHint[] { return this.options.pluginHints.map(hint => { - let cursor = join(this.appPackage.root, 'package.json'); + let cursor = join(this.origAppPackage.root, 'package.json'); for (let i = 0; i < hint.resolve.length; i++) { let target = hint.resolve[i]; if (i < hint.resolve.length - 1) { @@ -1125,7 +1131,7 @@ export class CompatAppBuilder { private topAppJSAsset(engines: Engine[], prepared: Map): InternalAsset { let [app, ...childEngines] = engines; - let relativePath = `assets/${this.appPackage.name}.js`; + let relativePath = `assets/${this.origAppPackage.name}.js`; return this.appJSAsset(relativePath, app, childEngines, prepared, { autoRun: this.compatApp.autoRun, appBoot: !this.compatApp.autoRun ? this.compatApp.appBoot.readAppBoot() : '', diff --git a/packages/compat/src/compat-app.ts b/packages/compat/src/compat-app.ts index 8316c086f..ac86893a6 100644 --- a/packages/compat/src/compat-app.ts +++ b/packages/compat/src/compat-app.ts @@ -1,5 +1,5 @@ import { Node as BroccoliNode } from 'broccoli-node-api'; -import { PackageCache, WaitForTrees, Stage } from '@embroider/core'; +import { PackageCache, WaitForTrees, Stage, RewrittenPackageCache } from '@embroider/core'; import Options, { optionsWithDefaults } from './options'; import { Memoize } from 'typescript-memoize'; import { sync as pkgUpSync } from 'pkg-up'; @@ -821,10 +821,18 @@ export default class CompatApp { }; } - private async instantiate(root: string, appSrcDir: string, packageCache: PackageCache, configTree: V1Config) { + private async instantiate( + root: string, + appSrcDir: string, + packageCache: RewrittenPackageCache, + configTree: V1Config + ) { + let origAppPkg = packageCache.get(appSrcDir); + let movedAppPkg = packageCache.withRewrittenDeps(origAppPkg); return new CompatAppBuilder( root, - packageCache.get(appSrcDir), + origAppPkg, + movedAppPkg, this.options, this, configTree, @@ -834,16 +842,18 @@ export default class CompatApp { } asStage(prevStage: Stage): Stage { - let resolve: (result: { packageCache: PackageCache; outputPath: string }) => void; - let promise: Promise<{ packageCache: PackageCache; outputPath: string }> = new Promise(r => (resolve = r)); + let resolve: (result: { outputPath: string }) => void; + let promise: Promise<{ outputPath: string }> = new Promise(r => (resolve = r)); let tree = () => { let inTrees = this.inTrees(prevStage.tree); return new WaitForTrees(inTrees, this.annotation, async treePaths => { if (!this.active) { - let { outputPath, packageCache } = await prevStage.ready(); + let { outputPath } = await prevStage.ready(); + // TODO: this will use shared caches once we refactor out MovedPackageCache + let packageCache = new RewrittenPackageCache(new PackageCache(this.root)); this.active = await this.instantiate(outputPath, prevStage.inputPath, packageCache, inTrees.configTree); - resolve({ packageCache, outputPath }); + resolve({ outputPath }); } await this.active.build(treePaths); }); diff --git a/packages/core/src/rewritten-package-cache.ts b/packages/core/src/rewritten-package-cache.ts index 55c0e0356..3c399300b 100644 --- a/packages/core/src/rewritten-package-cache.ts +++ b/packages/core/src/rewritten-package-cache.ts @@ -66,7 +66,7 @@ export class RewrittenPackageCache implements PackageCacheTheGoodParts { } // ensure we have the moved version of the package - private maybeMoved(pkg: Package): Package { + maybeMoved(pkg: Package): Package { let newRoot = this.index.oldToNew.get(pkg.root); if (newRoot) { return this.get(newRoot); @@ -86,6 +86,17 @@ export class RewrittenPackageCache implements PackageCacheTheGoodParts { } } + // given any package, give us a new representation of it where its deps are + // replaced with rewritten versions of those deps, as needed. + withRewrittenDeps(pkg: Package): Package { + let found = wrapped.get(pkg); + if (!found) { + found = new WrappedPackage(this, pkg); + wrapped.set(pkg, found); + } + return castToPackage(found); + } + ownerOfFile(filename: string): Package | undefined { let owner = this.plainCache.ownerOfFile(filename); if (owner) { @@ -133,12 +144,7 @@ export class RewrittenPackageCache implements PackageCacheTheGoodParts { private maybeWrap(pkg: Package) { let oldRoot = this.index.newToOld.get(pkg.root); if (oldRoot) { - let found = wrapped.get(pkg); - if (!found) { - found = new MovedPackage(this, pkg); - wrapped.set(pkg, found); - } - return castToPackage(found); + return this.withRewrittenDeps(pkg); } else { return pkg; } @@ -157,27 +163,32 @@ export class RewrittenPackageCache implements PackageCacheTheGoodParts { } const shared: Map = new Map(); -const wrapped = new WeakMap(); +const wrapped = new WeakMap(); // TODO: as our refactor lands we should be able to remove this from Package // itself. type PackageTheGoodParts = Omit, 'nonResolvableDeps'>; // TODO: this goes with the above TODO and can get deleted when it does. -function castToPackage(m: MovedPackage): Package { +function castToPackage(m: WrappedPackage): Package { return m as unknown as Package; } -class MovedPackage implements PackageTheGoodParts { - // plainPkg is not the Package in the original un-moved location, it's the - // plain representation of the moved location. That is, when you grab - // plainPkg.root it will show the new location, and plainPkg.packageJSON shows - // the rewritten package.json contained there. +class WrappedPackage implements PackageTheGoodParts { + // Questions about *this* package will be answered based on the given + // plainPkg. + // + // Questions about *this package's deps* will be translated through the set of + // moved packages. // - // The point of MovedPackage is to finesse the parts of plainPkg that wouldn't - // be correct if you used them directly. For example, plainPkg.dependencies - // won't necessarily even work, because if you try to resolve them from the - // moved location they might not be there. + // There are two different cases that this enables. The first is when we're + // representing a package that has itself been rewritten, in which case + // plainPkg points at the *rewritten* copy of the package, so that we see our + // own rewritten package.json, etc. The second case is in Stage2 when the + // dependencies have been rewritten but the app has not -- we represent the + // app as a WrappedPackage where plainPkg is the *original* app package, so + // we're still seeing the original package.json, etc, but while also seeing + // the rewritten addons. constructor(private packageCache: RewrittenPackageCache, private plainPkg: Package) {} get root() { @@ -261,7 +272,12 @@ class MovedPackage implements PackageTheGoodParts { return this.plainPkg.dependencyNames .map(name => { try { - return this.packageCache.resolve(name, castToPackage(this)); + // when this.plainPkg was itself moved, the result from resolve() is + // already a moved package if that dep was moved. In that case, the + // maybeMoved() is not needed. But when this.plainPkg is not moved and + // wants to see moved deps (which is the case for the app package in + // stage2), we do need the final maybeMoved() call to adjust them. + return this.packageCache.maybeMoved(this.packageCache.resolve(name, castToPackage(this))); } catch (error) { // if the package was not found do not error out here. this is relevant // for the case where a package might be an optional peerDependency and we dont diff --git a/packages/core/src/stage.ts b/packages/core/src/stage.ts index 8a9624ce6..b5c6e8616 100644 --- a/packages/core/src/stage.ts +++ b/packages/core/src/stage.ts @@ -1,5 +1,4 @@ import type { Node } from 'broccoli-node-api'; -import { PackageCache } from '@embroider/shared-internals'; // A build Stage is _kinda_ like a Broccoli transform, and it interoperates with // Broccoli, but it takes a different approach to how stages combine. @@ -27,9 +26,5 @@ export default interface Stage { // This is the actual directory in which the output will be. It's guaranteed // to not change once you get it. readonly outputPath: string; - - // Stages must propagate their PackageCache forward to the next stage so we - // don't repeat a lot of resolving work. - readonly packageCache: PackageCache; }>; } diff --git a/packages/core/src/to-broccoli-plugin.ts b/packages/core/src/to-broccoli-plugin.ts index a08af37ed..64a4cbd59 100644 --- a/packages/core/src/to-broccoli-plugin.ts +++ b/packages/core/src/to-broccoli-plugin.ts @@ -22,12 +22,7 @@ export default function toBroccoliPlugin( async build() { if (!this.packager) { - let { outputPath, packageCache } = await this.stage.ready(); - // We always register a shared stage3 packageCache so it can be used by - // things like babel plugins and template compilers. - if (packageCache) { - packageCache.shareAs('embroider-stage3'); - } + let { outputPath } = await this.stage.ready(); this.packager = new packagerClass( outputPath, this.outputPath, From 98c3e9ec45dc804e3584b32d362ee6780022ae8a Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Mon, 12 Jun 2023 13:00:36 -0400 Subject: [PATCH 25/72] stage2 explicit dummy app support --- packages/compat/src/compat-app.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/compat/src/compat-app.ts b/packages/compat/src/compat-app.ts index ac86893a6..46d501728 100644 --- a/packages/compat/src/compat-app.ts +++ b/packages/compat/src/compat-app.ts @@ -827,7 +827,16 @@ export default class CompatApp { packageCache: RewrittenPackageCache, configTree: V1Config ) { - let origAppPkg = packageCache.get(appSrcDir); + let origAppPkg; + if (this.isDummy) { + origAppPkg = new DummyPackage( + this.root, + this.legacyEmberAppInstance.project.root, + packageCache as unknown as PackageCache // TODO: cast won't be needed when refactor is complete + ); + } else { + origAppPkg = packageCache.get(appSrcDir); + } let movedAppPkg = packageCache.withRewrittenDeps(origAppPkg); return new CompatAppBuilder( root, From bf4d6453103de7e9cdf4cb9d78e85d6143be4a1e Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Mon, 12 Jun 2023 13:47:53 -0400 Subject: [PATCH 26/72] fix app's nonResolvableDeps support --- packages/compat/src/moved-package-cache.ts | 8 ++++++++ packages/core/src/rewritten-package-cache.ts | 14 +++++++------- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/packages/compat/src/moved-package-cache.ts b/packages/compat/src/moved-package-cache.ts index e58354bec..9aa17496d 100644 --- a/packages/compat/src/moved-package-cache.ts +++ b/packages/compat/src/moved-package-cache.ts @@ -113,6 +113,14 @@ export class MovedPackageCache extends PackageCache { content.extraResolutions[newPkg.root] = [...nonResolvableDeps.values()].map(v => v.root); } } + + // we aren't listing the app itself in the index but we need to list any + // extraResolutions it needs to find its nonresolvable deps + let nonResolvableDeps = this.app.nonResolvableDeps; + if (nonResolvableDeps) { + content.extraResolutions[this.app.root] = [...nonResolvableDeps.values()].map(v => v.root); + } + outputJSONSync(indexFile, content, { spaces: 2 }); } diff --git a/packages/core/src/rewritten-package-cache.ts b/packages/core/src/rewritten-package-cache.ts index 3c399300b..fae98b90d 100644 --- a/packages/core/src/rewritten-package-cache.ts +++ b/packages/core/src/rewritten-package-cache.ts @@ -39,13 +39,6 @@ export class RewrittenPackageCache implements PackageCacheTheGoodParts { } resolve(packageName: string, fromPackage: Package): Package { - let oldRoot = this.index.newToOld.get(fromPackage.root); - if (!oldRoot) { - // the fromPackage has not been moved, so we're just providing the plain - // behavior. - return this.plainCache.resolve(packageName, fromPackage); - } - // check for any extraResolutions let extraResolutions = this.index.extraResolutions.get(fromPackage.root); if (extraResolutions) { @@ -57,6 +50,13 @@ export class RewrittenPackageCache implements PackageCacheTheGoodParts { } } + let oldRoot = this.index.newToOld.get(fromPackage.root); + if (!oldRoot) { + // the fromPackage has not been moved, so we're just providing the plain + // behavior. + return this.plainCache.resolve(packageName, fromPackage); + } + // do the real resolving from the old location let oldSrc = this.plainCache.get(oldRoot); let oldDest = this.plainCache.resolve(packageName, oldSrc); From 7f83c9407b6e7367445fbdfc6a400bdbcfdc6fa4 Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Tue, 13 Jun 2023 11:00:31 -0400 Subject: [PATCH 27/72] drop moved packaged cache This will break many tests, next step is to work through them, the very next thing is failure to resolve ember-template-compiler. --- packages/compat/src/compat-addons.ts | 236 ++--------- packages/compat/src/compat-app.ts | 23 +- packages/compat/src/moved-package-cache.ts | 367 ------------------ packages/compat/src/standalone-addon-build.ts | 73 ++++ 4 files changed, 102 insertions(+), 597 deletions(-) delete mode 100644 packages/compat/src/moved-package-cache.ts create mode 100644 packages/compat/src/standalone-addon-build.ts diff --git a/packages/compat/src/compat-addons.ts b/packages/compat/src/compat-addons.ts index 77722cde9..b7f1d586a 100644 --- a/packages/compat/src/compat-addons.ts +++ b/packages/compat/src/compat-addons.ts @@ -1,14 +1,10 @@ import { Node } from 'broccoli-node-api'; -import { join, relative, dirname, isAbsolute, sep } from 'path'; -import { emptyDirSync, ensureSymlinkSync, ensureDirSync, realpathSync, copySync, writeJSONSync } from 'fs-extra'; -import { Stage, Package, PackageCache, WaitForTrees, mangledEngineRoot } from '@embroider/core'; -import V1InstanceCache from './v1-instance-cache'; -import { MovedPackageCache } from './moved-package-cache'; -import { Memoize } from 'typescript-memoize'; -import buildCompatAddon from './build-compat-addon'; +import { resolve } from 'path'; +import { ensureDirSync, realpathSync } from 'fs-extra'; +import { Stage, WaitForTrees } from '@embroider/core'; import TreeSync from 'tree-sync'; -import { WatchedDir } from 'broccoli-source'; import CompatApp from './compat-app'; +import { convertLegacyAddons } from './standalone-addon-build'; // This build stage expects to be run with broccoli memoization enabled in order // to get good rebuild performance. We turn it on by default here, but you can @@ -23,238 +19,52 @@ if (typeof process.env.BROCCOLI_ENABLED_MEMOIZE === 'undefined') { export default class CompatAddons implements Stage { private didBuild = false; - private destDir: string; - private packageCache: MovedPackageCache; - private treeSyncMap: WeakMap; - private v1Cache: V1InstanceCache; + private treeSync: TreeSync | undefined; readonly inputPath: string; - constructor(private compatApp: CompatApp) { + private destDir: string; + private addons: Node; + + constructor(compatApp: CompatApp) { // we want this to be stable across builds, because it becomes part of the // path to all of the files that the stage3 packager sees, and we want to // benefit from on-disk caching in stage3 packagers. ensureDirSync(compatApp.options.workspaceDir!); this.destDir = realpathSync(compatApp.options.workspaceDir!); - - let movablePackageCache = compatApp.makePackageCache(); - this.packageCache = movablePackageCache.moveAddons(this.destDir); + this.addons = convertLegacyAddons(compatApp); this.inputPath = compatApp.root; - this.treeSyncMap = new WeakMap(); - this.v1Cache = new V1InstanceCache(compatApp, movablePackageCache); } get tree(): Node { - let movedAddons = [...this.packageCache.moved.keys()].map(oldPkg => buildCompatAddon(oldPkg, this.v1Cache)); - - // these get watched so that EMBROIDER_REBUILD_ADDONS will still work - // correctly, even for v2 addons that have no v1 addon deps and therefore - // don't need to be moved. We don't consume these trees in our build step, - // we only do this to trigger rebuilds to happen. - let watchedUnmovedAddons = [...this.packageCache.unmovedAddons] - .filter(pkg => pkg.mayRebuild) - .map(pkg => new WatchedDir(pkg.root)); - - let { synthVendor, synthStyles } = this.getSyntheticPackages(this.compatApp, movedAddons); - return new WaitForTrees( - { movedAddons, synthVendor, synthStyles, watchedUnmovedAddons }, - '@embroider/compat/addons', - this.build.bind(this) - ); + return new WaitForTrees({ addons: this.addons }, '@embroider/compat/addons', this.build.bind(this)); } - async ready(): Promise<{ outputPath: string; packageCache: PackageCache }> { - await this.deferReady.promise; - writeJSONSync(join(this.destDir, '.embroider-reuse.json'), { - appDestDir: relative(this.destDir, this.packageCache.appRoot), - }); + async ready(): Promise<{ outputPath: string }> { return { - outputPath: this.packageCache.appRoot, - packageCache: this.packageCache, + outputPath: this.destDir, }; } - private get appDestDir(): string { - return this.packageCache.appRoot; - } - - private get app(): Package { - return this.packageCache.app; - } - private async build( { - movedAddons, - synthVendor, - synthStyles, + addons, }: { - movedAddons: string[]; - synthVendor: string; - synthStyles: string; + addons: string; }, changedMap: Map ) { - // empty the directory only on the first pass - if (!this.didBuild) { - emptyDirSync(this.destDir); - } - - [...this.packageCache.moved.entries()].forEach(([oldPkg, newPkg], index) => { - let treeInstance = this.treeSyncMap.get(newPkg); - - // we need to pull metadata off the oldPkg, not the newPkg, because the - // newPkg doesn't actually have anything in it yet (including - // package.json) - let isEngine = oldPkg.isEngine(); - - // Engines get built not into their real package name, but a mangled one. - // Their real one needs to be free for us to merge all their dependencies - // into. - let destination = isEngine ? mangledEngineRoot(newPkg) : newPkg.root; - - if (!treeInstance) { - let ignore = ['**/node_modules']; - - let rel = relative(destination, this.appDestDir); - if (!rel.startsWith('..') && !isAbsolute(rel)) { - // the app is inside our addon. We must not copy the app as part of - // the addon, because that would overwrite the real app build. - ignore.push(rel); - - if (rel === `tests${sep}dummy`) { - // special case: classic dummy apps are weird because they put the - // tests (which are truly part of the app, not the addon) inside the - // addon instead of inside the app. - ignore.push('tests'); - } - } - - treeInstance = new TreeSync(movedAddons[index], destination, { - ignore, - }); - - this.treeSyncMap.set(newPkg, treeInstance); - } - - if ( - !this.didBuild || // always copy on the first build - (newPkg.mayRebuild && changedMap.get(movedAddons[index])) - ) { - treeInstance.sync(); - if (!this.didBuild && isEngine) { - // The first time we encounter an engine, we also create the empty - // shell for its real module namespace. - copySync(join(destination, 'package.json'), join(newPkg.root, 'package.json')); - } - } - }); - - // this has to be a separate pass over the packages because - // linkNonCopiedDeps resolves dependencies, so we want all the packages - // already in their new places before they start trying to resolve each - // other. - [...this.packageCache.moved.values()].forEach((newPkg, index) => { - if ( - !this.didBuild || // always copy on the first build - (newPkg.mayRebuild && changedMap.get(movedAddons[index])) - ) { - // for engines, this isn't the mangled destination (we don't need - // resolvable node_modules there), this is the empty shell of their real - // location - this.linkNonCopiedDeps(newPkg, newPkg.root); - } - }); - - this.linkNonCopiedDeps(this.app, this.appDestDir); - await this.packageCache.updatePreexistingResolvableSymlinks(); - - if (changedMap && changedMap.get(synthVendor)) { - copySync(synthVendor, join(this.appDestDir, 'node_modules', '@embroider', 'synthesized-vendor'), { - dereference: true, - overwrite: true, - }); - } - - if (changedMap && changedMap.get(synthStyles)) { - copySync(synthStyles, join(this.appDestDir, 'node_modules', '@embroider', 'synthesized-styles'), { - dereference: true, - overwrite: true, + if (!this.treeSync) { + this.treeSync = new TreeSync(addons, resolve(this.inputPath, 'node_modules/.embroider/rewritten-packages'), { + ignore: ['**/node_modules'], }); } - if (!this.didBuild) { - this.handleNonResolvableDeps(); + if ( + !this.didBuild || // always copy on the first build + changedMap.get(addons) + ) { + this.treeSync.sync(); } this.didBuild = true; - this.deferReady.resolve(); - } - - private handleNonResolvableDeps() { - // non-resolvable deps in addons - for (let [oldPkg, newPkg] of this.packageCache.moved.entries()) { - if (!oldPkg.nonResolvableDeps) { - continue; - } - for (let dep of oldPkg.nonResolvableDeps.values()) { - let moved = this.packageCache.moved.get(dep); - if (moved) { - dep = moved; - } - let target = join(newPkg.root, 'node_modules', dep.name); - ensureDirSync(dirname(target)); - ensureSymlinkSync(dep.root, target, 'dir'); - } - } - // non-resolvable deps in app - if (this.packageCache.app.nonResolvableDeps) { - for (let dep of this.packageCache.app.nonResolvableDeps.values()) { - let moved = this.packageCache.moved.get(dep); - if (moved) { - dep = moved; - } - let target = join(this.appDestDir, 'node_modules', dep.name); - ensureDirSync(dirname(target)); - ensureSymlinkSync(dep.root, target, 'dir'); - } - } - } - - private getSyntheticPackages(v1App: CompatApp, movedAddons: Node[]): { synthVendor: Node; synthStyles: Node } { - let index = 0; - let upgradedAddonTrees = []; - for (let [oldPkg] of this.packageCache.moved.entries()) { - if (!oldPkg.isV2Ember()) { - upgradedAddonTrees.push(movedAddons[index]); - } - index++; - } - return { - synthVendor: v1App.synthesizeVendorPackage(upgradedAddonTrees), - synthStyles: v1App.synthesizeStylesPackage(upgradedAddonTrees), - }; - } - - @Memoize() - private get deferReady() { - let resolve: Function; - let promise: Promise = new Promise(r => (resolve = r)); - return { resolve: resolve!, promise }; - } - - @Memoize() - private isMoved(pkg: Package) { - for (let candidate of this.packageCache.moved.values()) { - if (candidate === pkg) { - return true; - } - } - return false; - } - - private linkNonCopiedDeps(pkg: Package, destRoot: string) { - for (let dep of pkg.dependencies) { - if (!this.isMoved(dep)) { - ensureSymlinkSync(dep.root, join(destRoot, 'node_modules', dep.packageJSON.name)); - } - } } } diff --git a/packages/compat/src/compat-app.ts b/packages/compat/src/compat-app.ts index 46d501728..82e08b02c 100644 --- a/packages/compat/src/compat-app.ts +++ b/packages/compat/src/compat-app.ts @@ -25,7 +25,6 @@ import prepHtmlbarsAstPluginsForUnwrap from './prepare-htmlbars-ast-plugins'; import { readFileSync } from 'fs'; import type { Options as HTMLBarsOptions } from 'ember-cli-htmlbars'; import semver from 'semver'; -import { MovablePackageCache } from './moved-package-cache'; import type { Transform } from 'babel-plugin-ember-template-compilation'; import { CompatAppBuilder } from './compat-app-builder'; @@ -789,20 +788,6 @@ export default class CompatApp { }); } - makePackageCache(): MovablePackageCache { - let movablePackageCache = new MovablePackageCache(this.macrosConfig, this.root); - - if (this.isDummy) { - movablePackageCache.setApp( - new DummyPackage(this.root, this.legacyEmberAppInstance.project.root, movablePackageCache) - ); - this.macrosConfig.enablePackageDevelopment(this.legacyEmberAppInstance.project.root); - } else { - movablePackageCache.setApp(movablePackageCache.get(this.root)); - } - return movablePackageCache; - } - private inTrees(prevStageTree: BroccoliNode) { let publicTree = this.publicTree; let configTree = this.config; @@ -845,8 +830,12 @@ export default class CompatApp { this.options, this, configTree, - packageCache.get(join(root, 'node_modules', '@embroider', 'synthesized-vendor')), - packageCache.get(join(root, 'node_modules', '@embroider', 'synthesized-styles')) + packageCache.get( + join(origAppPkg.root, 'node_modules', '.embroider', 'rewritten-packages', '@embroider', 'synthesized-vendor') + ), + packageCache.get( + join(origAppPkg.root, 'node_modules', '.embroider', 'rewritten-packages', '@embroider', 'synthesized-styles') + ) ); } diff --git a/packages/compat/src/moved-package-cache.ts b/packages/compat/src/moved-package-cache.ts deleted file mode 100644 index 9aa17496d..000000000 --- a/packages/compat/src/moved-package-cache.ts +++ /dev/null @@ -1,367 +0,0 @@ -import { join, sep, isAbsolute, resolve } from 'path'; -import { ensureSymlinkSync, readdirSync, realpathSync, lstatSync, outputJSONSync } from 'fs-extra'; -import { Memoize } from 'typescript-memoize'; -import { PackageCache, Package, getOrCreate, RewrittenPackageIndex } from '@embroider/core'; -import { MacrosConfig } from '@embroider/macros/src/node'; -import os from 'os'; - -function assertNoTildeExpansion(source: string, target: string) { - if (target.includes('~') && os.platform() !== 'win32') { - throw new Error( - `The symbolic link: ${source}'s target: ${target} contained a bash expansion '~' which is not supported.` - ); - } -} -export class MovablePackageCache extends PackageCache { - #appPackage: Package | undefined; - - constructor(private macrosConfig: MacrosConfig, appRoot: string) { - super(appRoot); - } - - moveAddons(destDir: string): MovedPackageCache { - // start with the plain old app package - let origApp = this.#appPackage; - - if (!origApp) { - throw new Error('You must call setApp() on MovablePackageCache before calling moveAddons()'); - } - - // discover the set of all packages that will need to be moved into the - // workspace - let movedSet = new MovedSet(origApp); - - return new MovedPackageCache(this.rootCache, this.resolutionCache, destDir, movedSet, origApp, this.macrosConfig); - } - - setApp(pkg: Package) { - this.#appPackage = pkg; - } -} - -export class MovedPackageCache extends PackageCache { - readonly app!: Package; - private commonSegmentCount: number; - readonly moved: Map = new Map(); - readonly unmovedAddons: Set; - - constructor( - rootCache: PackageCache['rootCache'], - resolutionCache: PackageCache['resolutionCache'], - private destDir: string, - movedSet: MovedSet, - private origApp: Package, - private macrosConfig: MacrosConfig - ) { - // this is the initial appRoot, which we can't know until just below here - super('not-the-real-root'); - - // that gives us our common segment count, which enables localPath mapping - this.commonSegmentCount = movedSet.commonSegmentCount; - - // so we can now determine where the app will go inside the workspace. THIS - // is where we fix 'not-the-real-root' from above. - this.appRoot = this.localPath(origApp.root); - - this.macrosConfig.packageMoved(origApp.root, this.appRoot); - - for (let originalPkg of movedSet.packages) { - // Update our rootCache so we don't need to rediscover moved packages - let movedPkg; - if (originalPkg === origApp) { - // this wraps the original app package with one that will use moved - // dependencies. The app itself hasn't moved yet, which is why a proxy - // is needed at this level. - movedPkg = packageProxy(origApp, (pkg: Package) => this.moved.get(pkg) || pkg); - this.app = movedPkg; - rootCache.set(movedPkg.root, movedPkg); - } else { - movedPkg = this.movedPackage(originalPkg); - this.moved.set(originalPkg, movedPkg); - this.macrosConfig.packageMoved(originalPkg.root, movedPkg.root); - } - - // Update our resolutionCache so we still know as much about the moved - // packages as we did before we moved them, without redoing package - // resolution. - let resolutions = new Map(); - for (let dep of originalPkg.dependencies) { - if (movedSet.packages.has(dep)) { - resolutions.set(dep.name, this.movedPackage(dep)); - } else { - resolutions.set(dep.name, dep); - } - } - resolutionCache.set(movedPkg, resolutions); - } - this.rootCache = rootCache; - this.resolutionCache = resolutionCache; - this.unmovedAddons = movedSet.unmovedAddons; - this.writeAddonIndex(); - } - - private writeAddonIndex() { - let indexFile = resolve(this.origApp.root, 'node_modules', '.embroider', 'rewritten-packages', 'index.json'); - let content: RewrittenPackageIndex = { - packages: {}, - extraResolutions: {}, - }; - for (let [oldPkg, newPkg] of this.moved) { - content.packages[oldPkg.root] = newPkg.root; - let nonResolvableDeps = oldPkg.nonResolvableDeps; - if (nonResolvableDeps) { - content.extraResolutions[newPkg.root] = [...nonResolvableDeps.values()].map(v => v.root); - } - } - - // we aren't listing the app itself in the index but we need to list any - // extraResolutions it needs to find its nonresolvable deps - let nonResolvableDeps = this.app.nonResolvableDeps; - if (nonResolvableDeps) { - content.extraResolutions[this.app.root] = [...nonResolvableDeps.values()].map(v => v.root); - } - - outputJSONSync(indexFile, content, { spaces: 2 }); - } - - private movedPackage(originalPkg: Package): Package { - let newRoot = this.localPath(originalPkg.root); - return getOrCreate(this.rootCache, newRoot, () => new (originalPkg.constructor as any)(newRoot, this, false)); - } - - private localPath(filename: string) { - return join(this.destDir, ...pathSegments(filename).slice(this.commonSegmentCount)); - } - - // hunt for symlinks that may be needed to do node_modules resolution from the - // given path. - async updatePreexistingResolvableSymlinks(): Promise { - let roots = this.originalRoots(); - [...this.candidateDirs()].map(path => { - let links = symlinksInNodeModules(path); - for (let { source, target } of links) { - let pkg = roots.get(target); - if (pkg) { - // we found a symlink that points at a package that was copied. - // Replicate it in the new structure pointing at the new package. - ensureSymlinkSync(pkg.root, this.localPath(source)); - } - } - }); - } - - // places that might have symlinks we need to mimic - private candidateDirs(): Set { - let candidates = new Set() as Set; - let originalPackages = [this.origApp, ...this.moved.keys()]; - for (let pkg of originalPackages) { - let segments = pathSegments(pkg.root); - - let candidate = join(pkg.root, 'node_modules'); - candidates.add(candidate); - - for (let i = segments.length - 1; i >= this.commonSegmentCount; i--) { - if (segments[i - 1] !== 'node_modules') { - let candidate = '/' + join(...segments.slice(0, i), 'node_modules'); - if (candidates.has(candidate)) { - break; - } - candidates.add(candidate); - } - } - } - return candidates; - } - - private originalRoots(): Map { - let originalRoots = new Map(); - for (let [originalPackage, movedPackage] of this.moved.entries()) { - originalRoots.set(originalPackage.root, movedPackage); - } - return originalRoots; - } -} - -function maybeReaddirSync(path: string) { - try { - return readdirSync(path); - } catch (err) { - if (err.code !== 'ENOTDIR' && err.code !== 'ENOENT') { - throw err; - } - return []; - } -} - -function isSymlink(path: string): boolean { - try { - let stat = lstatSync(path); - return stat.isSymbolicLink(); - } catch (err) { - if (err.code !== 'ENOTDIR' && err.code !== 'ENOENT') { - throw err; - } - - return false; - } -} - -function symlinksInNodeModules(path: string): { source: string; target: string }[] { - let results: { source: string; target: string }[] = []; - - // handles the full `node_modules` being symlinked (this is uncommon, but sometimes - // be useful for test harnesses to avoid multiple `npm install` invocations) - let parentIsSymlink = isSymlink(path); - - let names = maybeReaddirSync(path); - - names.map(name => { - let source = join(path, name); - let stats = lstatSync(source); - if (parentIsSymlink || stats.isSymbolicLink()) { - let target = realpathSync(source); - assertNoTildeExpansion(source, target); - - results.push({ source, target }); - } else if (stats.isDirectory() && name.startsWith('@')) { - // handle symlinked scope names (e.g. symlinking `@myorghere` to a shared location) - let isSourceSymlink = isSymlink(source); - let innerNames = maybeReaddirSync(source); - - innerNames.map(innerName => { - let innerSource = join(source, innerName); - let innerStats = lstatSync(innerSource); - if (parentIsSymlink || isSourceSymlink || innerStats.isSymbolicLink()) { - let target = realpathSync(innerSource); - assertNoTildeExpansion(innerSource, target); - - results.push({ source: innerSource, target }); - } - }); - } - }); - - return results; -} - -function pathSegments(filename: string) { - let segments = filename.split(sep); - if (isAbsolute(filename)) { - segments.shift(); - } - return segments; -} - -class MovedSet { - private mustMove: Map = new Map(); - unmovedAddons: Set = new Set(); - - constructor(private app: Package) { - this.check(app); - } - - private check(pkg: Package): boolean { - if (this.mustMove.has(pkg)) { - return this.mustMove.get(pkg)!; - } - - // non-ember packages don't need to move - if (pkg !== this.app && !pkg.isEmberPackage()) { - this.mustMove.set(pkg, false); - return false; - } - - let mustMove = - // The app always moves (because we need a place to mash all the - // addon-provided "app-js" trees), - pkg === this.app || - // For the same reason, engines need to move (we need a place to mash all - // their child addon's provided app-js trees into) - pkg.isEngine() || - // any other ember package that isn't native v2 must move because we've - // got to rewrite them - !pkg.isV2Ember(); - - // this is a partial answer. After we check our children, our own `mustMove` - // may change from false to true. But it's OK that our children see false in - // that case, because they don't need to move on our behalf. - // - // We need to already be in the `this.mustMove` cache at this moment in - // order to avoid infinite recursion if any of our children end up depending - // back on us. - this.mustMove.set(pkg, mustMove); - - for (let dep of pkg.dependencies) { - // or if any of your deps need to move - mustMove = this.check(dep) || mustMove; - } - this.mustMove.set(pkg, mustMove); - - if (!mustMove) { - this.unmovedAddons.add(pkg); - } - - return mustMove; - } - - @Memoize() - get packages(): Set { - let result = new Set() as Set; - for (let [pkg, mustMove] of this.mustMove) { - if (mustMove) { - result.add(pkg); - } - } - return result; - } - - // the npm structure we're shadowing could have a dependency nearly anywhere - // on disk. We want to maintain their relations to each other. So we must find - // the point in the filesystem that contains all of them, which could even be - // "/" (for example, if you npm-linked a dependency that lives in /tmp). - // - // The commonSegmentCount is how many leading path segments are shared by all - // our packages. - @Memoize() - get commonSegmentCount(): number { - return [...this.packages].reduce((longestPrefix, pkg) => { - let candidate = pathSegments(pkg.root); - let shorter, longer; - if (longestPrefix.length > candidate.length) { - shorter = candidate; - longer = longestPrefix; - } else { - shorter = longestPrefix; - longer = candidate; - } - let i = 0; - for (; i < shorter.length; i++) { - if (shorter[i] !== longer[i]) { - break; - } - } - return shorter.slice(0, i); - }, pathSegments(this.app.root)).length; - } -} - -function packageProxy(pkg: Package, getMovedPackage: (pkg: Package) => Package) { - let p: Package = new Proxy(pkg, { - get(pkg: Package, prop: string | number | symbol) { - if (prop === 'dependencies') { - return pkg.dependencies.map(getMovedPackage); - } - if (prop === 'nonResolvableDeps') { - if (!pkg.nonResolvableDeps) { - return pkg.nonResolvableDeps; - } - return new Map([...pkg.nonResolvableDeps.values()].map(dep => [dep.name, getMovedPackage(dep)])); - } - if (prop === 'findDescendants') { - return pkg.findDescendants.bind(p); - } - return (pkg as any)[prop]; - }, - }); - return p; -} diff --git a/packages/compat/src/standalone-addon-build.ts b/packages/compat/src/standalone-addon-build.ts new file mode 100644 index 000000000..cb116caad --- /dev/null +++ b/packages/compat/src/standalone-addon-build.ts @@ -0,0 +1,73 @@ +import { Package, PackageCache, RewrittenPackageIndex } from '@embroider/core'; +import V1InstanceCache from './v1-instance-cache'; +import buildCompatAddon from './build-compat-addon'; +import { Funnel } from 'broccoli-funnel'; +import crypto from 'crypto'; +import broccoliMergeTrees from 'broccoli-merge-trees'; +import writeFile from 'broccoli-file-creator'; +import type { Node } from 'broccoli-node-api'; +import CompatApp from './compat-app'; + +export function convertLegacyAddons(compatApp: CompatApp) { + let packageCache = PackageCache.shared('embroider', compatApp.root); + let instanceCache = new V1InstanceCache(compatApp, packageCache); + + let v1Addons = findV1Addons(packageCache.get(compatApp.root)); + let index = buildAddonIndex(v1Addons); + + let interiorTrees: Node[] = []; + let exteriorTrees = [...v1Addons].map(pkg => { + let interior = buildCompatAddon(pkg, instanceCache); + interiorTrees.push(interior); + return new Funnel(interior, { destDir: index.packages[pkg.root] }); + }); + + return broccoliMergeTrees([ + ...exteriorTrees, + new Funnel(compatApp.synthesizeStylesPackage(interiorTrees), { + destDir: '@embroider/synthesized-styles', + }), + new Funnel(compatApp.synthesizeVendorPackage(interiorTrees), { + destDir: '@embroider/synthesized-vendor', + }), + writeFile('index.json', JSON.stringify(buildAddonIndex(v1Addons), null, 2)), + ]); +} + +function buildAddonIndex(packages: Set): RewrittenPackageIndex { + let content: RewrittenPackageIndex = { + packages: {}, + extraResolutions: {}, + }; + for (let oldPkg of packages) { + let newRoot = `${oldPkg.name}.${hashed(oldPkg.root)}`; + content.packages[oldPkg.root] = `${oldPkg.name}.${hashed(oldPkg.root)}`; + let nonResolvableDeps = oldPkg.nonResolvableDeps; + if (nonResolvableDeps) { + content.extraResolutions[newRoot] = [...nonResolvableDeps.values()].map(v => v.root); + } + } + + return content; +} + +function findV1Addons(pkg: Package, seen: Set = new Set(), output: Set = new Set()): Set { + for (let dep of pkg.dependencies) { + if (seen.has(dep)) { + continue; + } + seen.add(dep); + if (dep.isEmberPackage()) { + if (!dep.isV2Addon()) { + output.add(dep); + } + findV1Addons(dep, seen, output); + } + } + return output; +} + +function hashed(path: string): string { + let h = crypto.createHash('sha1'); + return h.update(path).digest('hex').slice(0, 8); +} From 5752308ff431a990a6072d676d9499c93d434c78 Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Tue, 13 Jun 2023 14:58:30 -0400 Subject: [PATCH 28/72] resolve ember-template-compiler through module-resolver --- packages/compat/src/compat-app-builder.ts | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/packages/compat/src/compat-app-builder.ts b/packages/compat/src/compat-app-builder.ts index 04e21767d..1386f36da 100644 --- a/packages/compat/src/compat-app-builder.ts +++ b/packages/compat/src/compat-app-builder.ts @@ -15,6 +15,7 @@ import { templateColocationPluginPath, cacheBustingPluginVersion, cacheBustingPluginPath, + Resolver, } from '@embroider/core'; import walkSync from 'walk-sync'; import { resolve as resolvePath, posix } from 'path'; @@ -246,10 +247,6 @@ export class CompatAppBuilder { return this.configTree.readConfig().rootURL; } - private templateCompilerPath(): string { - return 'ember-source/vendor/ember/ember-template-compiler'; - } - @Memoize() private activeRules() { return activePackageRules(this.options.packageRules.concat(defaultAddonPackageRules()), [ @@ -1007,9 +1004,18 @@ export class CompatAppBuilder { transforms.push([require.resolve('./resolver-transform'), opts]); } + let resolver = new Resolver(resolverConfig); + let resolution = resolver.nodeResolve( + 'ember-source/vendor/ember/ember-template-compiler', + resolvePath(this.root, 'package.json') + ); + if (resolution.type !== 'real') { + throw new Error(`bug: unable to resolve ember-template-compiler from ${this.root}`); + } + return { transforms, - compilerPath: resolve.sync(this.templateCompilerPath(), { basedir: this.root }), + compilerPath: resolution.filename, enableLegacyModules: ['ember-cli-htmlbars', 'ember-cli-htmlbars-inline-precompile', 'htmlbars-inline-precompile'], }; } From 74fc34aa1c17754b20547cb9d1e849328e04129e Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Tue, 13 Jun 2023 14:58:51 -0400 Subject: [PATCH 29/72] make rewritten-packages/index.json mandatory --- packages/core/src/rewritten-package-cache.ts | 42 +++++++++----------- 1 file changed, 18 insertions(+), 24 deletions(-) diff --git a/packages/core/src/rewritten-package-cache.ts b/packages/core/src/rewritten-package-cache.ts index fae98b90d..8dc3eba0d 100644 --- a/packages/core/src/rewritten-package-cache.ts +++ b/packages/core/src/rewritten-package-cache.ts @@ -112,31 +112,25 @@ export class RewrittenPackageCache implements PackageCacheTheGoodParts { } { let addonsDir = resolve(this.appRoot, 'node_modules', '.embroider', 'rewritten-packages'); let indexFile = resolve(addonsDir, 'index.json'); - if (existsSync(indexFile)) { - // I should probably make the else case throw here soon. - let { packages, extraResolutions } = readJSONSync(indexFile) as RewrittenPackageIndex; - return { - oldToNew: new Map( - Object.entries(packages).map(([oldRoot, newRoot]) => [ - resolve(addonsDir, oldRoot), - resolve(addonsDir, newRoot), - ]) - ), - newToOld: new Map( - Object.entries(packages).map(([oldRoot, newRoot]) => [ - resolve(addonsDir, newRoot), - resolve(addonsDir, oldRoot), - ]) - ), - extraResolutions: new Map( - Object.entries(extraResolutions).map(([fromRoot, toRoots]) => [ - resolve(addonsDir, fromRoot), - toRoots.map(r => resolve(addonsDir, r)), - ]) - ), - }; + if (!existsSync(indexFile)) { + throw new Error(`RewrittenPackageCache expected ${indexFile} to exist`); } - return { oldToNew: new Map(), newToOld: new Map(), extraResolutions: new Map() }; + + let { packages, extraResolutions } = readJSONSync(indexFile) as RewrittenPackageIndex; + return { + oldToNew: new Map( + Object.entries(packages).map(([oldRoot, newRoot]) => [resolve(addonsDir, oldRoot), resolve(addonsDir, newRoot)]) + ), + newToOld: new Map( + Object.entries(packages).map(([oldRoot, newRoot]) => [resolve(addonsDir, newRoot), resolve(addonsDir, oldRoot)]) + ), + extraResolutions: new Map( + Object.entries(extraResolutions).map(([fromRoot, toRoots]) => [ + resolve(addonsDir, fromRoot), + toRoots.map(r => resolve(addonsDir, r)), + ]) + ), + }; } // put a WrappedPackage around Packages that do in fact represent ones that we From eda1ab76ada708bd7681bcc47db727687a39e566 Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Wed, 14 Jun 2023 11:28:05 -0400 Subject: [PATCH 30/72] move rewritten package cache into shared-internals --- packages/core/src/index.ts | 2 -- packages/core/src/module-resolver.ts | 3 +-- packages/shared-internals/src/index.ts | 2 ++ .../{core => shared-internals}/src/rewritten-package-cache.ts | 4 +++- 4 files changed, 6 insertions(+), 5 deletions(-) rename packages/{core => shared-internals}/src/rewritten-package-cache.ts (98%) diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 308836bc1..51d4e479c 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -25,8 +25,6 @@ export { } from './module-resolver'; export { virtualContent } from './virtual-content'; export type { Engine } from './app-files'; -export type { RewrittenPackageIndex } from './rewritten-package-cache'; -export { RewrittenPackageCache } from './rewritten-package-cache'; // this is reexported because we already make users manage a peerDep from some // other packages (like embroider/webpack and @embroider/compat diff --git a/packages/core/src/module-resolver.ts b/packages/core/src/module-resolver.ts index 53112ac33..47859dc51 100644 --- a/packages/core/src/module-resolver.ts +++ b/packages/core/src/module-resolver.ts @@ -5,7 +5,7 @@ import { packageName as getPackageName, } from '@embroider/shared-internals'; import { dirname, resolve } from 'path'; -import { Package, V2Package, explicitRelative } from '@embroider/shared-internals'; +import { Package, V2Package, explicitRelative, RewrittenPackageCache } from '@embroider/shared-internals'; import makeDebug from 'debug'; import assertNever from 'assert-never'; import resolveModule from 'resolve'; @@ -19,7 +19,6 @@ import { import { Memoize } from 'typescript-memoize'; import { describeExports } from './describe-exports'; import { readFileSync } from 'fs'; -import { RewrittenPackageCache } from './rewritten-package-cache'; const debug = makeDebug('embroider:resolver'); function logTransition(reason: string, before: R, after: R = before): R { diff --git a/packages/shared-internals/src/index.ts b/packages/shared-internals/src/index.ts index bdabe88f0..1f4a81aee 100644 --- a/packages/shared-internals/src/index.ts +++ b/packages/shared-internals/src/index.ts @@ -3,6 +3,8 @@ export { explicitRelative, extensionsPattern, unrelativize } from './paths'; export { getOrCreate } from './get-or-create'; export { default as Package, V2AddonPackage as AddonPackage, V2AppPackage as AppPackage, V2Package } from './package'; export { default as PackageCache } from './package-cache'; +export type { RewrittenPackageIndex } from './rewritten-package-cache'; +export { RewrittenPackageCache } from './rewritten-package-cache'; export { default as babelFilter } from './babel-filter'; export { default as packageName } from './package-name'; export { default as tmpdir } from './tmpdir'; diff --git a/packages/core/src/rewritten-package-cache.ts b/packages/shared-internals/src/rewritten-package-cache.ts similarity index 98% rename from packages/core/src/rewritten-package-cache.ts rename to packages/shared-internals/src/rewritten-package-cache.ts index 8dc3eba0d..9b753dac7 100644 --- a/packages/core/src/rewritten-package-cache.ts +++ b/packages/shared-internals/src/rewritten-package-cache.ts @@ -1,7 +1,9 @@ -import { PackageCache, Package, getOrCreate } from '@embroider/shared-internals'; +import PackageCache from './package-cache'; +import Package from './package'; import { existsSync, readJSONSync } from 'fs-extra'; import { resolve } from 'path'; import { Memoize } from 'typescript-memoize'; +import { getOrCreate } from './get-or-create'; export interface RewrittenPackageIndex { // keys are paths to original package root directories. From 8f80624f594dc8a6c91f504f63b811605c7f72ed Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Wed, 14 Jun 2023 12:04:31 -0400 Subject: [PATCH 31/72] using the real appRoot throughout stage3 --- packages/compat/src/audit.ts | 2 +- .../compat/src/babel-plugin-adjust-imports.ts | 4 +- packages/compat/src/compat-addons.ts | 7 +- packages/compat/src/compat-app-builder.ts | 8 +-- packages/compat/src/compat-app.ts | 5 +- packages/compat/src/default-pipeline.ts | 4 -- packages/compat/src/options.ts | 4 -- packages/compat/src/resolver-transform.ts | 2 +- packages/compat/src/standalone-addon-build.ts | 11 ++- packages/compat/tests/audit.test.ts | 6 +- packages/core/src/module-resolver.ts | 70 ++++++++++--------- packages/core/src/to-broccoli-plugin.ts | 3 +- packages/macros/src/babel/get-config.ts | 13 ++-- .../macros/src/babel/macros-babel-plugin.ts | 2 - packages/macros/src/babel/state.ts | 6 +- packages/macros/src/glimmer/ast-transform.ts | 2 +- packages/macros/src/macros-config.ts | 33 +++------ packages/shared-internals/src/babel-filter.ts | 2 +- .../shared-internals/src/package-cache.ts | 7 +- packages/shared-internals/src/package.ts | 6 +- .../src/rewritten-package-cache.ts | 23 +++--- .../src/template-colocation-plugin.ts | 2 +- packages/webpack/src/ember-webpack.ts | 15 ++-- tests/scenarios/compat-resolver-test.ts | 2 +- tests/scenarios/core-resolver-test.ts | 2 +- 25 files changed, 123 insertions(+), 118 deletions(-) diff --git a/packages/compat/src/audit.ts b/packages/compat/src/audit.ts index f83baf8ad..72bcfebbe 100644 --- a/packages/compat/src/audit.ts +++ b/packages/compat/src/audit.ts @@ -278,7 +278,7 @@ export class Audit { } private get resolverParams(): ResolverOptions { - return readJSONSync(join(this.appDir, '.embroider', 'resolver.json')); + return readJSONSync(join(this.appDir, 'node_modules', '.embroider', 'resolver.json')); } private resolver = new Resolver(this.resolverParams); diff --git a/packages/compat/src/babel-plugin-adjust-imports.ts b/packages/compat/src/babel-plugin-adjust-imports.ts index 0b35dd9f2..91d4b86fc 100644 --- a/packages/compat/src/babel-plugin-adjust-imports.ts +++ b/packages/compat/src/babel-plugin-adjust-imports.ts @@ -42,7 +42,9 @@ export default function main(babel: typeof Babel) { if (cached) { return cached; } - let resolverOptions: CompatResolverOptions = readJSONSync(join(appRoot, '.embroider', 'resolver.json')); + let resolverOptions: CompatResolverOptions = readJSONSync( + join(appRoot, 'node_modules', '.embroider', 'resolver.json') + ); let resolver = new Resolver(resolverOptions); cached = { resolverOptions, diff --git a/packages/compat/src/compat-addons.ts b/packages/compat/src/compat-addons.ts index b7f1d586a..26706a3a4 100644 --- a/packages/compat/src/compat-addons.ts +++ b/packages/compat/src/compat-addons.ts @@ -1,6 +1,5 @@ import { Node } from 'broccoli-node-api'; import { resolve } from 'path'; -import { ensureDirSync, realpathSync } from 'fs-extra'; import { Stage, WaitForTrees } from '@embroider/core'; import TreeSync from 'tree-sync'; import CompatApp from './compat-app'; @@ -26,11 +25,7 @@ export default class CompatAddons implements Stage { private addons: Node; constructor(compatApp: CompatApp) { - // we want this to be stable across builds, because it becomes part of the - // path to all of the files that the stage3 packager sees, and we want to - // benefit from on-disk caching in stage3 packagers. - ensureDirSync(compatApp.options.workspaceDir!); - this.destDir = realpathSync(compatApp.options.workspaceDir!); + this.destDir = resolve(compatApp.root, 'node_modules', '.embroider', 'rewritten-packages', compatApp.name); this.addons = convertLegacyAddons(compatApp); this.inputPath = compatApp.root; } diff --git a/packages/compat/src/compat-app-builder.ts b/packages/compat/src/compat-app-builder.ts index 1386f36da..294698555 100644 --- a/packages/compat/src/compat-app-builder.ts +++ b/packages/compat/src/compat-app-builder.ts @@ -270,7 +270,7 @@ export class CompatAppBuilder { renameModules, renamePackages, resolvableExtensions: this.resolvableExtensions(), - appRoot: this.root, + appRoot: this.origAppPackage.root, engines: engines.map((engine, index) => ({ packageName: engine.package.name, root: index === 0 ? this.root : engine.package.root, // first engine is the app, which has been relocated to this.roto @@ -441,7 +441,7 @@ export class CompatAppBuilder { babel.plugins.push(...this.compatApp.macrosConfig.babelPluginConfig()); let colocationOptions: TemplateColocationPluginOptions = { - appRoot: this.root, + appRoot: this.origAppPackage.root, // This extra weirdness is a compromise in favor of build performance. // @@ -1051,13 +1051,13 @@ export class CompatAppBuilder { ); writeFileSync( join(this.root, '_babel_filter_.js'), - babelFilterTemplate({ skipBabel: this.options.skipBabel, appRoot: this.root }), + babelFilterTemplate({ skipBabel: this.options.skipBabel, appRoot: this.origAppPackage.root }), 'utf8' ); } private addResolverConfig(config: CompatResolverOptions) { - outputJSONSync(join(this.root, '.embroider', 'resolver.json'), config); + outputJSONSync(join(this.origAppPackage.root, 'node_modules', '.embroider', 'resolver.json'), config); } private shouldSplitRoute(routeName: string) { diff --git a/packages/compat/src/compat-app.ts b/packages/compat/src/compat-app.ts index 82e08b02c..bf230cb3b 100644 --- a/packages/compat/src/compat-app.ts +++ b/packages/compat/src/compat-app.ts @@ -53,7 +53,7 @@ export default class CompatApp { return this.legacyEmberAppInstance.project.pkg.keywords?.includes('ember-addon') ?? false; } - private get name(): string { + get name(): string { if (this.isDummy) { // here we accept the ember-cli behavior return this.legacyEmberAppInstance.name; @@ -848,8 +848,7 @@ export default class CompatApp { return new WaitForTrees(inTrees, this.annotation, async treePaths => { if (!this.active) { let { outputPath } = await prevStage.ready(); - // TODO: this will use shared caches once we refactor out MovedPackageCache - let packageCache = new RewrittenPackageCache(new PackageCache(this.root)); + let packageCache = RewrittenPackageCache.shared('embroider', this.root); this.active = await this.instantiate(outputPath, prevStage.inputPath, packageCache, inTrees.configTree); resolve({ outputPath }); } diff --git a/packages/compat/src/default-pipeline.ts b/packages/compat/src/default-pipeline.ts index 4f3b50b41..5cffe2f22 100644 --- a/packages/compat/src/default-pipeline.ts +++ b/packages/compat/src/default-pipeline.ts @@ -29,10 +29,6 @@ export default function defaultPipeline( let outputPath: string; let addons; - options.workspaceDir = stableWorkspaceDir(emberApp.project.root, emberApp.env); - - emberApp.project.ui.write(`Building into ${options.workspaceDir}\n`); - let embroiderApp = new App(emberApp, options); addons = new CompatAddons(embroiderApp); diff --git a/packages/compat/src/options.ts b/packages/compat/src/options.ts index 43069802d..beaf7f055 100644 --- a/packages/compat/src/options.ts +++ b/packages/compat/src/options.ts @@ -60,10 +60,6 @@ export default interface Options extends CoreOptions { // setting your own value here (including null to completely disable it). compatAdapters?: Map; - // temporary directory where we will work when we're rewriting your addons - // and/or app to v2-compatible formats. - workspaceDir?: string | null; - // optional list of additional broccoli trees that should be incorporated into // the final build. This exists because the classic `app.toTree()` method // accepts an optional tree argument that has the same purpose. diff --git a/packages/compat/src/resolver-transform.ts b/packages/compat/src/resolver-transform.ts index cfbfeff61..e0bec2dfd 100644 --- a/packages/compat/src/resolver-transform.ts +++ b/packages/compat/src/resolver-transform.ts @@ -861,7 +861,7 @@ class TemplateResolver implements ASTPlugin { // This is the AST transform that resolves components, helpers and modifiers at build time export default function makeResolverTransform({ appRoot }: Options) { - let config: CompatResolverOptions = readJSONSync(join(appRoot, '.embroider', 'resolver.json')); + let config: CompatResolverOptions = readJSONSync(join(appRoot, 'node_modules', '.embroider', 'resolver.json')); const resolverTransform: ASTPluginBuilder = env => { if (env.strictMode) { return { diff --git a/packages/compat/src/standalone-addon-build.ts b/packages/compat/src/standalone-addon-build.ts index cb116caad..ab96e9e68 100644 --- a/packages/compat/src/standalone-addon-build.ts +++ b/packages/compat/src/standalone-addon-build.ts @@ -13,7 +13,7 @@ export function convertLegacyAddons(compatApp: CompatApp) { let instanceCache = new V1InstanceCache(compatApp, packageCache); let v1Addons = findV1Addons(packageCache.get(compatApp.root)); - let index = buildAddonIndex(v1Addons); + let index = buildAddonIndex(compatApp, v1Addons); let interiorTrees: Node[] = []; let exteriorTrees = [...v1Addons].map(pkg => { @@ -30,11 +30,11 @@ export function convertLegacyAddons(compatApp: CompatApp) { new Funnel(compatApp.synthesizeVendorPackage(interiorTrees), { destDir: '@embroider/synthesized-vendor', }), - writeFile('index.json', JSON.stringify(buildAddonIndex(v1Addons), null, 2)), + writeFile('index.json', JSON.stringify(index, null, 2)), ]); } -function buildAddonIndex(packages: Set): RewrittenPackageIndex { +function buildAddonIndex(compatApp: CompatApp, packages: Set): RewrittenPackageIndex { let content: RewrittenPackageIndex = { packages: {}, extraResolutions: {}, @@ -48,6 +48,11 @@ function buildAddonIndex(packages: Set): RewrittenPackageIndex { } } + // adding an entry for the app itself to have a place in the + // rewritten-packages, even though this stage hasn't actually put it there + // yet. + content.packages[compatApp.root] = compatApp.name; + return content; } diff --git a/packages/compat/tests/audit.test.ts b/packages/compat/tests/audit.test.ts index 17b1bb7c8..c5c3030e8 100644 --- a/packages/compat/tests/audit.test.ts +++ b/packages/compat/tests/audit.test.ts @@ -76,8 +76,10 @@ describe('audit', function () { null, 2 )}`, - '.embroider': { - 'resolver.json': JSON.stringify(resolverConfig), + node_modules: { + '.embroider': { + 'resolver.json': JSON.stringify(resolverConfig), + }, }, }); let appMeta: AppMeta = { diff --git a/packages/core/src/module-resolver.ts b/packages/core/src/module-resolver.ts index 47859dc51..c8a9ee8eb 100644 --- a/packages/core/src/module-resolver.ts +++ b/packages/core/src/module-resolver.ts @@ -155,9 +155,14 @@ export class Resolver { request = this.handleFastbootCompat(request); request = this.handleGlobalsCompat(request); - request = this.handleRewrittenPackages(request); request = this.handleRenaming(request); - return this.preHandleExternal(request); + request = this.preHandleExternal(request); + + // this should probably stay the last step in beforeResolve, because it can + // rehome requests to their un-rewritten locations, and for the most part we + // want to be dealing with the rewritten packages. + request = this.handleRewrittenPackages(request); + return request; } // This encapsulates the whole resolving process. Given a `defaultResolve` @@ -270,16 +275,18 @@ export class Resolver { } } + private get packageCache() { + return RewrittenPackageCache.shared('embroider', this.options.appRoot); + } + owningPackage(fromFile: string): Package | undefined { - return RewrittenPackageCache.shared('embroider-stage3', this.options.appRoot).ownerOfFile(fromFile); + return this.packageCache.ownerOfFile(fromFile); } private logicalPackage(owningPackage: V2Package, file: string): V2Package { let logicalLocation = this.reverseSearchAppTree(owningPackage, file); if (logicalLocation) { - let pkg = RewrittenPackageCache.shared('embroider-stage3', this.options.appRoot).get( - logicalLocation.owningEngine.root - ); + let pkg = this.packageCache.get(logicalLocation.owningEngine.root); if (!pkg.isV2Ember()) { throw new Error(`bug: all engines should be v2 addons by the time we see them here`); } @@ -482,12 +489,11 @@ export class Resolver { // out. @Memoize() private get mergeMap(): MergeMap { - let packageCache = RewrittenPackageCache.shared('embroider-stage3', this.options.appRoot); let result: MergeMap = new Map(); for (let engine of this.options.engines) { let engineModules: Map = new Map(); for (let addonConfig of engine.activeAddons) { - let addon = packageCache.get(addonConfig.root); + let addon = this.packageCache.get(addonConfig.root); if (!addon.isV2Addon()) { continue; } @@ -588,10 +594,6 @@ export class Resolver { } owningEngine(pkg: Package) { - if (pkg.root === this.options.appRoot) { - // the app is always the first engine - return this.options.engines[0]; - } let owningEngine = this.options.engines.find(e => pkg.isEngine() ? e.root === pkg.root : e.activeAddons.find(a => a.root === pkg.root) ); @@ -604,23 +606,21 @@ export class Resolver { } private handleRewrittenPackages(request: R): R { - let packageCache = RewrittenPackageCache.shared('embroider-stage3', this.options.appRoot); - let requestingPkg = this.owningPackage(request.fromFile); if (!requestingPkg) { return request; } + let packageName = getPackageName(request.specifier); + if (!packageName) { + // relative request + return request; + } let targetPkg: Package | undefined; - let packageName = getPackageName(request.specifier); - if (packageName && packageName !== requestingPkg.name) { + if (packageName !== requestingPkg.name) { // non-relative, non-self request, so check if it aims at a rewritten addon try { - // up above we already ensured that the source `pkg` here is always referring to the *original* copy if it had been rewritten. - targetPkg = RewrittenPackageCache.shared('embroider-stage3', this.options.appRoot).resolve( - packageName, - requestingPkg - ); + targetPkg = this.packageCache.resolve(packageName, requestingPkg); } catch (err) { // this is not the place to report resolution failures. If the thing // doesn't resolve, we're just not interested in redirecting it for @@ -632,8 +632,8 @@ export class Resolver { } } - let originalRequestingPkg = packageCache.original(requestingPkg); - let originalTargetPkg = targetPkg ? packageCache.original(targetPkg) : undefined; + let originalRequestingPkg = this.packageCache.original(requestingPkg); + let originalTargetPkg = targetPkg ? this.packageCache.original(targetPkg) : undefined; if (targetPkg && originalTargetPkg) { // in this case it doesn't matter whether or not the requesting package @@ -785,10 +785,7 @@ export class Resolver { if (logicalPackage.meta['auto-upgraded'] && !logicalPackage.hasDependency('ember-auto-import')) { try { - let dep = RewrittenPackageCache.shared('embroider-stage3', this.options.appRoot).resolve( - packageName, - logicalPackage - ); + let dep = this.packageCache.resolve(packageName, logicalPackage); if (!dep.isEmberPackage()) { // classic ember addons can only import non-ember dependencies if they // have ember-auto-import. @@ -837,7 +834,17 @@ export class Resolver { } let pkg = this.owningPackage(fromFile); - if (!pkg || !pkg.isV2Ember()) { + if (!pkg) { + return logTransition('no identifiable owningPackage', request); + } + + // if we rehomed this request to its un-rewritten location in order to try + // to do the defaultResolve from there, now we refer back to the rewritten + // location because that's what we want to use when asking things like + // isV2Ember() + pkg = this.packageCache.maybeMoved(pkg); + + if (!pkg.isV2Ember()) { return logTransition('fallbackResolve: not in an ember package', request); } @@ -873,12 +880,7 @@ export class Resolver { return logTransition( `activeAddons`, request, - this.resolveWithinPackage( - request, - RewrittenPackageCache.shared('embroider-stage3', this.options.appRoot).get( - this.options.activeAddons[packageName] - ) - ) + this.resolveWithinPackage(request, this.packageCache.get(this.options.activeAddons[packageName])) ); } diff --git a/packages/core/src/to-broccoli-plugin.ts b/packages/core/src/to-broccoli-plugin.ts index 64a4cbd59..f15c1d92e 100644 --- a/packages/core/src/to-broccoli-plugin.ts +++ b/packages/core/src/to-broccoli-plugin.ts @@ -22,9 +22,8 @@ export default function toBroccoliPlugin( async build() { if (!this.packager) { - let { outputPath } = await this.stage.ready(); this.packager = new packagerClass( - outputPath, + this.stage.inputPath, this.outputPath, this.variants, msg => console.log(msg.split(tmpdir).join('$TMPDIR')), diff --git a/packages/macros/src/babel/get-config.ts b/packages/macros/src/babel/get-config.ts index 5bd3a0157..b7d626613 100644 --- a/packages/macros/src/babel/get-config.ts +++ b/packages/macros/src/babel/get-config.ts @@ -1,6 +1,6 @@ import type { NodePath } from '@babel/traverse'; import State from './state'; -import { PackageCache, Package } from '@embroider/shared-internals'; +import { RewrittenPackageCache, Package } from '@embroider/shared-internals'; import error from './error'; import { Evaluator, assertArray, buildLiterals, ConfidentResult } from './evaluate-json'; import assertNever from 'assert-never'; @@ -74,16 +74,21 @@ export function insertConfig(path: NodePath, state: State, mod } } -function targetPackage(fromPath: string, packageName: string | undefined, packageCache: PackageCache): Package | null { +function targetPackage( + fromPath: string, + packageName: string | undefined, + packageCache: RewrittenPackageCache +): Package | null { let us = packageCache.ownerOfFile(fromPath); if (!us) { throw new Error(`unable to determine which npm package owns the file ${fromPath}`); } if (!packageName) { - return us; + return packageCache.original(us) || us; } try { - return packageCache.resolve(packageName, us); + let target = packageCache.resolve(packageName, us); + return packageCache.original(target) || target; } catch (err) { return null; } diff --git a/packages/macros/src/babel/macros-babel-plugin.ts b/packages/macros/src/babel/macros-babel-plugin.ts index 385f23807..53d53b174 100644 --- a/packages/macros/src/babel/macros-babel-plugin.ts +++ b/packages/macros/src/babel/macros-babel-plugin.ts @@ -17,8 +17,6 @@ export default function main(context: typeof Babel): unknown { Program: { enter(path: NodePath, state: State) { initState(t, path, state); - - state.packageCache = PackageCache.shared('embroider-stage3', state.opts.appPackageRoot); }, exit(_: NodePath, state: State) { // @embroider/macros itself has no runtime behaviors and should always be removed diff --git a/packages/macros/src/babel/state.ts b/packages/macros/src/babel/state.ts index 62cc04661..33faf7e80 100644 --- a/packages/macros/src/babel/state.ts +++ b/packages/macros/src/babel/state.ts @@ -2,7 +2,7 @@ import type { NodePath, Node } from '@babel/traverse'; import cloneDeepWith from 'lodash/cloneDeepWith'; import lodashCloneDeep from 'lodash/cloneDeep'; import { join, dirname, resolve } from 'path'; -import { explicitRelative, Package, PackageCache } from '@embroider/shared-internals'; +import { explicitRelative, Package, RewrittenPackageCache } from '@embroider/shared-internals'; import { ImportUtil } from 'babel-import-util'; import type * as Babel from '@babel/core'; @@ -12,7 +12,7 @@ export default interface State { removed: Set; calledIdentifiers: Set; jobs: (() => void)[]; - packageCache: PackageCache; + packageCache: RewrittenPackageCache; sourceFile: string; pathToOurAddon(moduleName: string): string; owningPackage(): Package; @@ -55,7 +55,7 @@ export function initState(t: typeof Babel.types, path: NodePath this.moves.get(root) || root), + isDevelopingPackageRoots: [...this.isDevelopingPackageRoots], - // lazy so that packageMoved() can still affect this get appPackageRoot() { return self.appRoot; }, @@ -434,27 +432,16 @@ export default class MacrosConfig { return defaultMergerFor(pkgRoot); } - // this exists because @embroider/compat rewrites and moves v1 addons, and - // their macro configs need to follow them to their new homes. - packageMoved(oldPath: string, newPath: string) { - if (!this._configWritable) { - throw new Error(`[Embroider:MacrosConfig] attempted to call packageMoved after configs have been finalized`); - } - - this.moves.set(oldPath, newPath); - } - - private moves: Map = new Map(); - private resolvePackage(fromPath: string, packageName?: string | undefined) { let us = this.packageCache.ownerOfFile(fromPath); if (!us) { throw new Error(`[Embroider:MacrosConfig] unable to determine which npm package owns the file ${fromPath}`); } if (packageName) { - return this.packageCache.resolve(packageName, us); + let target = this.packageCache.resolve(packageName, us); + return this.packageCache.original(target) || target; } else { - return us; + return this.packageCache.original(us) || us; } } diff --git a/packages/shared-internals/src/babel-filter.ts b/packages/shared-internals/src/babel-filter.ts index 0932aee27..68a9318df 100644 --- a/packages/shared-internals/src/babel-filter.ts +++ b/packages/shared-internals/src/babel-filter.ts @@ -8,7 +8,7 @@ export default function babelFilter(skipBabel: { package: string; semverRange?: return false; } - let owner = PackageCache.shared('embroider-stage3', appRoot).ownerOfFile(filename); + let owner = PackageCache.shared('embroider', appRoot).ownerOfFile(filename); if (owner) { for (let { package: pkg, semverRange } of skipBabel) { if (owner.name === pkg && (semverRange == null || semver.satisfies(owner.version, semverRange))) { diff --git a/packages/shared-internals/src/package-cache.ts b/packages/shared-internals/src/package-cache.ts index 88ad4b24e..c7ae58c0c 100644 --- a/packages/shared-internals/src/package-cache.ts +++ b/packages/shared-internals/src/package-cache.ts @@ -71,7 +71,12 @@ export default class PackageCache { } static shared(identifier: string, appRoot: string) { - let pk = getOrCreate(shared, identifier + appRoot, () => new PackageCache(appRoot)); + // it's intentional that the cache key here does not include the appRoot. We + // *want* to notice if two people are using the same identifier with + // different appRoots: that's a bug, and automatically separating them from + // each other defeats part of the point of using a shared package cache in + // the first place. + let pk = getOrCreate(shared, identifier, () => new PackageCache(appRoot)); if (pk.appRoot !== appRoot) { throw new Error(`bug: PackageCache appRoot disagreement ${appRoot}!=${pk.appRoot}`); } diff --git a/packages/shared-internals/src/package.ts b/packages/shared-internals/src/package.ts index e1e8ebfba..775c0a722 100644 --- a/packages/shared-internals/src/package.ts +++ b/packages/shared-internals/src/package.ts @@ -8,7 +8,7 @@ import flatMap from 'lodash/flatMap'; export default class Package { private dependencyKeys: ('dependencies' | 'devDependencies' | 'peerDependencies')[]; - constructor(readonly root: string, protected packageCache: PackageCache, isApp: boolean) { + constructor(readonly root: string, protected packageCache: PackageCache, private isApp: boolean) { this.dependencyKeys = isApp ? ['dependencies', 'devDependencies', 'peerDependencies'] : ['dependencies', 'peerDependencies']; @@ -57,6 +57,10 @@ export default class Package { } isEngine(): boolean { + if (this.isApp) { + // an app is implicitly an engine + return true; + } let keywords = this.packageJSON.keywords; return Boolean(keywords && (keywords as string[]).includes('ember-engine')); } diff --git a/packages/shared-internals/src/rewritten-package-cache.ts b/packages/shared-internals/src/rewritten-package-cache.ts index 9b753dac7..02cddb675 100644 --- a/packages/shared-internals/src/rewritten-package-cache.ts +++ b/packages/shared-internals/src/rewritten-package-cache.ts @@ -93,6 +93,12 @@ export class RewrittenPackageCache implements PackageCacheTheGoodParts { withRewrittenDeps(pkg: Package): Package { let found = wrapped.get(pkg); if (!found) { + if (pkg.root === this.index.oldToNew.get(this.appRoot)) { + // the plain representation of our moved app doesn't know that it's an + // app, so we instead make a plain Package with isApp set to true + // explicitly. + pkg = new Package(pkg.root, this.plainCache, true); + } found = new WrappedPackage(this, pkg); wrapped.set(pkg, found); } @@ -146,13 +152,14 @@ export class RewrittenPackageCache implements PackageCacheTheGoodParts { } } static shared(identifier: string, appRoot: string) { - let pk = getOrCreate( - shared, - identifier + appRoot, - () => new RewrittenPackageCache(PackageCache.shared(identifier, appRoot)) - ); + // it's intentional that the cache key here does not include the appRoot. We + // *want* to notice if two people are using the same identifier with + // different appRoots: that's a bug, and automatically separating them from + // each other defeats part of the point of using a shared package cache in + // the first place. + let pk = getOrCreate(shared, identifier, () => new RewrittenPackageCache(PackageCache.shared(identifier, appRoot))); if (pk.appRoot !== appRoot) { - throw new Error(`bug: PackageCache appRoot disagreement ${appRoot}!=${pk.appRoot}`); + throw new Error(`bug: RewrittenPackageCache appRoot disagreement ${appRoot}!=${pk.appRoot}`); } return pk; } @@ -211,8 +218,8 @@ class WrappedPackage implements PackageTheGoodParts { return this.plainPkg.isEmberPackage; } - get isEngine() { - return this.plainPkg.isEngine; + isEngine() { + return this.plainPkg.isEngine(); } get isLazyEngine() { diff --git a/packages/shared-internals/src/template-colocation-plugin.ts b/packages/shared-internals/src/template-colocation-plugin.ts index 3e6646726..c3adea216 100644 --- a/packages/shared-internals/src/template-colocation-plugin.ts +++ b/packages/shared-internals/src/template-colocation-plugin.ts @@ -55,7 +55,7 @@ export default function main(babel: typeof Babel) { let filename = (path.hub as any).file.opts.filename; if (state.opts.packageGuard) { - let owningPackage = PackageCache.shared('embroider-stage3', state.opts.appRoot).ownerOfFile(filename); + let owningPackage = PackageCache.shared('embroider', state.opts.appRoot).ownerOfFile(filename); if (!owningPackage || !owningPackage.isV2Ember() || !owningPackage.meta['auto-upgraded']) { return; } diff --git a/packages/webpack/src/ember-webpack.ts b/packages/webpack/src/ember-webpack.ts index dee70c47e..f7eca8c55 100644 --- a/packages/webpack/src/ember-webpack.ts +++ b/packages/webpack/src/ember-webpack.ts @@ -21,9 +21,9 @@ import { getOrCreate, ResolverOptions, } from '@embroider/core'; -import { tmpdir } from '@embroider/shared-internals'; +import { RewrittenPackageCache, tmpdir } from '@embroider/shared-internals'; import webpack, { Configuration, RuleSetUseItem, WebpackPluginInstance } from 'webpack'; -import { readFileSync, outputFileSync, copySync, realpathSync, Stats, statSync, readJSONSync } from 'fs-extra'; +import { readFileSync, outputFileSync, copySync, Stats, statSync, readJSONSync } from 'fs-extra'; import { join, dirname, relative, sep } from 'path'; import isEqual from 'lodash/isEqual'; import mergeWith from 'lodash/mergeWith'; @@ -110,7 +110,7 @@ function createBarrier(): [BeginFn, IncrementFn] { const Webpack: PackagerConstructor = class Webpack implements Packager { static annotation = '@embroider/webpack'; - pathToVanillaApp: string; + private pathToVanillaApp: string; private extraConfig: Configuration | undefined; private passthroughCache: Map = new Map(); private publicAssetURL: string | undefined; @@ -123,7 +123,7 @@ const Webpack: PackagerConstructor = class Webpack implements Packager private incrementBarrier: IncrementFn; constructor( - pathToVanillaApp: string, + private appRoot: string, private outputPath: string, private variants: Variant[], private consoleWrite: (msg: string) => void, @@ -133,7 +133,8 @@ const Webpack: PackagerConstructor = class Webpack implements Packager throw new Error(`@embroider/webpack requires webpack@^5.0.0, but found version ${webpack.version}`); } - this.pathToVanillaApp = realpathSync(pathToVanillaApp); + let packageCache = RewrittenPackageCache.shared('embroider', appRoot); + this.pathToVanillaApp = packageCache.maybeMoved(packageCache.get(appRoot)).root; this.extraConfig = options?.webpackConfig; this.publicAssetURL = options?.publicAssetURL; this.extraThreadLoaderOptions = options?.threadLoaderOptions; @@ -180,7 +181,9 @@ const Webpack: PackagerConstructor = class Webpack implements Packager } } - let resolverConfig: EmbroiderPluginOptions = readJSONSync(join(this.pathToVanillaApp, '.embroider/resolver.json')); + let resolverConfig: EmbroiderPluginOptions = readJSONSync( + join(this.appRoot, 'node_modules/.embroider/resolver.json') + ); return { entrypoints, otherAssets, babel, rootURL, resolverConfig, publicAssetURL, packageName: meta.name }; } diff --git a/tests/scenarios/compat-resolver-test.ts b/tests/scenarios/compat-resolver-test.ts index 51af1ee97..f2390f275 100644 --- a/tests/scenarios/compat-resolver-test.ts +++ b/tests/scenarios/compat-resolver-test.ts @@ -110,7 +110,7 @@ Scenarios.fromProject(() => new Project()) '_babel_filter.js': ` module.exports = function(filename) { return true } `, - '.embroider/resolver.json': JSON.stringify(resolverOptions), + 'node_modules/.embroider/resolver.json': JSON.stringify(resolverOptions), }); }; }); diff --git a/tests/scenarios/core-resolver-test.ts b/tests/scenarios/core-resolver-test.ts index 0377005a6..e0d0f4f7d 100644 --- a/tests/scenarios/core-resolver-test.ts +++ b/tests/scenarios/core-resolver-test.ts @@ -125,7 +125,7 @@ Scenarios.fromProject(() => new Project()) '_babel_filter.js': ` module.exports = function(filename) { return true } `, - '.embroider/resolver.json': JSON.stringify(resolverOptions), + 'node_modules/.embroider/resolver.json': JSON.stringify(resolverOptions), 'node_modules/my-addon/package.json': addonPackageJSON(opts?.addonMeta), }); From e43a836e8477a1bd7b2bbfdb66ce24c3d5e26d0d Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Wed, 14 Jun 2023 13:48:03 -0400 Subject: [PATCH 32/72] fixing a lint issue --- packages/macros/src/babel/macros-babel-plugin.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/macros/src/babel/macros-babel-plugin.ts b/packages/macros/src/babel/macros-babel-plugin.ts index 53d53b174..0f1515297 100644 --- a/packages/macros/src/babel/macros-babel-plugin.ts +++ b/packages/macros/src/babel/macros-babel-plugin.ts @@ -1,6 +1,5 @@ import type { NodePath } from '@babel/traverse'; import type { types as t } from '@babel/core'; -import { PackageCache } from '@embroider/shared-internals'; import State, { initState } from './state'; import { inlineRuntimeConfig, insertConfig, Mode as GetConfigMode } from './get-config'; import macroCondition, { isMacroConditionPath } from './macro-condition'; From 99fff1be5f9ccc676323dd1ff451656dc53c3558 Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Wed, 14 Jun 2023 15:20:51 -0400 Subject: [PATCH 33/72] adjust timing of rewritten package index generation and fix a merge typo --- packages/compat/src/compat-addons.ts | 5 ++-- packages/compat/src/compat-app-builder.ts | 2 +- .../src/rewritten-package-cache.ts | 26 +++++++++++++++++-- 3 files changed, 28 insertions(+), 5 deletions(-) diff --git a/packages/compat/src/compat-addons.ts b/packages/compat/src/compat-addons.ts index 26706a3a4..c8e973008 100644 --- a/packages/compat/src/compat-addons.ts +++ b/packages/compat/src/compat-addons.ts @@ -1,6 +1,6 @@ import { Node } from 'broccoli-node-api'; import { resolve } from 'path'; -import { Stage, WaitForTrees } from '@embroider/core'; +import { RewrittenPackageCache, Stage, WaitForTrees } from '@embroider/core'; import TreeSync from 'tree-sync'; import CompatApp from './compat-app'; import { convertLegacyAddons } from './standalone-addon-build'; @@ -24,7 +24,7 @@ export default class CompatAddons implements Stage { private destDir: string; private addons: Node; - constructor(compatApp: CompatApp) { + constructor(private compatApp: CompatApp) { this.destDir = resolve(compatApp.root, 'node_modules', '.embroider', 'rewritten-packages', compatApp.name); this.addons = convertLegacyAddons(compatApp); this.inputPath = compatApp.root; @@ -59,6 +59,7 @@ export default class CompatAddons implements Stage { changedMap.get(addons) ) { this.treeSync.sync(); + RewrittenPackageCache.shared('embroider', this.compatApp.root).invalidateIndex(); } this.didBuild = true; } diff --git a/packages/compat/src/compat-app-builder.ts b/packages/compat/src/compat-app-builder.ts index 3b85cce12..9f5bb1eaf 100644 --- a/packages/compat/src/compat-app-builder.ts +++ b/packages/compat/src/compat-app-builder.ts @@ -1526,7 +1526,7 @@ function stringOrBufferEqual(a: string | Buffer, b: string | Buffer): boolean { const babelFilterTemplate = jsHandlebarsCompile(` const { babelFilter } = require(${JSON.stringify(require.resolve('@embroider/core'))}); -module.exports = babelFilter({{json-stringify skipBabel}}}, "{{{js-string-escape appRoot}}"); +module.exports = babelFilter({{json-stringify skipBabel}}, "{{js-string-escape appRoot}}"); `) as (params: { skipBabel: Options['skipBabel']; appRoot: string }) => string; // meta['renamed-modules'] has mapping from classic filename to real filename. diff --git a/packages/shared-internals/src/rewritten-package-cache.ts b/packages/shared-internals/src/rewritten-package-cache.ts index 02cddb675..c4c7e7d53 100644 --- a/packages/shared-internals/src/rewritten-package-cache.ts +++ b/packages/shared-internals/src/rewritten-package-cache.ts @@ -112,16 +112,38 @@ export class RewrittenPackageCache implements PackageCacheTheGoodParts { } } - @Memoize() + private indexCache: + | { + oldToNew: Map; + newToOld: Map; + extraResolutions: Map; + } + | undefined; + private get index(): { oldToNew: Map; newToOld: Map; extraResolutions: Map; } { + if (!this.indexCache) { + this.indexCache = this.loadIndex(); + } + return this.indexCache; + } + + invalidateIndex(): void { + this.indexCache = undefined; + } + + private loadIndex(): RewrittenPackageCache['index'] { let addonsDir = resolve(this.appRoot, 'node_modules', '.embroider', 'rewritten-packages'); let indexFile = resolve(addonsDir, 'index.json'); if (!existsSync(indexFile)) { - throw new Error(`RewrittenPackageCache expected ${indexFile} to exist`); + return { + oldToNew: new Map(), + newToOld: new Map(), + extraResolutions: new Map(), + }; } let { packages, extraResolutions } = readJSONSync(indexFile) as RewrittenPackageIndex; From 503cca08803860bbdf7b9a0573fb731e44679759 Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Wed, 14 Jun 2023 20:21:33 -0400 Subject: [PATCH 34/72] fix another lint issue --- packages/shared-internals/src/rewritten-package-cache.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/shared-internals/src/rewritten-package-cache.ts b/packages/shared-internals/src/rewritten-package-cache.ts index c4c7e7d53..129bda358 100644 --- a/packages/shared-internals/src/rewritten-package-cache.ts +++ b/packages/shared-internals/src/rewritten-package-cache.ts @@ -2,7 +2,6 @@ import PackageCache from './package-cache'; import Package from './package'; import { existsSync, readJSONSync } from 'fs-extra'; import { resolve } from 'path'; -import { Memoize } from 'typescript-memoize'; import { getOrCreate } from './get-or-create'; export interface RewrittenPackageIndex { From 6a1a773f9b57c63f06bc80a662ed56a297c91ccb Mon Sep 17 00:00:00 2001 From: Chris Manson Date: Thu, 15 Jun 2023 12:23:06 +0100 Subject: [PATCH 35/72] include appRoot in the identifier for the shared package caches --- packages/shared-internals/src/package-cache.ts | 10 ++++------ .../src/rewritten-package-cache.ts | 16 +++++++++------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/packages/shared-internals/src/package-cache.ts b/packages/shared-internals/src/package-cache.ts index c7ae58c0c..085bc1677 100644 --- a/packages/shared-internals/src/package-cache.ts +++ b/packages/shared-internals/src/package-cache.ts @@ -71,12 +71,10 @@ export default class PackageCache { } static shared(identifier: string, appRoot: string) { - // it's intentional that the cache key here does not include the appRoot. We - // *want* to notice if two people are using the same identifier with - // different appRoots: that's a bug, and automatically separating them from - // each other defeats part of the point of using a shared package cache in - // the first place. - let pk = getOrCreate(shared, identifier, () => new PackageCache(appRoot)); + let pk = getOrCreate(shared, identifier + appRoot, () => new PackageCache(appRoot)); + + // it's not clear that this could ever happen because appRoot is part of the new identifier + // but it doesn't cost much to leave this code here. if (pk.appRoot !== appRoot) { throw new Error(`bug: PackageCache appRoot disagreement ${appRoot}!=${pk.appRoot}`); } diff --git a/packages/shared-internals/src/rewritten-package-cache.ts b/packages/shared-internals/src/rewritten-package-cache.ts index 129bda358..248735b8e 100644 --- a/packages/shared-internals/src/rewritten-package-cache.ts +++ b/packages/shared-internals/src/rewritten-package-cache.ts @@ -173,14 +173,16 @@ export class RewrittenPackageCache implements PackageCacheTheGoodParts { } } static shared(identifier: string, appRoot: string) { - // it's intentional that the cache key here does not include the appRoot. We - // *want* to notice if two people are using the same identifier with - // different appRoots: that's a bug, and automatically separating them from - // each other defeats part of the point of using a shared package cache in - // the first place. - let pk = getOrCreate(shared, identifier, () => new RewrittenPackageCache(PackageCache.shared(identifier, appRoot))); + let pk = getOrCreate( + shared, + identifier + appRoot, + () => new RewrittenPackageCache(PackageCache.shared(identifier, appRoot)) + ); + + // it's not clear that this could ever happen because appRoot is part of the new identifier + // but it doesn't cost much to leave this code here. if (pk.appRoot !== appRoot) { - throw new Error(`bug: RewrittenPackageCache appRoot disagreement ${appRoot}!=${pk.appRoot}`); + throw new Error(`bug: RewrittenPackageCache appRoot disagreement ${appRoot} != ${pk.appRoot}`); } return pk; } From 9805bf9ba943233b3309bf735617a141e9f0029d Mon Sep 17 00:00:00 2001 From: Chris Manson Date: Thu, 15 Jun 2023 13:28:44 +0100 Subject: [PATCH 36/72] add expectRewrittenAddonFilesAt utility to file-assertions --- .../support/file-assertions/qunit.ts | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/test-packages/support/file-assertions/qunit.ts b/test-packages/support/file-assertions/qunit.ts index 5a2c241c3..567b7dfa0 100644 --- a/test-packages/support/file-assertions/qunit.ts +++ b/test-packages/support/file-assertions/qunit.ts @@ -1,5 +1,7 @@ import { install } from 'code-equality-assertions/qunit'; import { AssertionAdapter, BoundExpectFile, ExpectFile } from '../file-assertions'; +import { packageName } from '../../../packages/shared-internals'; +import crypto from 'crypto'; class QUnitAdapter implements AssertionAdapter { constructor(private qassert: Assert) { @@ -39,4 +41,34 @@ export function expectFilesAt(basePath: string, params: { qunit: Assert }): Expe return func; } +function getRewrittenLocation(appDir: string, addonPath: string){ + let name = packageName(addonPath); + if (!name) { + throw new Error('getRewrittenLocation only accepts fully-qualified paths'); + } + + const syntheticPackages = ['@embroider/synthesized-styles', '@embroider/synthesized-vendor']; + + if (syntheticPackages.includes(name)) { + return `node_modules/.embroider/rewritten-packages/${name}/${addonPath.slice(name.length)}`; + } + + let h = crypto.createHash('sha1'); + let hash = h.update(`${appDir}/node_modules/${name}`).digest('hex').slice(0, 8); + + return `node_modules/.embroider/rewritten-packages/${name}.${hash}/${addonPath.slice(name.length)}`; +} + +export function expectRewrittenAddonFilesAt(basePath: string, params: { qunit: Assert }): ExpectFile { + let func: any = (addonPath: string) => { + return new BoundExpectFile(basePath, getRewrittenLocation(basePath, addonPath), new QUnitAdapter(params.qunit)); + }; + Object.defineProperty(func, 'basePath', { + get() { + return basePath; + }, + }); + return func; +} + export { ExpectFile }; From 490dd2de7229c596ddcbb699bd36a66db578229c Mon Sep 17 00:00:00 2001 From: Chris Manson Date: Thu, 15 Jun 2023 13:29:01 +0100 Subject: [PATCH 37/72] refactor addon-styles tests to use expectRewrittenAddonFilesAt --- tests/scenarios/compat-addon-styles-test.ts | 26 +++++++-------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/tests/scenarios/compat-addon-styles-test.ts b/tests/scenarios/compat-addon-styles-test.ts index f2dbe6045..b4335c73a 100644 --- a/tests/scenarios/compat-addon-styles-test.ts +++ b/tests/scenarios/compat-addon-styles-test.ts @@ -1,9 +1,7 @@ -import { expectFilesAt, ExpectFile } from '@embroider/test-support/file-assertions/qunit'; +import { expectRewrittenAddonFilesAt, ExpectFile } from '@embroider/test-support/file-assertions/qunit'; import { PreparedApp } from 'scenario-tester'; import { throwOnWarnings } from '@embroider/core'; import { appScenarios, baseAddon } from './scenarios'; -import { readFileSync } from 'fs'; -import { join } from 'path'; import QUnit from 'qunit'; import { merge } from 'lodash'; const { module: Qmodule, test } = QUnit; @@ -93,7 +91,7 @@ appScenarios let app: PreparedApp; - let expectFile: ExpectFile; + let expectAddonFile: ExpectFile; hooks.before(async assert => { app = await scenario.prepare(); @@ -102,32 +100,26 @@ appScenarios }); hooks.beforeEach(assert => { - expectFile = expectFilesAt(readFileSync(join(app.dir, 'dist/.stage1-output'), 'utf8'), { qunit: assert }); + expectAddonFile = expectRewrittenAddonFilesAt(app.dir, { qunit: assert }); }); test('treeForStyles adds styles to build', function () { - expectFile('node_modules/@embroider/synthesized-styles/assets/third-party1.css').matches( - '.error { color: red; }' - ); + expectAddonFile('@embroider/synthesized-styles/assets/third-party1.css').matches('.error { color: red; }'); }); // prevent regression of https://github.com/embroider-build/embroider/issues/164 test('treeForStyles not calling super adds styles to build', function () { - expectFile('node_modules/@embroider/synthesized-styles/assets/third-party2.css').matches( - '.success { color: green }' - ); + expectAddonFile('@embroider/synthesized-styles/assets/third-party2.css').matches('.success { color: green }'); }); test(`all addon CSS gets convert to implicit-styles`, function () { - let implicitStyles = expectFile('node_modules/my-addon3/package.json') - .json() - .get('ember-addon.implicit-styles'); + let implicitStyles = expectAddonFile('my-addon3/package.json').json().get('ember-addon.implicit-styles'); implicitStyles.includes('./my-addon3.css'); implicitStyles.includes('./outer.css'); implicitStyles.includes('./nested/inner.css'); - expectFile('node_modules/my-addon3/my-addon3.css').matches(`from-addon`); - expectFile('node_modules/my-addon3/outer.css').matches(`from-outer`); - expectFile('node_modules/my-addon3/nested/inner.css').matches(`from-inner`); + expectAddonFile('my-addon3/my-addon3.css').matches(`from-addon`); + expectAddonFile('my-addon3/outer.css').matches(`from-outer`); + expectAddonFile('my-addon3/nested/inner.css').matches(`from-inner`); }); }); }); From 18c3af862c195e45251e230a76e0b9159c30708b Mon Sep 17 00:00:00 2001 From: Chris Manson Date: Thu, 15 Jun 2023 13:29:29 +0100 Subject: [PATCH 38/72] fix: reuse variable for newRoot in standalone addon build --- packages/compat/src/standalone-addon-build.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/compat/src/standalone-addon-build.ts b/packages/compat/src/standalone-addon-build.ts index ab96e9e68..dae96ac7e 100644 --- a/packages/compat/src/standalone-addon-build.ts +++ b/packages/compat/src/standalone-addon-build.ts @@ -41,7 +41,7 @@ function buildAddonIndex(compatApp: CompatApp, packages: Set): Rewritte }; for (let oldPkg of packages) { let newRoot = `${oldPkg.name}.${hashed(oldPkg.root)}`; - content.packages[oldPkg.root] = `${oldPkg.name}.${hashed(oldPkg.root)}`; + content.packages[oldPkg.root] = newRoot; let nonResolvableDeps = oldPkg.nonResolvableDeps; if (nonResolvableDeps) { content.extraResolutions[newRoot] = [...nonResolvableDeps.values()].map(v => v.root); From c4057c977b5f99ff300bbf9b4f7e934a644f72c3 Mon Sep 17 00:00:00 2001 From: Chris Manson Date: Thu, 15 Jun 2023 15:57:08 +0100 Subject: [PATCH 39/72] make test audits use the resolver to find the moved app package --- packages/compat/src/audit.ts | 60 ++++++++++----------------- packages/compat/src/audit/options.ts | 3 +- tests/scenarios/core-resolver-test.ts | 2 +- 3 files changed, 24 insertions(+), 41 deletions(-) diff --git a/packages/compat/src/audit.ts b/packages/compat/src/audit.ts index 72bcfebbe..d60fe1024 100644 --- a/packages/compat/src/audit.ts +++ b/packages/compat/src/audit.ts @@ -1,6 +1,6 @@ import { readFileSync, readJSONSync } from 'fs-extra'; import { dirname, join, resolve as resolvePath } from 'path'; -import { AppMeta, explicitRelative, hbsToJS, Resolver, ResolverOptions } from '@embroider/core'; +import { AppMeta, explicitRelative, hbsToJS, Resolver, ResolverOptions, RewrittenPackageCache } from '@embroider/core'; import { Memoize } from 'typescript-memoize'; import chalk from 'chalk'; import jsdom from 'jsdom'; @@ -213,18 +213,11 @@ export class Audit { private frames = new CodeFrameStorage(); static async run(options: AuditBuildOptions): Promise { - let dir: string; - - if (options.outputDir) { - dir = options.outputDir; - } else { - if (!options['reuse-build']) { - await buildApp(options); - } - dir = await this.findStage2Output(options); + if (!options['reuse-build']) { + await buildApp(options); } - let audit = new this(dir, options); + let audit = new this(options.app, options); if (options['reuse-build']) { if (!audit.meta.babel.isParallelSafe) { throw new BuildError( @@ -237,29 +230,17 @@ export class Audit { return audit.run(); } - private static async findStage2Output(options: AuditBuildOptions): Promise { - try { - if (!options.app) { - throw new Error(`AuditBuildOptions needs "app" directory`); - } - return readFileSync(join(options.app, 'dist/.stage2-output'), 'utf8'); - } catch (err) { - if (err.code === 'ENOENT') { - throw new BuildError( - `${chalk.yellow( - 'Your build' - )} did not produce expected Embroider stage2 output.\nMake sure you actually have Embroider configured.` - ); - } - throw err; - } - } - - constructor(private appDir: string, private options: AuditOptions = {}) {} + constructor(private originAppRoot: string, private options: AuditOptions = {}) {} @Memoize() private get pkg() { - return readJSONSync(join(this.appDir, 'package.json')); + return readJSONSync(join(this.movedAppRoot, 'package.json')); + } + + @Memoize() + private get movedAppRoot() { + let cache = RewrittenPackageCache.shared('embroider', this.originAppRoot); + return cache.maybeMoved(cache.get(this.originAppRoot)).root; } private get meta() { @@ -269,7 +250,7 @@ export class Audit { @Memoize() private get babelConfig() { // eslint-disable-next-line @typescript-eslint/no-require-imports - let config = require(join(this.appDir, this.meta.babel.filename)); + let config = require(join(this.movedAppRoot, this.meta.babel.filename)); config = Object.assign({}, config); config.plugins = config.plugins.filter((p: any) => !isMacrosPlugin(p)); @@ -278,7 +259,7 @@ export class Audit { } private get resolverParams(): ResolverOptions { - return readJSONSync(join(this.appDir, 'node_modules', '.embroider', 'resolver.json')); + return readJSONSync(join(this.originAppRoot, 'node_modules', '.embroider', 'resolver.json')); } private resolver = new Resolver(this.resolverParams); @@ -344,13 +325,13 @@ export class Audit { this.debug(`meta`, this.meta); for (let asset of this.meta.assets) { if (asset.endsWith('.html')) { - this.scheduleVisit(resolvePath(this.appDir, asset), { isRoot: true }); + this.scheduleVisit(resolvePath(this.movedAppRoot, asset), { isRoot: true }); } } await this.drainQueue(); this.linkModules(); this.inspectModules(); - return AuditResults.create(this.appDir, this.findings, this.modules); + return AuditResults.create(this.movedAppRoot, this.findings, this.modules); } finally { delete (globalThis as any).embroider_audit; } @@ -468,7 +449,10 @@ export class Audit { } if (src.startsWith(this.meta['root-url'])) { // root-relative URLs are actually relative to the appDir - src = explicitRelative(dirname(filename), resolvePath(this.appDir, src.replace(this.meta['root-url'], ''))); + src = explicitRelative( + dirname(filename), + resolvePath(this.movedAppRoot, src.replace(this.meta['root-url'], '')) + ); } dependencies.push(src); } @@ -513,7 +497,7 @@ export class Audit { { filename, message: `failed to parse`, - detail: err.toString().replace(filename, explicitRelative(this.appDir, filename)), + detail: err.toString().replace(filename, explicitRelative(this.originAppRoot, filename)), }, ]; } else { @@ -544,7 +528,7 @@ export class Audit { { filename, message: `failed to parse JSON`, - detail: err.toString().replace(filename, explicitRelative(this.appDir, filename)), + detail: err.toString().replace(filename, explicitRelative(this.originAppRoot, filename)), }, ]; } diff --git a/packages/compat/src/audit/options.ts b/packages/compat/src/audit/options.ts index 7dd63c1d7..f84cf9d57 100644 --- a/packages/compat/src/audit/options.ts +++ b/packages/compat/src/audit/options.ts @@ -4,6 +4,5 @@ export interface AuditOptions { export interface AuditBuildOptions extends AuditOptions { 'reuse-build'?: boolean; - app?: string; - outputDir?: string; + app: string; } diff --git a/tests/scenarios/core-resolver-test.ts b/tests/scenarios/core-resolver-test.ts index fbf0456c6..96bff154c 100644 --- a/tests/scenarios/core-resolver-test.ts +++ b/tests/scenarios/core-resolver-test.ts @@ -129,7 +129,7 @@ Scenarios.fromProject(() => new Project()) 'node_modules/my-addon/package.json': addonPackageJSON(opts?.addonMeta), }); - expectAudit = await assert.audit({ outputDir: app.dir }); + expectAudit = await assert.audit({ app: app.dir, 'reuse-build': true }); }; }); From cb6fd31e99d07cdfd412d7f3edf42b46e56e55bd Mon Sep 17 00:00:00 2001 From: Chris Manson Date: Thu, 15 Jun 2023 16:00:05 +0100 Subject: [PATCH 40/72] fix lint --- test-packages/support/file-assertions/qunit.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-packages/support/file-assertions/qunit.ts b/test-packages/support/file-assertions/qunit.ts index 567b7dfa0..8c8050759 100644 --- a/test-packages/support/file-assertions/qunit.ts +++ b/test-packages/support/file-assertions/qunit.ts @@ -41,7 +41,7 @@ export function expectFilesAt(basePath: string, params: { qunit: Assert }): Expe return func; } -function getRewrittenLocation(appDir: string, addonPath: string){ +function getRewrittenLocation(appDir: string, addonPath: string) { let name = packageName(addonPath); if (!name) { throw new Error('getRewrittenLocation only accepts fully-qualified paths'); From 402e91397adf8e358ca4625530a585759fe61b9c Mon Sep 17 00:00:00 2001 From: Chris Manson Date: Thu, 15 Jun 2023 16:58:01 +0100 Subject: [PATCH 41/72] fix more `expectFilesAt` tests --- tests/scenarios/compat-app-dot-import-test.ts | 13 ++++++------ .../compat-exclude-dot-files-test.ts | 10 ++++++---- tests/scenarios/compat-preprocessors-test.ts | 20 ++++++++++--------- 3 files changed, 24 insertions(+), 19 deletions(-) diff --git a/tests/scenarios/compat-app-dot-import-test.ts b/tests/scenarios/compat-app-dot-import-test.ts index 306d72517..6bcb8302c 100644 --- a/tests/scenarios/compat-app-dot-import-test.ts +++ b/tests/scenarios/compat-app-dot-import-test.ts @@ -1,8 +1,7 @@ -import { expectFilesAt, ExpectFile } from '@embroider/test-support/file-assertions/qunit'; +import { expectRewrittenAddonFilesAt, ExpectFile } from '@embroider/test-support/file-assertions/qunit'; import { throwOnWarnings } from '@embroider/core'; import { PreparedApp } from 'scenario-tester'; import { join } from 'path'; -import { readFileSync } from 'fs'; import { appScenarios, baseAddon } from './scenarios'; import QUnit from 'qunit'; import { merge } from 'lodash'; @@ -46,18 +45,20 @@ appScenarios }); hooks.beforeEach(assert => { - expectFile = expectFilesAt(readFileSync(join(app.dir, 'dist/.stage2-output'), 'utf8'), { qunit: assert }); + expectFile = expectRewrittenAddonFilesAt(app.dir, { + qunit: assert, + }); }); test('destDir puts vendor files into public assets', function () { - expectFile('node_modules/@embroider/synthesized-vendor/package.json') + expectFile('@embroider/synthesized-vendor/package.json') .json() .get(['ember-addon', 'public-assets', './vendor/some-font.ttf']) .equals('fonts/some-font.ttf'); - expectFile('node_modules/@embroider/synthesized-vendor/vendor/some-font.ttf').exists(); + expectFile('@embroider/synthesized-vendor/vendor/some-font.ttf').exists(); }); test('handle non-transformed node_module with explicit outputFile', function () { - expectFile('node_modules/@embroider/synthesized-vendor/package.json') + expectFile('@embroider/synthesized-vendor/package.json') .json() .get([ 'ember-addon', diff --git a/tests/scenarios/compat-exclude-dot-files-test.ts b/tests/scenarios/compat-exclude-dot-files-test.ts index 86460d676..5cf71a54b 100644 --- a/tests/scenarios/compat-exclude-dot-files-test.ts +++ b/tests/scenarios/compat-exclude-dot-files-test.ts @@ -1,4 +1,4 @@ -import { ExpectFile, expectFilesAt } from '@embroider/test-support/file-assertions/qunit'; +import { ExpectFile, expectFilesAt, expectRewrittenAddonFilesAt } from '@embroider/test-support/file-assertions/qunit'; import { throwOnWarnings } from '@embroider/core'; import { PreparedApp } from 'scenario-tester'; import { appScenarios, baseAddon } from './scenarios'; @@ -35,6 +35,7 @@ appScenarios let app: PreparedApp; let expectFile: ExpectFile; + let expectAddonFile: ExpectFile; hooks.before(async assert => { app = await scenario.prepare(); @@ -44,6 +45,7 @@ appScenarios hooks.beforeEach(assert => { expectFile = expectFilesAt(readFileSync(join(app.dir, 'dist/.stage2-output'), 'utf8'), { qunit: assert }); + expectAddonFile = expectRewrittenAddonFilesAt(app.dir, { qunit: assert }); }); test('dot files are not included as app modules', function () { @@ -60,10 +62,10 @@ appScenarios test('dot files are not included as addon implicit-modules', function () { // Dot files should exist on disk - expectFile('node_modules/my-addon/.fooaddon.js').exists(); - expectFile('node_modules/my-addon/baraddon.js').exists(); + expectAddonFile('my-addon/.fooaddon.js').exists(); + expectAddonFile('my-addon/baraddon.js').exists(); - let myAddonPackage = expectFile('node_modules/my-addon/package.json').json(); + let myAddonPackage = expectAddonFile('my-addon/package.json').json(); // dot files are not included as implicit-modules myAddonPackage.get(['ember-addon', 'implicit-modules']).deepEquals(['./baraddon']); diff --git a/tests/scenarios/compat-preprocessors-test.ts b/tests/scenarios/compat-preprocessors-test.ts index a422241ac..99b5e51dd 100644 --- a/tests/scenarios/compat-preprocessors-test.ts +++ b/tests/scenarios/compat-preprocessors-test.ts @@ -2,7 +2,7 @@ import { PreparedApp } from 'scenario-tester'; import { appScenarios, baseAddon } from './scenarios'; import { readFileSync } from 'fs'; import { join } from 'path'; -import { ExpectFile, expectFilesAt } from '@embroider/test-support/file-assertions/qunit'; +import { ExpectFile, expectFilesAt, expectRewrittenAddonFilesAt } from '@embroider/test-support/file-assertions/qunit'; import { throwOnWarnings } from '@embroider/core'; import merge from 'lodash/merge'; import QUnit from 'qunit'; @@ -20,19 +20,19 @@ appScenarios module.exports = { name: require('./package').name, - + setupPreprocessorRegistry(type, registry) { if (type !== 'parent') { return; } - + registry.add('js', { name: 'special-path-processor', toTree(tree, inputPath) { if (inputPath !== '/') { return tree; } - + let augmented = map( tree, '**/*.{js,css}', @@ -92,6 +92,7 @@ appScenarios let app: PreparedApp; let expectFile: ExpectFile; + let expectAddonFile: ExpectFile; hooks.before(async assert => { app = await scenario.prepare(); @@ -101,14 +102,15 @@ appScenarios hooks.beforeEach(assert => { expectFile = expectFilesAt(readFileSync(join(app.dir, 'dist/.stage2-output'), 'utf8'), { qunit: assert }); + expectAddonFile = expectRewrittenAddonFilesAt(app.dir, { qunit: assert }); }); test('dependencies are setup for this test suite correctly', () => { expectFile('package.json').exists(); expectFile('package.json').matches(/my-preprocessor/, 'has the preprocessor dependency'); - expectFile('node_modules/my-addon/package.json').exists(); - expectFile('node_modules/my-addon/package.json').matches(/my-preprocessor/, 'has the preprocessor dependency'); - expectFile('node_modules/my-preprocessor/package.json').exists(); + expectAddonFile('my-addon/package.json').exists(); + expectAddonFile('my-addon/package.json').matches(/my-preprocessor/, 'has the preprocessor dependency'); + expectAddonFile('my-preprocessor/package.json').exists(); }); test('app has correct path embedded in comment', () => { @@ -119,8 +121,8 @@ appScenarios }); test('addon has correct path embedded in comment', () => { - expectFile('node_modules/my-preprocessor/package.json').exists(); - const assertFile = expectFile('node_modules/my-addon/components/greeting.js'); + expectAddonFile('my-preprocessor/package.json').exists(); + const assertFile = expectAddonFile('my-addon/components/greeting.js'); assertFile.matches(/path@my-addon\/components\/greeting\.js/, 'has a path comment in app components'); }); }); From 84b881b853dbaa8eeaae11ef19793c9fc5dba796 Mon Sep 17 00:00:00 2001 From: Chris Manson Date: Thu, 15 Jun 2023 17:50:14 +0100 Subject: [PATCH 42/72] fix more `expectFilesAt` tests --- tests/scenarios/compat-renaming-test.ts | 18 +++++----- tests/scenarios/compat-route-split-test.ts | 4 +-- tests/scenarios/compat-stage2-test.ts | 33 +++++++------------ .../compat-template-colocation-test.ts | 14 +++++--- 4 files changed, 32 insertions(+), 37 deletions(-) diff --git a/tests/scenarios/compat-renaming-test.ts b/tests/scenarios/compat-renaming-test.ts index 145f551ab..375f2ed3d 100644 --- a/tests/scenarios/compat-renaming-test.ts +++ b/tests/scenarios/compat-renaming-test.ts @@ -6,7 +6,7 @@ import QUnit from 'qunit'; const { module: Qmodule, test } = QUnit; import { definesPattern, Transpiler } from '@embroider/test-support'; -import { ExpectFile, expectFilesAt } from '@embroider/test-support/file-assertions/qunit'; +import { ExpectFile, expectFilesAt, expectRewrittenAddonFilesAt } from '@embroider/test-support/file-assertions/qunit'; import { throwOnWarnings } from '@embroider/core'; import merge from 'lodash/merge'; @@ -164,6 +164,7 @@ appScenarios let app: PreparedApp; let expectFile: ExpectFile; + let expectAddonFile: ExpectFile; let build: Transpiler; hooks.before(async () => { @@ -172,6 +173,7 @@ appScenarios hooks.beforeEach(assert => { expectFile = expectFilesAt(readFileSync(join(app.dir, 'dist/.stage2-output'), 'utf8'), { qunit: assert }); + expectAddonFile = expectRewrittenAddonFilesAt(app.dir, { qunit: assert }); build = new Transpiler(expectFile.basePath); }); @@ -186,7 +188,7 @@ appScenarios .module('./components/import-lodash.js') .resolves('lodash') .to('./node_modules/ember-lodash/index.js'); - expectFile('node_modules/ember-lodash/index.js').matches(/lodash index/); + expectAddonFile('ember-lodash/index.js').matches(/lodash index/); }); test('whole package renaming works for interior module', function () { @@ -195,7 +197,7 @@ appScenarios .resolves('lodash/capitalize') .to('./node_modules/ember-lodash/capitalize.js'); - expectFile('node_modules/ember-lodash/capitalize.js').matches(/lodash capitalize/); + expectAddonFile('ember-lodash/capitalize.js').matches(/lodash capitalize/); }); test("modules in own namespace don't get renamed", function () { @@ -203,7 +205,7 @@ appScenarios .module('./components/import-own-thing.js') .resolves('emits-multiple-packages/own-thing') .to('./node_modules/emits-multiple-packages/own-thing.js'); - expectFile('node_modules/emits-multiple-packages/own-thing.js').matches(/own thing/); + expectAddonFile('emits-multiple-packages/own-thing.js').matches(/own thing/); }); test('modules outside our namespace do get renamed', function () { @@ -211,7 +213,7 @@ appScenarios .module('./components/import-somebody-elses.js') .resolves('somebody-elses-package/environment') .to('./node_modules/emits-multiple-packages/somebody-elses-package/environment.js'); - expectFile('node_modules/emits-multiple-packages/somebody-elses-package/environment.js').matches( + expectAddonFile('emits-multiple-packages/somebody-elses-package/environment.js').matches( /somebody elses environment/ ); }); @@ -221,7 +223,7 @@ appScenarios .module('./components/import-somebody-elses-utils.js') .resolves('somebody-elses-package/utils') .to('./node_modules/emits-multiple-packages/somebody-elses-package/utils/index.js'); - expectFile('node_modules/emits-multiple-packages/somebody-elses-package/utils/index.js').matches( + expectAddonFile('emits-multiple-packages/somebody-elses-package/utils/index.js').matches( /somebody elses utils/ ); }); @@ -263,9 +265,7 @@ appScenarios .module('./components/import-single-file-package.js') .resolves('single-file-package') .to('./node_modules/emits-multiple-packages/single-file-package/index.js'); - expectFile('./node_modules/emits-multiple-packages/single-file-package/index.js').matches( - /single file package/ - ); + expectAddonFile('emits-multiple-packages/single-file-package/index.js').matches(/single file package/); }); test('files logically copied into app from addons resolve their own original packages', function () { expectAudit diff --git a/tests/scenarios/compat-route-split-test.ts b/tests/scenarios/compat-route-split-test.ts index ae0704a9d..4af286921 100644 --- a/tests/scenarios/compat-route-split-test.ts +++ b/tests/scenarios/compat-route-split-test.ts @@ -175,7 +175,7 @@ splitScenarios Qmodule('audit', function (hooks) { let auditResults: AuditResults; hooks.before(async function () { - let audit = new Audit(expectFile.basePath); + let audit = new Audit(app.dir); auditResults = await audit.run(); }); @@ -357,7 +357,7 @@ splitScenarios Qmodule('audit', function (hooks) { let auditResults: AuditResults; hooks.before(async function () { - let audit = new Audit(expectFile.basePath); + let audit = new Audit(app.dir); auditResults = await audit.run(); }); diff --git a/tests/scenarios/compat-stage2-test.ts b/tests/scenarios/compat-stage2-test.ts index d427d4dfb..9a6963de4 100644 --- a/tests/scenarios/compat-stage2-test.ts +++ b/tests/scenarios/compat-stage2-test.ts @@ -5,7 +5,7 @@ import { appScenarios, baseAddon, dummyAppScenarios, renameApp } from './scenari import { readFileSync } from 'fs'; import { join } from 'path'; import { Rebuilder, Transpiler } from '@embroider/test-support'; -import { expectFilesAt, ExpectFile } from '@embroider/test-support/file-assertions/qunit'; +import { expectFilesAt, expectRewrittenAddonFilesAt, ExpectFile } from '@embroider/test-support/file-assertions/qunit'; import { throwOnWarnings } from '@embroider/core'; import merge from 'lodash/merge'; import QUnit from 'qunit'; @@ -82,7 +82,7 @@ stage2Scenarios throwOnWarnings(hooks); let app: PreparedApp; - let expectFile: ExpectFile; + let expectAddonFile: ExpectFile; hooks.before(async assert => { app = await scenario.prepare(); @@ -91,35 +91,26 @@ stage2Scenarios }); hooks.beforeEach(assert => { - expectFile = expectFilesAt(readFileSync(join(app.dir, 'dist/.stage2-output'), 'utf8'), { qunit: assert }); + expectAddonFile = expectRewrittenAddonFilesAt(app.dir, { qunit: assert }); }); let expectAudit = setupAuditTest(hooks, () => app.dir); test('in repo addons are symlinked correctly', function () { // check that package json contains in repo dep - expectFile('./node_modules/dep-a/package.json').json().get('dependencies.in-repo-a').equals('0.0.0'); - expectFile('./node_modules/dep-b/package.json').json().get('dependencies.in-repo-b').equals('0.0.0'); - expectFile('./node_modules/dep-b/package.json').json().get('dependencies.in-repo-c').equals('0.0.0'); + expectAddonFile('dep-a/package.json').json().get('dependencies.in-repo-a').equals('0.0.0'); + expectAddonFile('dep-b/package.json').json().get('dependencies.in-repo-c').equals('0.0.0'); + expectAddonFile('dep-b/package.json').json().get('dependencies.in-repo-b').equals('0.0.0'); // check that symlinks are correct - expectFile('./node_modules/dep-a/node_modules/in-repo-a/package.json').exists(); - expectFile('./node_modules/dep-b/node_modules/in-repo-b/package.json').exists(); - expectFile('./node_modules/dep-b/node_modules/in-repo-c/package.json').exists(); + expectAddonFile('dep-a/node_modules/in-repo-a/package.json').exists(); + expectAddonFile('dep-b/node_modules/in-repo-b/package.json').exists(); + expectAddonFile('dep-b/node_modules/in-repo-c/package.json').exists(); // check that the in repo addons are correct upgraded - expectFile('./node_modules/dep-a/node_modules/in-repo-a/package.json') - .json() - .get('ember-addon.version') - .equals(2); - expectFile('./node_modules/dep-b/node_modules/in-repo-b/package.json') - .json() - .get('ember-addon.version') - .equals(2); - expectFile('./node_modules/dep-b/node_modules/in-repo-c/package.json') - .json() - .get('ember-addon.version') - .equals(2); + expectAddonFile('dep-a/node_modules/in-repo-a/package.json').json().get('ember-addon.version').equals(2); + expectAddonFile('dep-b/node_modules/in-repo-b/package.json').json().get('ember-addon.version').equals(2); + expectAddonFile('dep-b/node_modules/in-repo-c/package.json').json().get('ember-addon.version').equals(2); // check that the app trees with in repo addon are combined correctly expectAudit diff --git a/tests/scenarios/compat-template-colocation-test.ts b/tests/scenarios/compat-template-colocation-test.ts index a4032865c..392313d54 100644 --- a/tests/scenarios/compat-template-colocation-test.ts +++ b/tests/scenarios/compat-template-colocation-test.ts @@ -3,7 +3,7 @@ import { appScenarios, baseAddon, renameApp } from './scenarios'; import { readFileSync } from 'fs'; import { join } from 'path'; import { Transpiler } from '@embroider/test-support'; -import { ExpectFile, expectFilesAt } from '@embroider/test-support/file-assertions/qunit'; +import { ExpectFile, expectFilesAt, expectRewrittenAddonFilesAt } from '@embroider/test-support/file-assertions/qunit'; import { throwOnWarnings } from '@embroider/core'; import merge from 'lodash/merge'; import QUnit from 'qunit'; @@ -83,6 +83,7 @@ scenarios let app: PreparedApp; let expectFile: ExpectFile; + let expectAddonFile: ExpectFile; let build: Transpiler; hooks.before(async assert => { @@ -93,6 +94,7 @@ scenarios hooks.beforeEach(assert => { expectFile = expectFilesAt(readFileSync(join(app.dir, 'dist/.stage2-output'), 'utf8'), { qunit: assert }); + expectAddonFile = expectRewrittenAddonFilesAt(app.dir, { qunit: assert }); build = new Transpiler(expectFile.basePath); }); @@ -135,7 +137,7 @@ scenarios }); test(`addon's colocated template is associated with JS`, function () { - let assertFile = expectFile('node_modules/my-addon/components/component-one.js').transform(build.transpile); + let assertFile = expectAddonFile('my-addon/components/component-one.js').transform(build.transpile); assertFile.matches(/import TEMPLATE from ['"]\.\/component-one.hbs['"];/, 'imported template'); assertFile.matches(/import \{ setComponentTemplate \}/, 'found setComponentTemplate'); assertFile.matches( @@ -145,7 +147,7 @@ scenarios }); test(`addon's template-only component JS is synthesized`, function () { - let assertFile = expectFile('node_modules/my-addon/components/component-two.js').transform(build.transpile); + let assertFile = expectAddonFile('my-addon/components/component-two.js').transform(build.transpile); assertFile.matches(/import TEMPLATE from ['"]\.\/component-two.hbs['"];/, 'imported template'); assertFile.matches(/import \{ setComponentTemplate \}/, 'found setComponentTemplate'); assertFile.matches(/import templateOnlyComponent/, 'found templateOnlyComponent'); @@ -156,7 +158,7 @@ scenarios }); test(`addon's colocated components are correct in implicit-modules`, function () { - let assertFile = expectFile('node_modules/my-addon/package.json').json(); + let assertFile = expectAddonFile('my-addon/package.json').json(); assertFile.get(['ember-addon', 'implicit-modules']).includes('./components/component-one'); assertFile.get(['ember-addon', 'implicit-modules']).includes('./components/component-two'); assertFile.get(['ember-addon', 'implicit-modules']).doesNotInclude('./components/component-one.hbs'); @@ -188,6 +190,7 @@ scenarios let app: PreparedApp; let expectFile: ExpectFile; + let expectAddonFile: ExpectFile; hooks.before(async assert => { app = await scenario.prepare(); @@ -197,6 +200,7 @@ scenarios hooks.beforeEach(assert => { expectFile = expectFilesAt(readFileSync(join(app.dir, 'dist/.stage2-output'), 'utf8'), { qunit: assert }); + expectAddonFile = expectRewrittenAddonFilesAt(app.dir, { qunit: assert }); }); test(`app's colocated components are not implicitly included`, function () { @@ -210,7 +214,7 @@ scenarios }); test(`addon's colocated components are not in implicit-modules`, function () { - let assertFile = expectFile('node_modules/my-addon/package.json').json(); + let assertFile = expectAddonFile('my-addon/package.json').json(); assertFile.get(['ember-addon', 'implicit-modules']).equals(undefined); }); }); From 03ee0a40a25e93f08a44790483f1155ab2596757 Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Fri, 16 Jun 2023 12:26:02 -0400 Subject: [PATCH 43/72] make audit assertions understand rewritten packages And fix some detected resolver issues. --- packages/compat/src/audit.ts | 3 +- packages/core/src/module-resolver.ts | 26 ++++---- test-packages/support/audit-assertions.ts | 77 ++++++++++++++++------- tests/scenarios/compat-renaming-test.ts | 2 +- 4 files changed, 73 insertions(+), 35 deletions(-) diff --git a/packages/compat/src/audit.ts b/packages/compat/src/audit.ts index d60fe1024..16ec301dc 100644 --- a/packages/compat/src/audit.ts +++ b/packages/compat/src/audit.ts @@ -331,7 +331,8 @@ export class Audit { await this.drainQueue(); this.linkModules(); this.inspectModules(); - return AuditResults.create(this.movedAppRoot, this.findings, this.modules); + + return AuditResults.create(this.originAppRoot, this.findings, this.modules); } finally { delete (globalThis as any).embroider_audit; } diff --git a/packages/core/src/module-resolver.ts b/packages/core/src/module-resolver.ts index ef4b89307..79947e0e7 100644 --- a/packages/core/src/module-resolver.ts +++ b/packages/core/src/module-resolver.ts @@ -645,7 +645,7 @@ export class Resolver { return logTransition( 'outbound request from moved package', request, - request.rehome(resolve(originalRequestingPkg.root, 'package.json')) + request.rehome(resolve(originalRequestingPkg.root, request.fromFile.slice(requestingPkg.root.length + 1))) ); } @@ -840,7 +840,11 @@ export class Resolver { // to do the defaultResolve from there, now we refer back to the rewritten // location because that's what we want to use when asking things like // isV2Ember() - pkg = this.packageCache.maybeMoved(pkg); + let movedPkg = this.packageCache.maybeMoved(pkg); + if (movedPkg !== pkg) { + fromFile = resolve(movedPkg.root, request.fromFile.slice(pkg.root.length + 1)); + pkg = movedPkg; + } if (!pkg.isV2Ember()) { return logTransition('fallbackResolve: not in an ember package', request); @@ -882,15 +886,7 @@ export class Resolver { ); } - let targetingEngine = this.engineConfig(packageName); - if (targetingEngine) { - let appJSMatch = this.searchAppTree(request, targetingEngine, specifier.replace(packageName, '.')); - if (appJSMatch) { - return logTransition('fallbackResolve: non-relative appJsMatch', request, appJSMatch); - } - } - - let logicalLocation = this.reverseSearchAppTree(pkg, request.fromFile); + let logicalLocation = this.reverseSearchAppTree(pkg, fromFile); if (logicalLocation) { // the requesting file is in an addon's appTree. We didn't succeed in // resolving this (non-relative) request from inside the actual addon, so @@ -903,6 +899,14 @@ export class Resolver { ); } + let targetingEngine = this.engineConfig(packageName); + if (targetingEngine) { + let appJSMatch = this.searchAppTree(request, targetingEngine, specifier.replace(packageName, '.')); + if (appJSMatch) { + return logTransition('fallbackResolve: non-relative appJsMatch', request, appJSMatch); + } + } + if (pkg.meta['auto-upgraded']) { // auto-upgraded packages can fall back to attempting to find dependencies at // runtime. Native v2 packages can only get this behavior in the diff --git a/test-packages/support/audit-assertions.ts b/test-packages/support/audit-assertions.ts index 4c923d27f..255797672 100644 --- a/test-packages/support/audit-assertions.ts +++ b/test-packages/support/audit-assertions.ts @@ -1,7 +1,7 @@ import { Audit, AuditBuildOptions, AuditResults, Module } from '../../packages/compat/src/audit'; -import { explicitRelative } from '../../packages/shared-internals'; +import { explicitRelative, RewrittenPackageCache } from '../../packages/shared-internals'; import { install as installCodeEqualityAssertions } from 'code-equality-assertions/qunit'; -import { posix } from 'path'; +import { posix, resolve } from 'path'; import { distance } from 'fastest-levenshtein'; import { sortBy } from 'lodash'; @@ -21,7 +21,7 @@ export function setupAuditTest(hooks: NestedHooks, getAppDir: () => string) { hooks.beforeEach(assert => { installAuditAssertions(assert); - expectAudit = new ExpectAuditResults(result, assert); + expectAudit = new ExpectAuditResults(result, assert, getAppDir()); }); return { @@ -38,7 +38,7 @@ export function setupAuditTest(hooks: NestedHooks, getAppDir: () => string) { } async function audit(this: Assert, opts: AuditBuildOptions): Promise { - return new ExpectAuditResults(await Audit.run(opts), this); + return new ExpectAuditResults(await Audit.run(opts), this, opts.app); } export function installAuditAssertions(assert: Assert) { @@ -53,21 +53,40 @@ declare global { } export class ExpectAuditResults { - constructor(readonly result: AuditResults, private assert: Assert) {} + private packageCache = RewrittenPackageCache.shared('embroider', this.appDir); - module(name: string) { - let m = this.result.modules[name]; + constructor(readonly result: AuditResults, private assert: Assert, private appDir: string) {} + + // input and output paths are relative to getAppDir() + private toRewritten = (path: string) => { + let fullPath = resolve(this.appDir, path); + let owner = this.packageCache.ownerOfFile(fullPath); + if (!owner) { + return path; + } + let movedOwner = this.packageCache.maybeMoved(owner); + if (movedOwner === owner) { + return path; + } + let movedFullPath = fullPath.replace(owner.root, movedOwner.root); + return explicitRelative(this.appDir, movedFullPath); + }; + + module(inputName: string) { + let outputName = this.toRewritten(inputName); + let m = this.result.modules[outputName]; const showNearMisses = 4; if (!m) { - let actuals = sortBy(Object.keys(this.result.modules), candidate => distance(candidate, name)); + let actuals = sortBy(Object.keys(this.result.modules), candidate => distance(candidate, outputName)); this.assert.pushResult({ result: false, actual: actuals.length > showNearMisses ? actuals.slice(0, showNearMisses).join(', ') + '...' : actuals.join(', '), - expected: name, + expected: outputName, + message: `Can't locate module ${inputName}`, }); } - return new ExpectModule(this.assert, this.result, m); + return new ExpectModule(this.assert, this.result, m, this.toRewritten); } get findings() { @@ -84,7 +103,12 @@ export class ExpectAuditResults { } export class ExpectModule { - constructor(private assert: Assert, private result: AuditResults, private module: Module | undefined) {} + constructor( + private assert: Assert, + private result: AuditResults, + private module: Module | undefined, + private toRewritten: (s: string) => string + ) {} codeEquals(expectedSource: string) { if (this.module) { @@ -96,7 +120,7 @@ export class ExpectModule { if (!this.module) { // the place that instantiated us already pushed the exception that this // module doesn't exist - return new ExpectResolution(this.assert, this.result, undefined); + return new ExpectResolution(this.assert, this.result, undefined, this.toRewritten); } if (!(specifier in this.module.resolutions)) { this.assert.pushResult({ @@ -104,7 +128,7 @@ export class ExpectModule { expected: `${this.module.appRelativePath} does not refer to ${specifier}`, actual: Object.keys(this.module.resolutions), }); - return new ExpectResolution(this.assert, this.result, undefined); + return new ExpectResolution(this.assert, this.result, undefined, this.toRewritten); } let resolution = this.module.resolutions[specifier]; if (!resolution) { @@ -113,7 +137,7 @@ export class ExpectModule { expected: `${specifier} fails to resolve in ${this.module.appRelativePath}`, actual: `${specifier} to resolve to something`, }); - return new ExpectResolution(this.assert, this.result, undefined); + return new ExpectResolution(this.assert, this.result, undefined, this.toRewritten); } let target = this.result.modules[resolution]; if (!target) { @@ -122,9 +146,9 @@ export class ExpectModule { expected: `${specifier} resolves to ${resolution} but ${resolution} is not found in audit results`, actual: `${resolution} exists`, }); - return new ExpectResolution(this.assert, this.result, undefined); + return new ExpectResolution(this.assert, this.result, undefined, this.toRewritten); } - return new ExpectResolution(this.assert, this.result, target); + return new ExpectResolution(this.assert, this.result, target, this.toRewritten); } // this is testing explicitly for the template-only component moduels that we @@ -143,13 +167,22 @@ export class ExpectModule { } export class ExpectResolution { - constructor(private assert: Assert, private result: AuditResults, private module: Module | undefined) {} - - to(filename: string | null, message?: string) { + constructor( + private assert: Assert, + private result: AuditResults, + private module: Module | undefined, + private toRewritten: (s: string) => string + ) {} + + to(inputName: string | null, message?: string) { + let outputName: string | null = null; + if (inputName) { + outputName = this.toRewritten(inputName); + } if (this.module) { this.assert.pushResult({ - result: this.module.appRelativePath === filename, - expected: filename, + result: this.module.appRelativePath === outputName, + expected: inputName, actual: this.module.appRelativePath, message, }); @@ -157,6 +190,6 @@ export class ExpectResolution { } toModule(): ExpectModule { - return new ExpectModule(this.assert, this.result, this.module); + return new ExpectModule(this.assert, this.result, this.module, this.toRewritten); } } diff --git a/tests/scenarios/compat-renaming-test.ts b/tests/scenarios/compat-renaming-test.ts index 375f2ed3d..525ca19f9 100644 --- a/tests/scenarios/compat-renaming-test.ts +++ b/tests/scenarios/compat-renaming-test.ts @@ -283,7 +283,7 @@ appScenarios .resolves('inner-dep') .to('./node_modules/has-app-tree-import/node_modules/inner-dep/index.js'); }); - test(`app-tree files from addons can import from the app`, function () { + QUnit.only(`app-tree files from addons can import from the app`, function () { expectAudit .module('./node_modules/mirage-like/_app_/mirage/config.js') .resolves('app-template/components/import-lodash') From 7960c7675fa8fcc21525bd9baeec4d151c24dd17 Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Sat, 17 Jun 2023 05:54:15 -0400 Subject: [PATCH 44/72] removing accidental QUnit.only --- tests/scenarios/compat-renaming-test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scenarios/compat-renaming-test.ts b/tests/scenarios/compat-renaming-test.ts index 525ca19f9..375f2ed3d 100644 --- a/tests/scenarios/compat-renaming-test.ts +++ b/tests/scenarios/compat-renaming-test.ts @@ -283,7 +283,7 @@ appScenarios .resolves('inner-dep') .to('./node_modules/has-app-tree-import/node_modules/inner-dep/index.js'); }); - QUnit.only(`app-tree files from addons can import from the app`, function () { + test(`app-tree files from addons can import from the app`, function () { expectAudit .module('./node_modules/mirage-like/_app_/mirage/config.js') .resolves('app-template/components/import-lodash') From 6be2f54754733eb6cf76a1aaf55cd5b92bbab78f Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Sat, 17 Jun 2023 06:10:12 -0400 Subject: [PATCH 45/72] rename --- test-packages/support/file-assertions/qunit.ts | 2 +- tests/scenarios/compat-addon-styles-test.ts | 4 ++-- tests/scenarios/compat-app-dot-import-test.ts | 4 ++-- tests/scenarios/compat-exclude-dot-files-test.ts | 4 ++-- tests/scenarios/compat-preprocessors-test.ts | 4 ++-- tests/scenarios/compat-renaming-test.ts | 4 ++-- tests/scenarios/compat-stage2-test.ts | 4 ++-- tests/scenarios/compat-template-colocation-test.ts | 6 +++--- 8 files changed, 16 insertions(+), 16 deletions(-) diff --git a/test-packages/support/file-assertions/qunit.ts b/test-packages/support/file-assertions/qunit.ts index 8c8050759..1d57b5bac 100644 --- a/test-packages/support/file-assertions/qunit.ts +++ b/test-packages/support/file-assertions/qunit.ts @@ -59,7 +59,7 @@ function getRewrittenLocation(appDir: string, addonPath: string) { return `node_modules/.embroider/rewritten-packages/${name}.${hash}/${addonPath.slice(name.length)}`; } -export function expectRewrittenAddonFilesAt(basePath: string, params: { qunit: Assert }): ExpectFile { +export function expectRewrittenFilesAt(basePath: string, params: { qunit: Assert }): ExpectFile { let func: any = (addonPath: string) => { return new BoundExpectFile(basePath, getRewrittenLocation(basePath, addonPath), new QUnitAdapter(params.qunit)); }; diff --git a/tests/scenarios/compat-addon-styles-test.ts b/tests/scenarios/compat-addon-styles-test.ts index b4335c73a..3c7b3a855 100644 --- a/tests/scenarios/compat-addon-styles-test.ts +++ b/tests/scenarios/compat-addon-styles-test.ts @@ -1,4 +1,4 @@ -import { expectRewrittenAddonFilesAt, ExpectFile } from '@embroider/test-support/file-assertions/qunit'; +import { expectRewrittenFilesAt, ExpectFile } from '@embroider/test-support/file-assertions/qunit'; import { PreparedApp } from 'scenario-tester'; import { throwOnWarnings } from '@embroider/core'; import { appScenarios, baseAddon } from './scenarios'; @@ -100,7 +100,7 @@ appScenarios }); hooks.beforeEach(assert => { - expectAddonFile = expectRewrittenAddonFilesAt(app.dir, { qunit: assert }); + expectAddonFile = expectRewrittenFilesAt(app.dir, { qunit: assert }); }); test('treeForStyles adds styles to build', function () { diff --git a/tests/scenarios/compat-app-dot-import-test.ts b/tests/scenarios/compat-app-dot-import-test.ts index 6bcb8302c..8296f7993 100644 --- a/tests/scenarios/compat-app-dot-import-test.ts +++ b/tests/scenarios/compat-app-dot-import-test.ts @@ -1,4 +1,4 @@ -import { expectRewrittenAddonFilesAt, ExpectFile } from '@embroider/test-support/file-assertions/qunit'; +import { expectRewrittenFilesAt, ExpectFile } from '@embroider/test-support/file-assertions/qunit'; import { throwOnWarnings } from '@embroider/core'; import { PreparedApp } from 'scenario-tester'; import { join } from 'path'; @@ -45,7 +45,7 @@ appScenarios }); hooks.beforeEach(assert => { - expectFile = expectRewrittenAddonFilesAt(app.dir, { + expectFile = expectRewrittenFilesAt(app.dir, { qunit: assert, }); }); diff --git a/tests/scenarios/compat-exclude-dot-files-test.ts b/tests/scenarios/compat-exclude-dot-files-test.ts index 5cf71a54b..79fde5198 100644 --- a/tests/scenarios/compat-exclude-dot-files-test.ts +++ b/tests/scenarios/compat-exclude-dot-files-test.ts @@ -1,4 +1,4 @@ -import { ExpectFile, expectFilesAt, expectRewrittenAddonFilesAt } from '@embroider/test-support/file-assertions/qunit'; +import { ExpectFile, expectFilesAt, expectRewrittenFilesAt } from '@embroider/test-support/file-assertions/qunit'; import { throwOnWarnings } from '@embroider/core'; import { PreparedApp } from 'scenario-tester'; import { appScenarios, baseAddon } from './scenarios'; @@ -45,7 +45,7 @@ appScenarios hooks.beforeEach(assert => { expectFile = expectFilesAt(readFileSync(join(app.dir, 'dist/.stage2-output'), 'utf8'), { qunit: assert }); - expectAddonFile = expectRewrittenAddonFilesAt(app.dir, { qunit: assert }); + expectAddonFile = expectRewrittenFilesAt(app.dir, { qunit: assert }); }); test('dot files are not included as app modules', function () { diff --git a/tests/scenarios/compat-preprocessors-test.ts b/tests/scenarios/compat-preprocessors-test.ts index 99b5e51dd..4489ce002 100644 --- a/tests/scenarios/compat-preprocessors-test.ts +++ b/tests/scenarios/compat-preprocessors-test.ts @@ -2,7 +2,7 @@ import { PreparedApp } from 'scenario-tester'; import { appScenarios, baseAddon } from './scenarios'; import { readFileSync } from 'fs'; import { join } from 'path'; -import { ExpectFile, expectFilesAt, expectRewrittenAddonFilesAt } from '@embroider/test-support/file-assertions/qunit'; +import { ExpectFile, expectFilesAt, expectRewrittenFilesAt } from '@embroider/test-support/file-assertions/qunit'; import { throwOnWarnings } from '@embroider/core'; import merge from 'lodash/merge'; import QUnit from 'qunit'; @@ -102,7 +102,7 @@ appScenarios hooks.beforeEach(assert => { expectFile = expectFilesAt(readFileSync(join(app.dir, 'dist/.stage2-output'), 'utf8'), { qunit: assert }); - expectAddonFile = expectRewrittenAddonFilesAt(app.dir, { qunit: assert }); + expectAddonFile = expectRewrittenFilesAt(app.dir, { qunit: assert }); }); test('dependencies are setup for this test suite correctly', () => { diff --git a/tests/scenarios/compat-renaming-test.ts b/tests/scenarios/compat-renaming-test.ts index 375f2ed3d..02c52e0f4 100644 --- a/tests/scenarios/compat-renaming-test.ts +++ b/tests/scenarios/compat-renaming-test.ts @@ -6,7 +6,7 @@ import QUnit from 'qunit'; const { module: Qmodule, test } = QUnit; import { definesPattern, Transpiler } from '@embroider/test-support'; -import { ExpectFile, expectFilesAt, expectRewrittenAddonFilesAt } from '@embroider/test-support/file-assertions/qunit'; +import { ExpectFile, expectFilesAt, expectRewrittenFilesAt } from '@embroider/test-support/file-assertions/qunit'; import { throwOnWarnings } from '@embroider/core'; import merge from 'lodash/merge'; @@ -173,7 +173,7 @@ appScenarios hooks.beforeEach(assert => { expectFile = expectFilesAt(readFileSync(join(app.dir, 'dist/.stage2-output'), 'utf8'), { qunit: assert }); - expectAddonFile = expectRewrittenAddonFilesAt(app.dir, { qunit: assert }); + expectAddonFile = expectRewrittenFilesAt(app.dir, { qunit: assert }); build = new Transpiler(expectFile.basePath); }); diff --git a/tests/scenarios/compat-stage2-test.ts b/tests/scenarios/compat-stage2-test.ts index 9a6963de4..1edcf27bc 100644 --- a/tests/scenarios/compat-stage2-test.ts +++ b/tests/scenarios/compat-stage2-test.ts @@ -5,7 +5,7 @@ import { appScenarios, baseAddon, dummyAppScenarios, renameApp } from './scenari import { readFileSync } from 'fs'; import { join } from 'path'; import { Rebuilder, Transpiler } from '@embroider/test-support'; -import { expectFilesAt, expectRewrittenAddonFilesAt, ExpectFile } from '@embroider/test-support/file-assertions/qunit'; +import { expectFilesAt, expectRewrittenFilesAt, ExpectFile } from '@embroider/test-support/file-assertions/qunit'; import { throwOnWarnings } from '@embroider/core'; import merge from 'lodash/merge'; import QUnit from 'qunit'; @@ -91,7 +91,7 @@ stage2Scenarios }); hooks.beforeEach(assert => { - expectAddonFile = expectRewrittenAddonFilesAt(app.dir, { qunit: assert }); + expectAddonFile = expectRewrittenFilesAt(app.dir, { qunit: assert }); }); let expectAudit = setupAuditTest(hooks, () => app.dir); diff --git a/tests/scenarios/compat-template-colocation-test.ts b/tests/scenarios/compat-template-colocation-test.ts index 392313d54..98161b418 100644 --- a/tests/scenarios/compat-template-colocation-test.ts +++ b/tests/scenarios/compat-template-colocation-test.ts @@ -3,7 +3,7 @@ import { appScenarios, baseAddon, renameApp } from './scenarios'; import { readFileSync } from 'fs'; import { join } from 'path'; import { Transpiler } from '@embroider/test-support'; -import { ExpectFile, expectFilesAt, expectRewrittenAddonFilesAt } from '@embroider/test-support/file-assertions/qunit'; +import { ExpectFile, expectFilesAt, expectRewrittenFilesAt } from '@embroider/test-support/file-assertions/qunit'; import { throwOnWarnings } from '@embroider/core'; import merge from 'lodash/merge'; import QUnit from 'qunit'; @@ -94,7 +94,7 @@ scenarios hooks.beforeEach(assert => { expectFile = expectFilesAt(readFileSync(join(app.dir, 'dist/.stage2-output'), 'utf8'), { qunit: assert }); - expectAddonFile = expectRewrittenAddonFilesAt(app.dir, { qunit: assert }); + expectAddonFile = expectRewrittenFilesAt(app.dir, { qunit: assert }); build = new Transpiler(expectFile.basePath); }); @@ -200,7 +200,7 @@ scenarios hooks.beforeEach(assert => { expectFile = expectFilesAt(readFileSync(join(app.dir, 'dist/.stage2-output'), 'utf8'), { qunit: assert }); - expectAddonFile = expectRewrittenAddonFilesAt(app.dir, { qunit: assert }); + expectAddonFile = expectRewrittenFilesAt(app.dir, { qunit: assert }); }); test(`app's colocated components are not implicitly included`, function () { From 7ac453a6705eb4b45b512075329d0651a9e6d4fd Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Sat, 17 Jun 2023 06:19:05 -0400 Subject: [PATCH 46/72] adding a backward-compat fallback --- packages/core/src/module-resolver.ts | 21 +++++++++++++++++++++ tests/scenarios/preprocess-test.ts | 11 +++++------ 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/packages/core/src/module-resolver.ts b/packages/core/src/module-resolver.ts index 79947e0e7..6be62e3b3 100644 --- a/packages/core/src/module-resolver.ts +++ b/packages/core/src/module-resolver.ts @@ -869,6 +869,27 @@ export class Resolver { return logTransition('fallbackResolve: relative appJs search failure', request); } } else { + if (pkg.meta['auto-upgraded'] && dirname(request.fromFile) === pkg.root) { + let otherRoot = this.options.activeAddons[pkg.name]; + if (otherRoot && otherRoot !== pkg.root) { + // This provides some backward-compatibility with the way things + // would have resolved in earlier embroider versions, where the + // final fallback was always the activeAddons. These requests now + // end up getting converted to relative requests inside a particular + // moved package, which is why we need to handle them here. + // + // TODO: We shouldn't need this if we generate notional per-package + // entrypoints that pull in all the implicit-modules, so that the + // imports for implicit-modules happen in places with normal + // dependency resolvability. + return logTransition( + 'fallbackResolve: relative path falling through to activeAddons', + request, + request.rehome(resolve(otherRoot, 'package.json')) + ); + } + } + // nothing else to do for relative imports return logTransition('fallbackResolve: relative failure', request); } diff --git a/tests/scenarios/preprocess-test.ts b/tests/scenarios/preprocess-test.ts index 8fcadead6..f610a52fe 100644 --- a/tests/scenarios/preprocess-test.ts +++ b/tests/scenarios/preprocess-test.ts @@ -3,8 +3,7 @@ import { PreparedApp } from 'scenario-tester'; import QUnit from 'qunit'; import merge from 'lodash/merge'; import { loadFromFixtureData } from './helpers'; -import { join } from 'path'; -import fs from 'fs'; +import { expectFilesAt } from '@embroider/test-support/file-assertions/qunit'; const { module: Qmodule, test } = QUnit; appScenarios @@ -29,10 +28,10 @@ appScenarios Qmodule(scenario.name, function () { test(`pnpm test`, async function (assert) { let app: PreparedApp = await scenario.prepare(); - await app.execute('node ./node_modules/ember-cli/bin/ember b'); - - const data = fs.readFileSync(join(app.dir, 'dist', 'assets', 'app-template.css'), 'utf8'); - assert.equal(data, 'body { background: red; }'); + let result = await app.execute('node ./node_modules/ember-cli/bin/ember b'); + assert.equal(result.exitCode, 0, result.output); + let expectFile = expectFilesAt(app.dir, { qunit: assert }); + expectFile('./dist/assets/app-template.css').matches('body { background: red; }'); }); }); }); From c48f934cf842290bf69183bfb92d23d198dc0764 Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Sat, 17 Jun 2023 07:23:51 -0400 Subject: [PATCH 47/72] make audit assertions support asserting about nonexist modules --- test-packages/support/audit-assertions.ts | 171 ++++++++++++--------- tests/scenarios/compat-renaming-test.ts | 2 +- tests/scenarios/compat-route-split-test.ts | 71 +++------ tests/scenarios/compat-stage2-test.ts | 6 +- 4 files changed, 131 insertions(+), 119 deletions(-) diff --git a/test-packages/support/audit-assertions.ts b/test-packages/support/audit-assertions.ts index 255797672..162d7448b 100644 --- a/test-packages/support/audit-assertions.ts +++ b/test-packages/support/audit-assertions.ts @@ -11,17 +11,17 @@ import { sortBy } from 'lodash'; take advantage of the audit tool within our test suite to help us analyze Embroider's output. */ -export function setupAuditTest(hooks: NestedHooks, getAppDir: () => string) { +export function setupAuditTest(hooks: NestedHooks, opts: () => AuditBuildOptions) { let result: AuditResults; let expectAudit: ExpectAuditResults; hooks.before(async () => { - result = await Audit.run({ app: getAppDir(), 'reuse-build': false }); + result = await Audit.run(opts()); }); hooks.beforeEach(assert => { installAuditAssertions(assert); - expectAudit = new ExpectAuditResults(result, assert, getAppDir()); + expectAudit = new ExpectAuditResults(result, assert, opts().app); }); return { @@ -55,10 +55,10 @@ declare global { export class ExpectAuditResults { private packageCache = RewrittenPackageCache.shared('embroider', this.appDir); - constructor(readonly result: AuditResults, private assert: Assert, private appDir: string) {} + constructor(readonly result: AuditResults, readonly assert: Assert, private appDir: string) {} // input and output paths are relative to getAppDir() - private toRewritten = (path: string) => { + toRewrittenPath = (path: string) => { let fullPath = resolve(this.appDir, path); let owner = this.packageCache.ownerOfFile(fullPath); if (!owner) { @@ -72,21 +72,8 @@ export class ExpectAuditResults { return explicitRelative(this.appDir, movedFullPath); }; - module(inputName: string) { - let outputName = this.toRewritten(inputName); - let m = this.result.modules[outputName]; - const showNearMisses = 4; - if (!m) { - let actuals = sortBy(Object.keys(this.result.modules), candidate => distance(candidate, outputName)); - this.assert.pushResult({ - result: false, - actual: - actuals.length > showNearMisses ? actuals.slice(0, showNearMisses).join(', ') + '...' : actuals.join(', '), - expected: outputName, - message: `Can't locate module ${inputName}`, - }); - } - return new ExpectModule(this.assert, this.result, m, this.toRewritten); + module(inputName: string): PublicAPI { + return new ExpectModule(this, inputName); } get findings() { @@ -103,93 +90,139 @@ export class ExpectAuditResults { } export class ExpectModule { - constructor( - private assert: Assert, - private result: AuditResults, - private module: Module | undefined, - private toRewritten: (s: string) => string - ) {} + constructor(private expectAudit: ExpectAuditResults, private inputName: string) {} + + private get module() { + let outputName = this.expectAudit.toRewrittenPath(this.inputName); + return this.expectAudit.result.modules[outputName]; + } + + private emitMissingModule() { + let outputName = this.expectAudit.toRewrittenPath(this.inputName); + const showNearMisses = 4; + let actuals = sortBy(Object.keys(this.expectAudit.result.modules), candidate => distance(candidate, outputName)); + this.expectAudit.assert.pushResult({ + result: false, + actual: + actuals.length > showNearMisses ? actuals.slice(0, showNearMisses).join(', ') + '...' : actuals.join(', '), + expected: outputName, + message: `Can't locate module ${this.inputName}`, + }); + } + + doesNotExist() { + this.expectAudit.assert.pushResult({ + result: !this.module, + actual: `${this.inputName} exists`, + expected: `${this.inputName} not to exist`, + message: `Expected ${this.inputName} not to exist`, + }); + } codeEquals(expectedSource: string) { - if (this.module) { - this.assert.codeEqual(this.module.content, expectedSource); + if (!this.module) { + this.emitMissingModule(); + return; } + this.expectAudit.assert.codeEqual(this.module.content, expectedSource); } - resolves(specifier: string): ExpectResolution { + resolves(specifier: string): PublicAPI { if (!this.module) { - // the place that instantiated us already pushed the exception that this - // module doesn't exist - return new ExpectResolution(this.assert, this.result, undefined, this.toRewritten); + this.emitMissingModule(); + return new EmptyExpectResolution(); } if (!(specifier in this.module.resolutions)) { - this.assert.pushResult({ + this.expectAudit.assert.pushResult({ result: false, expected: `${this.module.appRelativePath} does not refer to ${specifier}`, actual: Object.keys(this.module.resolutions), }); - return new ExpectResolution(this.assert, this.result, undefined, this.toRewritten); + return new EmptyExpectResolution(); } let resolution = this.module.resolutions[specifier]; if (!resolution) { - this.assert.pushResult({ + this.expectAudit.assert.pushResult({ result: false, expected: `${specifier} fails to resolve in ${this.module.appRelativePath}`, actual: `${specifier} to resolve to something`, }); - return new ExpectResolution(this.assert, this.result, undefined, this.toRewritten); + return new EmptyExpectResolution(); } - let target = this.result.modules[resolution]; + let target = this.expectAudit.result.modules[resolution]; if (!target) { - this.assert.pushResult({ + this.expectAudit.assert.pushResult({ result: false, expected: `${specifier} resolves to ${resolution} but ${resolution} is not found in audit results`, actual: `${resolution} exists`, }); - return new ExpectResolution(this.assert, this.result, undefined, this.toRewritten); + return new EmptyExpectResolution(); } - return new ExpectResolution(this.assert, this.result, target, this.toRewritten); + return new ExpectResolution(this.expectAudit, target, resolution); } // this is testing explicitly for the template-only component moduels that we // synthesize in our module-resolver isTemplateOnlyComponent(template: string, message?: string) { - if (this.module) { - this.resolves( - explicitRelative( - posix.dirname(posix.resolve('/APP', this.module.appRelativePath)), - posix.resolve('/APP', template) - ) - ).to(template, message); - this.resolves('@ember/component/template-only'); + if (!this.module) { + this.emitMissingModule(); + return; + } + this.resolves( + explicitRelative( + posix.dirname(posix.resolve('/APP', this.module.appRelativePath)), + posix.resolve('/APP', template) + ) + ).to(template, message); + this.resolves('@ember/component/template-only'); + } + + hasConsumers(paths: string[]) { + if (!this.module) { + this.emitMissingModule(); + return; } + this.expectAudit.assert.deepEqual(this.module.consumedFrom, paths.map(this.expectAudit.toRewrittenPath)); } } export class ExpectResolution { - constructor( - private assert: Assert, - private result: AuditResults, - private module: Module | undefined, - private toRewritten: (s: string) => string - ) {} - - to(inputName: string | null, message?: string) { - let outputName: string | null = null; - if (inputName) { - outputName = this.toRewritten(inputName); - } - if (this.module) { - this.assert.pushResult({ - result: this.module.appRelativePath === outputName, - expected: inputName, - actual: this.module.appRelativePath, - message, - }); + constructor(private expectAudit: ExpectAuditResults, private module: Module, private moduleInputName: string) {} + + to(targetInputName: string | null, message?: string) { + let targetOutputName: string | null = null; + if (targetInputName) { + targetOutputName = this.expectAudit.toRewrittenPath(targetInputName); } + + this.expectAudit.assert.pushResult({ + result: this.module.appRelativePath === targetOutputName, + expected: targetInputName, + actual: this.module.appRelativePath, + message, + }); } - toModule(): ExpectModule { - return new ExpectModule(this.assert, this.result, this.module, this.toRewritten); + toModule(): PublicAPI { + return new ExpectModule(this.expectAudit, this.moduleInputName); + } +} + +type PublicAPI = { [K in keyof T]: T[K] }; + +class EmptyExpectModule implements PublicAPI { + doesNotExist() {} + codeEquals() {} + resolves(): PublicAPI { + return new EmptyExpectResolution() as PublicAPI; + } + isTemplateOnlyComponent() {} + hasConsumers() {} +} + +class EmptyExpectResolution implements PublicAPI { + to() {} + toModule() { + return new EmptyExpectModule() as PublicAPI; } } diff --git a/tests/scenarios/compat-renaming-test.ts b/tests/scenarios/compat-renaming-test.ts index 02c52e0f4..d4191a428 100644 --- a/tests/scenarios/compat-renaming-test.ts +++ b/tests/scenarios/compat-renaming-test.ts @@ -177,7 +177,7 @@ appScenarios build = new Transpiler(expectFile.basePath); }); - let expectAudit = setupAuditTest(hooks, () => app.dir); + let expectAudit = setupAuditTest(hooks, () => ({ app: app.dir })); test('audit issues', function () { expectAudit.hasNoFindings(); diff --git a/tests/scenarios/compat-route-split-test.ts b/tests/scenarios/compat-route-split-test.ts index 4af286921..bc0206df1 100644 --- a/tests/scenarios/compat-route-split-test.ts +++ b/tests/scenarios/compat-route-split-test.ts @@ -2,13 +2,12 @@ import { PreparedApp } from 'scenario-tester'; import { appScenarios, renameApp } from './scenarios'; import { readFileSync } from 'fs'; import { join } from 'path'; -import QUnit from 'qunit'; -const { module: Qmodule, test } = QUnit; - import { ExpectFile, expectFilesAt } from '@embroider/test-support/file-assertions/qunit'; import { throwOnWarnings } from '@embroider/core'; import merge from 'lodash/merge'; -import { Audit, AuditResults } from '@embroider/compat/src/audit'; +import { setupAuditTest } from '@embroider/test-support/audit-assertions'; +import QUnit from 'qunit'; +const { module: Qmodule, test } = QUnit; let splitScenarios = appScenarios.map('compat-splitAtRoutes', app => { renameApp(app, 'my-app'); @@ -173,36 +172,26 @@ splitScenarios }); Qmodule('audit', function (hooks) { - let auditResults: AuditResults; - hooks.before(async function () { - let audit = new Audit(app.dir); - auditResults = await audit.run(); - }); + let expectAudit = setupAuditTest(hooks, () => ({ app: app.dir, 'reuse-build': true })); - test('has no issues', function (assert) { - assert.deepEqual(auditResults.findings, []); + test('has no issues', function () { + expectAudit.hasNoFindings(); }); - test('helper is consumed only from the template that uses it', function (assert) { - assert.deepEqual(auditResults.modules['./helpers/capitalize.js']?.consumedFrom, [ - './components/one-person.hbs', - ]); + test('helper is consumed only from the template that uses it', function () { + expectAudit.module('./helpers/capitalize.js').hasConsumers(['./components/one-person.hbs']); }); - test('component is consumed only from the template that uses it', function (assert) { - assert.deepEqual(auditResults.modules['./components/one-person.js']?.consumedFrom, [ - './templates/people/show.hbs', - ]); + test('component is consumed only from the template that uses it', function () { + expectAudit.module('./components/one-person.js').hasConsumers(['./templates/people/show.hbs']); }); - test('modifier is consumed only from the template that uses it', function (assert) { - assert.deepEqual(auditResults.modules['./modifiers/auto-focus.js']?.consumedFrom, [ - './templates/people/edit.hbs', - ]); + test('modifier is consumed only from the template that uses it', function () { + expectAudit.module('./modifiers/auto-focus.js').hasConsumers(['./templates/people/edit.hbs']); }); - test('does not include unused component', function (assert) { - assert.strictEqual(auditResults.modules['./components/unused.hbs'], undefined); + test('does not include unused component', function () { + expectAudit.module('./components/unused.hbs').doesNotExist(); }); }); }); @@ -355,36 +344,26 @@ splitScenarios }); Qmodule('audit', function (hooks) { - let auditResults: AuditResults; - hooks.before(async function () { - let audit = new Audit(app.dir); - auditResults = await audit.run(); - }); + let expectAudit = setupAuditTest(hooks, () => ({ app: app.dir, 'reuse-build': true })); - test('has no issues', function (assert) { - assert.deepEqual(auditResults.findings, []); + test('has no issues', function () { + expectAudit.hasNoFindings(); }); - test('helper is consumed only from the template that uses it', function (assert) { - assert.deepEqual(auditResults.modules['./helpers/capitalize.js']?.consumedFrom, [ - './components/one-person.hbs', - ]); + test('helper is consumed only from the template that uses it', function () { + expectAudit.module('./helpers/capitalize.js').hasConsumers(['./components/one-person.hbs']); }); - test('component is consumed only from the template that uses it', function (assert) { - assert.deepEqual(auditResults.modules['./components/one-person.js']?.consumedFrom, [ - './pods/people/show/template.hbs', - ]); + test('component is consumed only from the template that uses it', function () { + expectAudit.module('./components/one-person.js').hasConsumers(['./pods/people/show/template.hbs']); }); - test('modifier is consumed only from the template that uses it', function (assert) { - assert.deepEqual(auditResults.modules['./modifiers/auto-focus.js']?.consumedFrom, [ - './pods/people/edit/template.hbs', - ]); + test('modifier is consumed only from the template that uses it', function () { + expectAudit.module('./modifiers/auto-focus.js').hasConsumers(['./pods/people/edit/template.hbs']); }); - test('does not include unused component', function (assert) { - assert.strictEqual(auditResults.modules['./components/unused.hbs'], undefined); + test('does not include unused component', function () { + expectAudit.module('./components/unused.hbs').doesNotExist(); }); }); }); diff --git a/tests/scenarios/compat-stage2-test.ts b/tests/scenarios/compat-stage2-test.ts index 1edcf27bc..334d2e225 100644 --- a/tests/scenarios/compat-stage2-test.ts +++ b/tests/scenarios/compat-stage2-test.ts @@ -94,7 +94,7 @@ stage2Scenarios expectAddonFile = expectRewrittenFilesAt(app.dir, { qunit: assert }); }); - let expectAudit = setupAuditTest(hooks, () => app.dir); + let expectAudit = setupAuditTest(hooks, () => ({ app: app.dir, 'reuse-build': true })); test('in repo addons are symlinked correctly', function () { // check that package json contains in repo dep @@ -182,7 +182,7 @@ stage2Scenarios assert.equal(result.exitCode, 0, result.output); }); - let expectAudit = setupAuditTest(hooks, () => app.dir); + let expectAudit = setupAuditTest(hooks, () => ({ app: app.dir, 'reuse-build': true })); test('verifies that the correct lexigraphically sorted addons win', function () { let expectModule = expectAudit.module('./assets/my-app.js'); @@ -477,7 +477,7 @@ stage2Scenarios build = new Transpiler(expectFile.basePath); }); - let expectAudit = setupAuditTest(hooks, () => app.dir); + let expectAudit = setupAuditTest(hooks, () => ({ app: app.dir, 'reuse-build': true })); test('no audit issues', function () { // among other things, this is asserting that dynamicComponent in From 1bc378e277d876282e6204a9c5821e9a0390743d Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Sat, 17 Jun 2023 08:22:38 -0400 Subject: [PATCH 48/72] changing expectRewrittenFiles so it can handle more complex input paths The problem with the previous attempt is that it assumes all deps live directly under node_modules in the input. But we have plenty of tests that do more complicated things than that, like using nested npm dependencies or in-repo addons. This changes the API so that we always talk about locations in the input scenarios and then map them reliably through RewrittenPackageCache to assert about their output locations. --- .../support/file-assertions/qunit.ts | 43 +++++++++++-------- test-packages/support/index.ts | 2 +- tests/scenarios/compat-addon-styles-test.ts | 26 +++++++---- tests/scenarios/compat-app-dot-import-test.ts | 8 ++-- .../compat-exclude-dot-files-test.ts | 26 +++++------ tests/scenarios/compat-preprocessors-test.ts | 27 ++++++------ tests/scenarios/compat-renaming-test.ts | 28 ++++++------ .../compat-template-colocation-test.ts | 20 ++++----- 8 files changed, 91 insertions(+), 89 deletions(-) diff --git a/test-packages/support/file-assertions/qunit.ts b/test-packages/support/file-assertions/qunit.ts index 1d57b5bac..27acf0ccb 100644 --- a/test-packages/support/file-assertions/qunit.ts +++ b/test-packages/support/file-assertions/qunit.ts @@ -1,7 +1,7 @@ import { install } from 'code-equality-assertions/qunit'; import { AssertionAdapter, BoundExpectFile, ExpectFile } from '../file-assertions'; -import { packageName } from '../../../packages/shared-internals'; -import crypto from 'crypto'; +import { explicitRelative, RewrittenPackageCache } from '../../../packages/shared-internals'; +import { resolve } from 'path'; class QUnitAdapter implements AssertionAdapter { constructor(private qassert: Assert) { @@ -41,33 +41,38 @@ export function expectFilesAt(basePath: string, params: { qunit: Assert }): Expe return func; } -function getRewrittenLocation(appDir: string, addonPath: string) { - let name = packageName(addonPath); - if (!name) { - throw new Error('getRewrittenLocation only accepts fully-qualified paths'); +function getRewrittenLocation(appDir: string, inputPath: string) { + let packageCache = RewrittenPackageCache.shared('embroider', appDir); + let fullInputPath = resolve(appDir, inputPath); + let owner = packageCache.ownerOfFile(fullInputPath); + if (!owner) { + return inputPath; } - - const syntheticPackages = ['@embroider/synthesized-styles', '@embroider/synthesized-vendor']; - - if (syntheticPackages.includes(name)) { - return `node_modules/.embroider/rewritten-packages/${name}/${addonPath.slice(name.length)}`; + let movedOwner = packageCache.maybeMoved(owner); + if (movedOwner === owner) { + return inputPath; } - - let h = crypto.createHash('sha1'); - let hash = h.update(`${appDir}/node_modules/${name}`).digest('hex').slice(0, 8); - - return `node_modules/.embroider/rewritten-packages/${name}.${hash}/${addonPath.slice(name.length)}`; + let movedFullPath = fullInputPath.replace(owner.root, movedOwner.root); + return explicitRelative(appDir, movedFullPath); } -export function expectRewrittenFilesAt(basePath: string, params: { qunit: Assert }): ExpectFile { - let func: any = (addonPath: string) => { - return new BoundExpectFile(basePath, getRewrittenLocation(basePath, addonPath), new QUnitAdapter(params.qunit)); +export function expectRewrittenFilesAt( + basePath: string, + params: { qunit: Assert } +): ExpectFile & { toRewrittenPath: (s: string) => string } { + let func: any = (inputPath: string) => { + return new BoundExpectFile(basePath, getRewrittenLocation(basePath, inputPath), new QUnitAdapter(params.qunit)); }; Object.defineProperty(func, 'basePath', { get() { return basePath; }, }); + Object.defineProperty(func, 'toRewrittenPath', { + get() { + return (p: string) => getRewrittenLocation(basePath, p); + }, + }); return func; } diff --git a/test-packages/support/index.ts b/test-packages/support/index.ts index af1a1f950..79f88e44e 100644 --- a/test-packages/support/index.ts +++ b/test-packages/support/index.ts @@ -106,7 +106,7 @@ export function definesPattern(runtimeName: string, buildTimeName: string): RegE runtimeName = escapeRegExp(runtimeName); buildTimeName = escapeRegExp(buildTimeName); return new RegExp( - `d\\(['"]${runtimeName}['"], *function *\\(\\) *\\{[\\s\\n]*return esc\\(require\\(['"]${buildTimeName}['"]\\)\\);?[\\s\\n]*\\}\\)` + `d\\(['"]${runtimeName}['"], *function *\\(\\) *\\{[\\s\\n]*return i\\(['"]${buildTimeName}['"]\\);?[\\s\\n]*\\}\\)` ); } diff --git a/tests/scenarios/compat-addon-styles-test.ts b/tests/scenarios/compat-addon-styles-test.ts index 3c7b3a855..9d8493da1 100644 --- a/tests/scenarios/compat-addon-styles-test.ts +++ b/tests/scenarios/compat-addon-styles-test.ts @@ -1,4 +1,4 @@ -import { expectRewrittenFilesAt, ExpectFile } from '@embroider/test-support/file-assertions/qunit'; +import { expectFilesAt, expectRewrittenFilesAt, ExpectFile } from '@embroider/test-support/file-assertions/qunit'; import { PreparedApp } from 'scenario-tester'; import { throwOnWarnings } from '@embroider/core'; import { appScenarios, baseAddon } from './scenarios'; @@ -91,7 +91,8 @@ appScenarios let app: PreparedApp; - let expectAddonFile: ExpectFile; + let expectFile: ExpectFile; + let expectRewrittenFile: ExpectFile; hooks.before(async assert => { app = await scenario.prepare(); @@ -100,26 +101,33 @@ appScenarios }); hooks.beforeEach(assert => { - expectAddonFile = expectRewrittenFilesAt(app.dir, { qunit: assert }); + expectFile = expectFilesAt(app.dir, { qunit: assert }); + expectRewrittenFile = expectRewrittenFilesAt(app.dir, { qunit: assert }); }); test('treeForStyles adds styles to build', function () { - expectAddonFile('@embroider/synthesized-styles/assets/third-party1.css').matches('.error { color: red; }'); + expectFile( + './node_modules/.embroider/rewritten-packages/@embroider/synthesized-styles/assets/third-party1.css' + ).matches('.error { color: red; }'); }); // prevent regression of https://github.com/embroider-build/embroider/issues/164 test('treeForStyles not calling super adds styles to build', function () { - expectAddonFile('@embroider/synthesized-styles/assets/third-party2.css').matches('.success { color: green }'); + expectFile( + './node_modules/.embroider/rewritten-packages/@embroider/synthesized-styles/assets/third-party2.css' + ).matches('.success { color: green }'); }); test(`all addon CSS gets convert to implicit-styles`, function () { - let implicitStyles = expectAddonFile('my-addon3/package.json').json().get('ember-addon.implicit-styles'); + let implicitStyles = expectRewrittenFile('./node_modules/my-addon3/package.json') + .json() + .get('ember-addon.implicit-styles'); implicitStyles.includes('./my-addon3.css'); implicitStyles.includes('./outer.css'); implicitStyles.includes('./nested/inner.css'); - expectAddonFile('my-addon3/my-addon3.css').matches(`from-addon`); - expectAddonFile('my-addon3/outer.css').matches(`from-outer`); - expectAddonFile('my-addon3/nested/inner.css').matches(`from-inner`); + expectRewrittenFile('./node_modules/my-addon3/my-addon3.css').matches(`from-addon`); + expectRewrittenFile('./node_modules/my-addon3/outer.css').matches(`from-outer`); + expectRewrittenFile('./node_modules/my-addon3/nested/inner.css').matches(`from-inner`); }); }); }); diff --git a/tests/scenarios/compat-app-dot-import-test.ts b/tests/scenarios/compat-app-dot-import-test.ts index 8296f7993..34a9d432e 100644 --- a/tests/scenarios/compat-app-dot-import-test.ts +++ b/tests/scenarios/compat-app-dot-import-test.ts @@ -50,15 +50,17 @@ appScenarios }); }); test('destDir puts vendor files into public assets', function () { - expectFile('@embroider/synthesized-vendor/package.json') + expectFile('./node_modules/.embroider/rewritten-packages/@embroider/synthesized-vendor/package.json') .json() .get(['ember-addon', 'public-assets', './vendor/some-font.ttf']) .equals('fonts/some-font.ttf'); - expectFile('@embroider/synthesized-vendor/vendor/some-font.ttf').exists(); + expectFile( + './node_modules/.embroider/rewritten-packages/@embroider/synthesized-vendor/vendor/some-font.ttf' + ).exists(); }); test('handle non-transformed node_module with explicit outputFile', function () { - expectFile('@embroider/synthesized-vendor/package.json') + expectFile('./node_modules/.embroider/rewritten-packages/@embroider/synthesized-vendor/package.json') .json() .get([ 'ember-addon', diff --git a/tests/scenarios/compat-exclude-dot-files-test.ts b/tests/scenarios/compat-exclude-dot-files-test.ts index 79fde5198..a7b211a8e 100644 --- a/tests/scenarios/compat-exclude-dot-files-test.ts +++ b/tests/scenarios/compat-exclude-dot-files-test.ts @@ -1,9 +1,7 @@ -import { ExpectFile, expectFilesAt, expectRewrittenFilesAt } from '@embroider/test-support/file-assertions/qunit'; +import { ExpectFile, expectRewrittenFilesAt } from '@embroider/test-support/file-assertions/qunit'; import { throwOnWarnings } from '@embroider/core'; import { PreparedApp } from 'scenario-tester'; import { appScenarios, baseAddon } from './scenarios'; -import { readFileSync } from 'fs'; -import { join } from 'path'; import QUnit from 'qunit'; import { merge } from 'lodash'; const { module: Qmodule, test } = QUnit; @@ -35,7 +33,6 @@ appScenarios let app: PreparedApp; let expectFile: ExpectFile; - let expectAddonFile: ExpectFile; hooks.before(async assert => { app = await scenario.prepare(); @@ -44,28 +41,27 @@ appScenarios }); hooks.beforeEach(assert => { - expectFile = expectFilesAt(readFileSync(join(app.dir, 'dist/.stage2-output'), 'utf8'), { qunit: assert }); - expectAddonFile = expectRewrittenFilesAt(app.dir, { qunit: assert }); + expectFile = expectRewrittenFilesAt(app.dir, { qunit: assert }); }); test('dot files are not included as app modules', function () { // dot files should exist on disk - expectFile('.foobar.js').exists(); - expectFile('.barbaz.js').exists(); - expectFile('bizbiz.js').exists(); + expectFile('./.foobar.js').exists(); + expectFile('./.barbaz.js').exists(); + expectFile('./bizbiz.js').exists(); // dot files should not be included as modules - expectFile('assets/app-template.js').doesNotMatch('app-template/.foobar'); - expectFile('assets/app-template.js').doesNotMatch('app-template/.barbaz'); - expectFile('assets/app-template.js').matches('app-template/bizbiz'); + expectFile('./assets/app-template.js').doesNotMatch('app-template/.foobar'); + expectFile('./assets/app-template.js').doesNotMatch('app-template/.barbaz'); + expectFile('./assets/app-template.js').matches('app-template/bizbiz'); }); test('dot files are not included as addon implicit-modules', function () { // Dot files should exist on disk - expectAddonFile('my-addon/.fooaddon.js').exists(); - expectAddonFile('my-addon/baraddon.js').exists(); + expectFile('./node_modules/my-addon/.fooaddon.js').exists(); + expectFile('./node_modules/my-addon/baraddon.js').exists(); - let myAddonPackage = expectAddonFile('my-addon/package.json').json(); + let myAddonPackage = expectFile('./node_modules/my-addon/package.json').json(); // dot files are not included as implicit-modules myAddonPackage.get(['ember-addon', 'implicit-modules']).deepEquals(['./baraddon']); diff --git a/tests/scenarios/compat-preprocessors-test.ts b/tests/scenarios/compat-preprocessors-test.ts index 4489ce002..d46156fea 100644 --- a/tests/scenarios/compat-preprocessors-test.ts +++ b/tests/scenarios/compat-preprocessors-test.ts @@ -1,8 +1,6 @@ import { PreparedApp } from 'scenario-tester'; import { appScenarios, baseAddon } from './scenarios'; -import { readFileSync } from 'fs'; -import { join } from 'path'; -import { ExpectFile, expectFilesAt, expectRewrittenFilesAt } from '@embroider/test-support/file-assertions/qunit'; +import { ExpectFile, expectRewrittenFilesAt } from '@embroider/test-support/file-assertions/qunit'; import { throwOnWarnings } from '@embroider/core'; import merge from 'lodash/merge'; import QUnit from 'qunit'; @@ -92,7 +90,6 @@ appScenarios let app: PreparedApp; let expectFile: ExpectFile; - let expectAddonFile: ExpectFile; hooks.before(async assert => { app = await scenario.prepare(); @@ -101,28 +98,30 @@ appScenarios }); hooks.beforeEach(assert => { - expectFile = expectFilesAt(readFileSync(join(app.dir, 'dist/.stage2-output'), 'utf8'), { qunit: assert }); - expectAddonFile = expectRewrittenFilesAt(app.dir, { qunit: assert }); + expectFile = expectRewrittenFilesAt(app.dir, { qunit: assert }); }); test('dependencies are setup for this test suite correctly', () => { - expectFile('package.json').exists(); - expectFile('package.json').matches(/my-preprocessor/, 'has the preprocessor dependency'); - expectAddonFile('my-addon/package.json').exists(); - expectAddonFile('my-addon/package.json').matches(/my-preprocessor/, 'has the preprocessor dependency'); - expectAddonFile('my-preprocessor/package.json').exists(); + expectFile('./package.json').exists(); + expectFile('./package.json').matches(/my-preprocessor/, 'has the preprocessor dependency'); + expectFile('./node_modules/my-addon/package.json').exists(); + expectFile('./node_modules/my-addon/package.json').matches( + /my-preprocessor/, + 'has the preprocessor dependency' + ); + expectFile('./node_modules/my-preprocessor/package.json').exists(); }); test('app has correct path embedded in comment', () => { - const assertFile = expectFile('components/from-the-app.js'); + const assertFile = expectFile('./components/from-the-app.js'); assertFile.exists(); // This is the expected output during an classic build. assertFile.matches(/path@app-template\/components\/from-the-app\.js/, 'has a path comment in app components'); }); test('addon has correct path embedded in comment', () => { - expectAddonFile('my-preprocessor/package.json').exists(); - const assertFile = expectAddonFile('my-addon/components/greeting.js'); + expectFile('./node_modules/my-preprocessor/package.json').exists(); + const assertFile = expectFile('./node_modules/my-addon/components/greeting.js'); assertFile.matches(/path@my-addon\/components\/greeting\.js/, 'has a path comment in app components'); }); }); diff --git a/tests/scenarios/compat-renaming-test.ts b/tests/scenarios/compat-renaming-test.ts index d4191a428..455b2637d 100644 --- a/tests/scenarios/compat-renaming-test.ts +++ b/tests/scenarios/compat-renaming-test.ts @@ -1,12 +1,10 @@ import { PreparedApp } from 'scenario-tester'; import { appScenarios, baseAddon } from './scenarios'; -import { readFileSync } from 'fs'; -import { join } from 'path'; import QUnit from 'qunit'; const { module: Qmodule, test } = QUnit; -import { definesPattern, Transpiler } from '@embroider/test-support'; -import { ExpectFile, expectFilesAt, expectRewrittenFilesAt } from '@embroider/test-support/file-assertions/qunit'; +import { definesPattern } from '@embroider/test-support'; +import { ExpectFile, expectRewrittenFilesAt } from '@embroider/test-support/file-assertions/qunit'; import { throwOnWarnings } from '@embroider/core'; import merge from 'lodash/merge'; @@ -164,17 +162,13 @@ appScenarios let app: PreparedApp; let expectFile: ExpectFile; - let expectAddonFile: ExpectFile; - let build: Transpiler; hooks.before(async () => { app = await scenario.prepare(); }); hooks.beforeEach(assert => { - expectFile = expectFilesAt(readFileSync(join(app.dir, 'dist/.stage2-output'), 'utf8'), { qunit: assert }); - expectAddonFile = expectRewrittenFilesAt(app.dir, { qunit: assert }); - build = new Transpiler(expectFile.basePath); + expectFile = expectRewrittenFilesAt(app.dir, { qunit: assert }); }); let expectAudit = setupAuditTest(hooks, () => ({ app: app.dir })); @@ -188,7 +182,7 @@ appScenarios .module('./components/import-lodash.js') .resolves('lodash') .to('./node_modules/ember-lodash/index.js'); - expectAddonFile('ember-lodash/index.js').matches(/lodash index/); + expectFile('./node_modules/ember-lodash/index.js').matches(/lodash index/); }); test('whole package renaming works for interior module', function () { @@ -197,7 +191,7 @@ appScenarios .resolves('lodash/capitalize') .to('./node_modules/ember-lodash/capitalize.js'); - expectAddonFile('ember-lodash/capitalize.js').matches(/lodash capitalize/); + expectFile('./node_modules/ember-lodash/capitalize.js').matches(/lodash capitalize/); }); test("modules in own namespace don't get renamed", function () { @@ -205,7 +199,7 @@ appScenarios .module('./components/import-own-thing.js') .resolves('emits-multiple-packages/own-thing') .to('./node_modules/emits-multiple-packages/own-thing.js'); - expectAddonFile('emits-multiple-packages/own-thing.js').matches(/own thing/); + expectFile('./node_modules/emits-multiple-packages/own-thing.js').matches(/own thing/); }); test('modules outside our namespace do get renamed', function () { @@ -213,7 +207,7 @@ appScenarios .module('./components/import-somebody-elses.js') .resolves('somebody-elses-package/environment') .to('./node_modules/emits-multiple-packages/somebody-elses-package/environment.js'); - expectAddonFile('emits-multiple-packages/somebody-elses-package/environment.js').matches( + expectFile('./node_modules/emits-multiple-packages/somebody-elses-package/environment.js').matches( /somebody elses environment/ ); }); @@ -223,7 +217,7 @@ appScenarios .module('./components/import-somebody-elses-utils.js') .resolves('somebody-elses-package/utils') .to('./node_modules/emits-multiple-packages/somebody-elses-package/utils/index.js'); - expectAddonFile('emits-multiple-packages/somebody-elses-package/utils/index.js').matches( + expectFile('./node_modules/emits-multiple-packages/somebody-elses-package/utils/index.js').matches( /somebody elses utils/ ); }); @@ -241,7 +235,7 @@ appScenarios .to('./node_modules/emits-multiple-packages/somebody-elses-package/utils/index.js'); }); test('renamed modules keep their classic runtime name when used as implicit-modules', function () { - let assertFile = expectFile('assets/app-template.js').transform(build.transpile); + let assertFile = expectFile('assets/app-template.js'); assertFile.matches( definesPattern( 'somebody-elses-package/environment', @@ -265,7 +259,9 @@ appScenarios .module('./components/import-single-file-package.js') .resolves('single-file-package') .to('./node_modules/emits-multiple-packages/single-file-package/index.js'); - expectAddonFile('emits-multiple-packages/single-file-package/index.js').matches(/single file package/); + expectFile('./node_modules/emits-multiple-packages/single-file-package/index.js').matches( + /single file package/ + ); }); test('files logically copied into app from addons resolve their own original packages', function () { expectAudit diff --git a/tests/scenarios/compat-template-colocation-test.ts b/tests/scenarios/compat-template-colocation-test.ts index 98161b418..5a50f9a72 100644 --- a/tests/scenarios/compat-template-colocation-test.ts +++ b/tests/scenarios/compat-template-colocation-test.ts @@ -1,7 +1,7 @@ import { PreparedApp } from 'scenario-tester'; import { appScenarios, baseAddon, renameApp } from './scenarios'; import { readFileSync } from 'fs'; -import { join } from 'path'; +import { join, resolve } from 'path'; import { Transpiler } from '@embroider/test-support'; import { ExpectFile, expectFilesAt, expectRewrittenFilesAt } from '@embroider/test-support/file-assertions/qunit'; import { throwOnWarnings } from '@embroider/core'; @@ -83,7 +83,6 @@ scenarios let app: PreparedApp; let expectFile: ExpectFile; - let expectAddonFile: ExpectFile; let build: Transpiler; hooks.before(async assert => { @@ -93,9 +92,8 @@ scenarios }); hooks.beforeEach(assert => { - expectFile = expectFilesAt(readFileSync(join(app.dir, 'dist/.stage2-output'), 'utf8'), { qunit: assert }); - expectAddonFile = expectRewrittenFilesAt(app.dir, { qunit: assert }); - build = new Transpiler(expectFile.basePath); + let r = (expectFile = expectRewrittenFilesAt(app.dir, { qunit: assert })); + build = new Transpiler(resolve(app.dir, r.toRewrittenPath(app.dir))); }); test(`app's colocated template is associated with JS`, function () { @@ -137,7 +135,7 @@ scenarios }); test(`addon's colocated template is associated with JS`, function () { - let assertFile = expectAddonFile('my-addon/components/component-one.js').transform(build.transpile); + let assertFile = expectFile('./node_modules/my-addon/components/component-one.js').transform(build.transpile); assertFile.matches(/import TEMPLATE from ['"]\.\/component-one.hbs['"];/, 'imported template'); assertFile.matches(/import \{ setComponentTemplate \}/, 'found setComponentTemplate'); assertFile.matches( @@ -147,7 +145,7 @@ scenarios }); test(`addon's template-only component JS is synthesized`, function () { - let assertFile = expectAddonFile('my-addon/components/component-two.js').transform(build.transpile); + let assertFile = expectFile('./node_modules/my-addon/components/component-two.js').transform(build.transpile); assertFile.matches(/import TEMPLATE from ['"]\.\/component-two.hbs['"];/, 'imported template'); assertFile.matches(/import \{ setComponentTemplate \}/, 'found setComponentTemplate'); assertFile.matches(/import templateOnlyComponent/, 'found templateOnlyComponent'); @@ -158,7 +156,7 @@ scenarios }); test(`addon's colocated components are correct in implicit-modules`, function () { - let assertFile = expectAddonFile('my-addon/package.json').json(); + let assertFile = expectFile('./node_modules/my-addon/package.json').json(); assertFile.get(['ember-addon', 'implicit-modules']).includes('./components/component-one'); assertFile.get(['ember-addon', 'implicit-modules']).includes('./components/component-two'); assertFile.get(['ember-addon', 'implicit-modules']).doesNotInclude('./components/component-one.hbs'); @@ -190,7 +188,6 @@ scenarios let app: PreparedApp; let expectFile: ExpectFile; - let expectAddonFile: ExpectFile; hooks.before(async assert => { app = await scenario.prepare(); @@ -199,8 +196,7 @@ scenarios }); hooks.beforeEach(assert => { - expectFile = expectFilesAt(readFileSync(join(app.dir, 'dist/.stage2-output'), 'utf8'), { qunit: assert }); - expectAddonFile = expectRewrittenFilesAt(app.dir, { qunit: assert }); + expectFile = expectRewrittenFilesAt(app.dir, { qunit: assert }); }); test(`app's colocated components are not implicitly included`, function () { @@ -214,7 +210,7 @@ scenarios }); test(`addon's colocated components are not in implicit-modules`, function () { - let assertFile = expectAddonFile('my-addon/package.json').json(); + let assertFile = expectFile('./node_modules/my-addon/package.json').json(); assertFile.get(['ember-addon', 'implicit-modules']).equals(undefined); }); }); From c5b1ae9c908a6b084bd6a3ff7cc90f1e33253af0 Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Sat, 17 Jun 2023 14:48:19 -0400 Subject: [PATCH 49/72] extraResolutions for the app --- packages/compat/src/standalone-addon-build.ts | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/packages/compat/src/standalone-addon-build.ts b/packages/compat/src/standalone-addon-build.ts index dae96ac7e..8f043275a 100644 --- a/packages/compat/src/standalone-addon-build.ts +++ b/packages/compat/src/standalone-addon-build.ts @@ -12,8 +12,9 @@ export function convertLegacyAddons(compatApp: CompatApp) { let packageCache = PackageCache.shared('embroider', compatApp.root); let instanceCache = new V1InstanceCache(compatApp, packageCache); - let v1Addons = findV1Addons(packageCache.get(compatApp.root)); - let index = buildAddonIndex(compatApp, v1Addons); + let appPackage = packageCache.get(compatApp.root); + let v1Addons = findV1Addons(appPackage); + let index = buildAddonIndex(compatApp, appPackage, v1Addons); let interiorTrees: Node[] = []; let exteriorTrees = [...v1Addons].map(pkg => { @@ -34,7 +35,7 @@ export function convertLegacyAddons(compatApp: CompatApp) { ]); } -function buildAddonIndex(compatApp: CompatApp, packages: Set): RewrittenPackageIndex { +function buildAddonIndex(compatApp: CompatApp, appPackage: Package, packages: Set): RewrittenPackageIndex { let content: RewrittenPackageIndex = { packages: {}, extraResolutions: {}, @@ -53,6 +54,19 @@ function buildAddonIndex(compatApp: CompatApp, packages: Set): Rewritte // yet. content.packages[compatApp.root] = compatApp.name; + let nonResolvableDeps = appPackage.nonResolvableDeps; + if (nonResolvableDeps) { + let extraRoots = [...nonResolvableDeps.values()].map(v => v.root); + + // the app gets extraResolutions support just like every addon does + content.extraResolutions[compatApp.name] = extraRoots; + + // but it also gets extraResolutions registered against its *original* + // location, because the app is unique because stage2 needs a Package + // representing the *unmoved* app but seeing *moved* deps. + content.extraResolutions[appPackage.root] = extraRoots; + } + return content; } From eed27a1f56e8df6ff61578d70bc14af9e465b496 Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Sat, 17 Jun 2023 14:48:57 -0400 Subject: [PATCH 50/72] updating test Transpiling infra and making more tests pass --- test-packages/support/audit-assertions.ts | 21 ++----- .../support/file-assertions/qunit.ts | 18 +----- test-packages/support/rewritten-path.ts | 17 ++++++ test-packages/support/transpiler.ts | 18 +++--- tests/scenarios/compat-stage2-test.ts | 61 +++++++++++-------- .../compat-template-colocation-test.ts | 6 +- 6 files changed, 72 insertions(+), 69 deletions(-) create mode 100644 test-packages/support/rewritten-path.ts diff --git a/test-packages/support/audit-assertions.ts b/test-packages/support/audit-assertions.ts index 162d7448b..cbf5c1a5d 100644 --- a/test-packages/support/audit-assertions.ts +++ b/test-packages/support/audit-assertions.ts @@ -1,9 +1,10 @@ import { Audit, AuditBuildOptions, AuditResults, Module } from '../../packages/compat/src/audit'; -import { explicitRelative, RewrittenPackageCache } from '../../packages/shared-internals'; +import { explicitRelative } from '../../packages/shared-internals'; import { install as installCodeEqualityAssertions } from 'code-equality-assertions/qunit'; -import { posix, resolve } from 'path'; +import { posix } from 'path'; import { distance } from 'fastest-levenshtein'; import { sortBy } from 'lodash'; +import { getRewrittenLocation } from './rewritten-path'; /* The audit tool in @embroider/compat can be used directly to tell you about @@ -53,23 +54,11 @@ declare global { } export class ExpectAuditResults { - private packageCache = RewrittenPackageCache.shared('embroider', this.appDir); - constructor(readonly result: AuditResults, readonly assert: Assert, private appDir: string) {} // input and output paths are relative to getAppDir() toRewrittenPath = (path: string) => { - let fullPath = resolve(this.appDir, path); - let owner = this.packageCache.ownerOfFile(fullPath); - if (!owner) { - return path; - } - let movedOwner = this.packageCache.maybeMoved(owner); - if (movedOwner === owner) { - return path; - } - let movedFullPath = fullPath.replace(owner.root, movedOwner.root); - return explicitRelative(this.appDir, movedFullPath); + return getRewrittenLocation(this.appDir, path); }; module(inputName: string): PublicAPI { @@ -171,7 +160,7 @@ export class ExpectModule { this.resolves( explicitRelative( posix.dirname(posix.resolve('/APP', this.module.appRelativePath)), - posix.resolve('/APP', template) + posix.resolve('/APP', this.expectAudit.toRewrittenPath(template)) ) ).to(template, message); this.resolves('@ember/component/template-only'); diff --git a/test-packages/support/file-assertions/qunit.ts b/test-packages/support/file-assertions/qunit.ts index 27acf0ccb..2d55e9aa3 100644 --- a/test-packages/support/file-assertions/qunit.ts +++ b/test-packages/support/file-assertions/qunit.ts @@ -1,7 +1,6 @@ import { install } from 'code-equality-assertions/qunit'; import { AssertionAdapter, BoundExpectFile, ExpectFile } from '../file-assertions'; -import { explicitRelative, RewrittenPackageCache } from '../../../packages/shared-internals'; -import { resolve } from 'path'; +import { getRewrittenLocation } from '../rewritten-path'; class QUnitAdapter implements AssertionAdapter { constructor(private qassert: Assert) { @@ -41,21 +40,6 @@ export function expectFilesAt(basePath: string, params: { qunit: Assert }): Expe return func; } -function getRewrittenLocation(appDir: string, inputPath: string) { - let packageCache = RewrittenPackageCache.shared('embroider', appDir); - let fullInputPath = resolve(appDir, inputPath); - let owner = packageCache.ownerOfFile(fullInputPath); - if (!owner) { - return inputPath; - } - let movedOwner = packageCache.maybeMoved(owner); - if (movedOwner === owner) { - return inputPath; - } - let movedFullPath = fullInputPath.replace(owner.root, movedOwner.root); - return explicitRelative(appDir, movedFullPath); -} - export function expectRewrittenFilesAt( basePath: string, params: { qunit: Assert } diff --git a/test-packages/support/rewritten-path.ts b/test-packages/support/rewritten-path.ts new file mode 100644 index 000000000..73b3190e2 --- /dev/null +++ b/test-packages/support/rewritten-path.ts @@ -0,0 +1,17 @@ +import { explicitRelative, RewrittenPackageCache } from '../../packages/shared-internals'; +import { resolve } from 'path'; + +export function getRewrittenLocation(appDir: string, inputPath: string) { + let packageCache = RewrittenPackageCache.shared('embroider', appDir); + let fullInputPath = resolve(appDir, inputPath); + let owner = packageCache.ownerOfFile(fullInputPath); + if (!owner) { + return inputPath; + } + let movedOwner = packageCache.maybeMoved(owner); + if (movedOwner === owner) { + return inputPath; + } + let movedFullPath = fullInputPath.replace(owner.root, movedOwner.root); + return explicitRelative(appDir, movedFullPath); +} diff --git a/test-packages/support/transpiler.ts b/test-packages/support/transpiler.ts index 1132c659f..96c3c0604 100644 --- a/test-packages/support/transpiler.ts +++ b/test-packages/support/transpiler.ts @@ -2,11 +2,15 @@ import { readJSONSync } from 'fs-extra'; import { join } from 'path'; import { TransformOptions, transform } from '@babel/core'; import { BoundExpectFile } from './file-assertions'; -import { AppMeta, hbsToJS } from '../../packages/core/src/index'; +import { AppMeta, hbsToJS, RewrittenPackageCache } from '../../packages/core/src/index'; import { Memoize } from 'typescript-memoize'; +import { getRewrittenLocation } from './rewritten-path'; export class Transpiler { - constructor(private outputPath: string) { + private appOutputPath: string; + constructor(private appDir: string) { + let packageCache = RewrittenPackageCache.shared('embroider', appDir); + this.appOutputPath = packageCache.maybeMoved(packageCache.get(appDir)).root; this.transpile = this.transpile.bind(this); this.shouldTranspile = this.shouldTranspile.bind(this); } @@ -16,7 +20,7 @@ export class Transpiler { return transform( hbsToJS(contents, { filename: fileAssert.fullPath, - compatModuleNaming: { rootDir: this.outputPath, modulePrefix: this.pkgJSON.name }, + compatModuleNaming: { rootDir: this.appOutputPath, modulePrefix: this.pkgJSON.name }, }), Object.assign({ filename: fileAssert.fullPath }, this.babelConfig) )!.code!; @@ -29,13 +33,13 @@ export class Transpiler { shouldTranspile(relativePath: string) { // eslint-disable-next-line @typescript-eslint/no-require-imports - let shouldTranspile = require(join(this.outputPath, '_babel_filter_')); - return shouldTranspile(join(this.outputPath, relativePath)) as boolean; + let shouldTranspile = require(join(this.appOutputPath, '_babel_filter_')); + return shouldTranspile(join(this.appDir, getRewrittenLocation(this.appDir, relativePath))) as boolean; } @Memoize() private get pkgJSON() { - return readJSONSync(join(this.outputPath, 'package.json')); + return readJSONSync(join(this.appOutputPath, 'package.json')); } private get emberMeta(): AppMeta { @@ -48,6 +52,6 @@ export class Transpiler { throw new Error(`@embroider/test-support only suports babel 7`); } // eslint-disable-next-line @typescript-eslint/no-require-imports - return require(join(this.outputPath, this.emberMeta['babel'].filename)) as TransformOptions; + return require(join(this.appOutputPath, this.emberMeta['babel'].filename)) as TransformOptions; } } diff --git a/tests/scenarios/compat-stage2-test.ts b/tests/scenarios/compat-stage2-test.ts index 334d2e225..c6b03f967 100644 --- a/tests/scenarios/compat-stage2-test.ts +++ b/tests/scenarios/compat-stage2-test.ts @@ -27,16 +27,26 @@ stage2Scenarios depB.linkDependency('dep-c', { project: depC }); addInRepoAddon(depC, 'in-repo-d', { - app: { service: { 'in-repo.js': 'in-repo-d' } }, + app: { service: { 'in-repo.js': '//in-repo-d' } }, }); addInRepoAddon(depA, 'in-repo-a', { - app: { service: { 'in-repo.js': 'in-repo-a' } }, + app: { service: { 'in-repo.js': '//in-repo-a' } }, + addon: { + 'check-resolution-target.js': 'export {}', + }, + }); + merge(depA.files, { + addon: { + 'check-resolution.js': ` + import 'in-repo-a/check-resolution-target'; + `, + }, }); addInRepoAddon(depB, 'in-repo-b', { - app: { service: { 'in-repo.js': 'in-repo-b' } }, + app: { service: { 'in-repo.js': '//in-repo-b' } }, }); addInRepoAddon(depB, 'in-repo-c', { - app: { service: { 'in-repo.js': 'in-repo-c' } }, + app: { service: { 'in-repo.js': '//in-repo-c' } }, }); // make an in-repo addon with a dependency on a secondary in-repo-addon @@ -82,7 +92,7 @@ stage2Scenarios throwOnWarnings(hooks); let app: PreparedApp; - let expectAddonFile: ExpectFile; + let expectFile: ExpectFile; hooks.before(async assert => { app = await scenario.prepare(); @@ -91,26 +101,27 @@ stage2Scenarios }); hooks.beforeEach(assert => { - expectAddonFile = expectRewrittenFilesAt(app.dir, { qunit: assert }); + expectFile = expectRewrittenFilesAt(app.dir, { qunit: assert }); }); let expectAudit = setupAuditTest(hooks, () => ({ app: app.dir, 'reuse-build': true })); test('in repo addons are symlinked correctly', function () { // check that package json contains in repo dep - expectAddonFile('dep-a/package.json').json().get('dependencies.in-repo-a').equals('0.0.0'); - expectAddonFile('dep-b/package.json').json().get('dependencies.in-repo-c').equals('0.0.0'); - expectAddonFile('dep-b/package.json').json().get('dependencies.in-repo-b').equals('0.0.0'); + expectFile('./node_modules/dep-a/package.json').json().get('dependencies.in-repo-a').equals('0.0.0'); + expectFile('./node_modules/dep-b/package.json').json().get('dependencies.in-repo-c').equals('0.0.0'); + expectFile('./node_modules/dep-b/package.json').json().get('dependencies.in-repo-b').equals('0.0.0'); - // check that symlinks are correct - expectAddonFile('dep-a/node_modules/in-repo-a/package.json').exists(); - expectAddonFile('dep-b/node_modules/in-repo-b/package.json').exists(); - expectAddonFile('dep-b/node_modules/in-repo-c/package.json').exists(); + // check that in-repo addons are resolvable + expectAudit + .module('./node_modules/dep-a/check-resolution.js') + .resolves('in-repo-a/check-resolution-target') + .to('./node_modules/dep-a/lib/in-repo-a/check-resolution-target.js'); - // check that the in repo addons are correct upgraded - expectAddonFile('dep-a/node_modules/in-repo-a/package.json').json().get('ember-addon.version').equals(2); - expectAddonFile('dep-b/node_modules/in-repo-b/package.json').json().get('ember-addon.version').equals(2); - expectAddonFile('dep-b/node_modules/in-repo-c/package.json').json().get('ember-addon.version').equals(2); + // check that the in repo addons are correctly upgraded + expectFile('./node_modules/dep-a/lib/in-repo-a/package.json').json().get('ember-addon.version').equals(2); + expectFile('./node_modules/dep-b/lib/in-repo-b/package.json').json().get('ember-addon.version').equals(2); + expectFile('./node_modules/dep-b/lib/in-repo-c/package.json').json().get('ember-addon.version').equals(2); // check that the app trees with in repo addon are combined correctly expectAudit @@ -130,7 +141,7 @@ stage2Scenarios expectAudit .module('./lib/primary-in-repo-addon/_app_/services/primary.js') .resolves('secondary-in-repo-addon/components/secondary') - .to('./lib/primary-in-repo-addon/node_modules/secondary-in-repo-addon/components/secondary.js'); + .to('./lib/secondary-in-repo-addon/components/secondary.js'); }); }); }); @@ -471,10 +482,8 @@ stage2Scenarios }); hooks.beforeEach(assert => { - expectFile = expectFilesAt(readFileSync(join(builder.outputPath, '.stage2-output'), 'utf8'), { - qunit: assert, - }); - build = new Transpiler(expectFile.basePath); + expectFile = expectRewrittenFilesAt(app.dir, { qunit: assert }); + build = new Transpiler(app.dir); }); let expectAudit = setupAuditTest(hooks, () => ({ app: app.dir, 'reuse-build': true })); @@ -576,10 +585,10 @@ stage2Scenarios }); test('component with relative import of arbitrarily placed template', function () { - let assertFile = expectFile('node_modules/my-addon/components/has-relative-template.js').transform( - build.transpile - ); - assertFile.matches(/import layout from ["']\.\/t['"]/, 'arbitrary relative template remains the same'); + expectAudit + .module('node_modules/my-addon/components/has-relative-template.js') + .resolves('./t') + .to('node_modules/my-addon/components/t.js'); }); test('app can import a deep addon', function () { diff --git a/tests/scenarios/compat-template-colocation-test.ts b/tests/scenarios/compat-template-colocation-test.ts index 5a50f9a72..e6e96c133 100644 --- a/tests/scenarios/compat-template-colocation-test.ts +++ b/tests/scenarios/compat-template-colocation-test.ts @@ -1,7 +1,7 @@ import { PreparedApp } from 'scenario-tester'; import { appScenarios, baseAddon, renameApp } from './scenarios'; import { readFileSync } from 'fs'; -import { join, resolve } from 'path'; +import { join } from 'path'; import { Transpiler } from '@embroider/test-support'; import { ExpectFile, expectFilesAt, expectRewrittenFilesAt } from '@embroider/test-support/file-assertions/qunit'; import { throwOnWarnings } from '@embroider/core'; @@ -92,8 +92,8 @@ scenarios }); hooks.beforeEach(assert => { - let r = (expectFile = expectRewrittenFilesAt(app.dir, { qunit: assert })); - build = new Transpiler(resolve(app.dir, r.toRewrittenPath(app.dir))); + expectFile = expectRewrittenFilesAt(app.dir, { qunit: assert }); + build = new Transpiler(app.dir); }); test(`app's colocated template is associated with JS`, function () { From b3b6cd8cacd24a24e0d7d979be3bad6a041833d6 Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Sat, 17 Jun 2023 14:58:58 -0400 Subject: [PATCH 51/72] fix rebuilds by keeping rewritten-app apart from rewritten-addons --- packages/compat/src/compat-addons.ts | 5 ++--- packages/compat/src/standalone-addon-build.ts | 9 ++++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/packages/compat/src/compat-addons.ts b/packages/compat/src/compat-addons.ts index c8e973008..c6541879e 100644 --- a/packages/compat/src/compat-addons.ts +++ b/packages/compat/src/compat-addons.ts @@ -21,11 +21,9 @@ export default class CompatAddons implements Stage { private treeSync: TreeSync | undefined; readonly inputPath: string; - private destDir: string; private addons: Node; constructor(private compatApp: CompatApp) { - this.destDir = resolve(compatApp.root, 'node_modules', '.embroider', 'rewritten-packages', compatApp.name); this.addons = convertLegacyAddons(compatApp); this.inputPath = compatApp.root; } @@ -36,7 +34,7 @@ export default class CompatAddons implements Stage { async ready(): Promise<{ outputPath: string }> { return { - outputPath: this.destDir, + outputPath: resolve(this.compatApp.root, 'node_modules', '.embroider', 'rewritten-app'), }; } @@ -58,6 +56,7 @@ export default class CompatAddons implements Stage { !this.didBuild || // always copy on the first build changedMap.get(addons) ) { + // the problem this.treeSync.sync(); RewrittenPackageCache.shared('embroider', this.compatApp.root).invalidateIndex(); } diff --git a/packages/compat/src/standalone-addon-build.ts b/packages/compat/src/standalone-addon-build.ts index 8f043275a..3de0d4dea 100644 --- a/packages/compat/src/standalone-addon-build.ts +++ b/packages/compat/src/standalone-addon-build.ts @@ -7,6 +7,7 @@ import broccoliMergeTrees from 'broccoli-merge-trees'; import writeFile from 'broccoli-file-creator'; import type { Node } from 'broccoli-node-api'; import CompatApp from './compat-app'; +import { join } from 'path'; export function convertLegacyAddons(compatApp: CompatApp) { let packageCache = PackageCache.shared('embroider', compatApp.root); @@ -51,15 +52,17 @@ function buildAddonIndex(compatApp: CompatApp, appPackage: Package, packages: Se // adding an entry for the app itself to have a place in the // rewritten-packages, even though this stage hasn't actually put it there - // yet. - content.packages[compatApp.root] = compatApp.name; + // yet. This directory lives outside our rewritten-pacakges directory because + // it's produced by a separate build stage, and it's easier to have them + // writing into separate directories. + content.packages[compatApp.root] = join('..', 'rewritten-app'); let nonResolvableDeps = appPackage.nonResolvableDeps; if (nonResolvableDeps) { let extraRoots = [...nonResolvableDeps.values()].map(v => v.root); // the app gets extraResolutions support just like every addon does - content.extraResolutions[compatApp.name] = extraRoots; + content.extraResolutions[join('..', 'rewritten-app')] = extraRoots; // but it also gets extraResolutions registered against its *original* // location, because the app is unique because stage2 needs a Package From e047151d4a5d12ee555e5c2af17c2a19dbae8cce Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Sat, 17 Jun 2023 16:50:17 -0400 Subject: [PATCH 52/72] restoring dummy app support --- packages/compat/src/compat-addons.ts | 1 - packages/compat/src/compat-app.ts | 22 +++++++++---------- packages/compat/src/standalone-addon-build.ts | 2 +- tests/scenarios/compat-dummy-app-test.ts | 22 ++++++++++++++----- 4 files changed, 28 insertions(+), 19 deletions(-) diff --git a/packages/compat/src/compat-addons.ts b/packages/compat/src/compat-addons.ts index c6541879e..3ed436927 100644 --- a/packages/compat/src/compat-addons.ts +++ b/packages/compat/src/compat-addons.ts @@ -56,7 +56,6 @@ export default class CompatAddons implements Stage { !this.didBuild || // always copy on the first build changedMap.get(addons) ) { - // the problem this.treeSync.sync(); RewrittenPackageCache.shared('embroider', this.compatApp.root).invalidateIndex(); } diff --git a/packages/compat/src/compat-app.ts b/packages/compat/src/compat-app.ts index bf230cb3b..7e967e453 100644 --- a/packages/compat/src/compat-app.ts +++ b/packages/compat/src/compat-app.ts @@ -1,5 +1,5 @@ import { Node as BroccoliNode } from 'broccoli-node-api'; -import { PackageCache, WaitForTrees, Stage, RewrittenPackageCache } from '@embroider/core'; +import { PackageCache, WaitForTrees, Stage, RewrittenPackageCache, Package } from '@embroider/core'; import Options, { optionsWithDefaults } from './options'; import { Memoize } from 'typescript-memoize'; import { sync as pkgUpSync } from 'pkg-up'; @@ -806,22 +806,22 @@ export default class CompatApp { }; } - private async instantiate( - root: string, - appSrcDir: string, - packageCache: RewrittenPackageCache, - configTree: V1Config - ) { - let origAppPkg; + @Memoize() + appPackage(): Package { + let packageCache = RewrittenPackageCache.shared('embroider', this.root); if (this.isDummy) { - origAppPkg = new DummyPackage( + return new DummyPackage( this.root, this.legacyEmberAppInstance.project.root, packageCache as unknown as PackageCache // TODO: cast won't be needed when refactor is complete ); } else { - origAppPkg = packageCache.get(appSrcDir); + return packageCache.get(this.root); } + } + + private async instantiate(root: string, packageCache: RewrittenPackageCache, configTree: V1Config) { + let origAppPkg = this.appPackage(); let movedAppPkg = packageCache.withRewrittenDeps(origAppPkg); return new CompatAppBuilder( root, @@ -849,7 +849,7 @@ export default class CompatApp { if (!this.active) { let { outputPath } = await prevStage.ready(); let packageCache = RewrittenPackageCache.shared('embroider', this.root); - this.active = await this.instantiate(outputPath, prevStage.inputPath, packageCache, inTrees.configTree); + this.active = await this.instantiate(outputPath, packageCache, inTrees.configTree); resolve({ outputPath }); } await this.active.build(treePaths); diff --git a/packages/compat/src/standalone-addon-build.ts b/packages/compat/src/standalone-addon-build.ts index 3de0d4dea..e71f27606 100644 --- a/packages/compat/src/standalone-addon-build.ts +++ b/packages/compat/src/standalone-addon-build.ts @@ -13,7 +13,7 @@ export function convertLegacyAddons(compatApp: CompatApp) { let packageCache = PackageCache.shared('embroider', compatApp.root); let instanceCache = new V1InstanceCache(compatApp, packageCache); - let appPackage = packageCache.get(compatApp.root); + let appPackage = compatApp.appPackage(); let v1Addons = findV1Addons(appPackage); let index = buildAddonIndex(compatApp, appPackage, v1Addons); diff --git a/tests/scenarios/compat-dummy-app-test.ts b/tests/scenarios/compat-dummy-app-test.ts index f7df7777a..1c02fb7b4 100644 --- a/tests/scenarios/compat-dummy-app-test.ts +++ b/tests/scenarios/compat-dummy-app-test.ts @@ -1,10 +1,10 @@ -import { ExpectFile, expectFilesAt } from '@embroider/test-support/file-assertions/qunit'; +import { ExpectFile, expectRewrittenFilesAt } from '@embroider/test-support/file-assertions/qunit'; import { Rebuilder } from '@embroider/test-support'; import { PreparedApp } from 'scenario-tester'; import { throwOnWarnings } from '@embroider/core'; import merge from 'lodash/merge'; -import { readFileSync, writeFileSync } from 'fs'; -import { join } from 'path'; +import { writeFileSync } from 'fs'; +import { join, resolve } from 'path'; import QUnit from 'qunit'; const { module: Qmodule, test } = QUnit; @@ -47,7 +47,9 @@ dummyAppScenarios }); hooks.beforeEach(assert => { - expectFile = expectFilesAt(readFileSync(join(builder.outputPath, '.stage2-output'), 'utf8'), { qunit: assert }); + expectFile = expectRewrittenFilesAt(resolve(app.dir, 'tests/dummy'), { + qunit: assert, + }); }); test('rebuilds addon code', async function () { @@ -58,8 +60,16 @@ dummyAppScenarios }); test('contains public assets from dummy app', async function () { - expectFile('robots.txt').exists(); - expectFile('package.json').json().get('ember-addon.assets').includes('robots.txt'); + // expectRewrittenFilesAt doesn't understand dummy apps, so even though + // we initialized it on app.dir/tests/dummy, we can't just say + // "robots.txt" here because it thinks that file belongs to the + // containing addon. By writing out the rewritten paths ourselves we + // sidestep that problem≥ + expectFile('./node_modules/.embroider/rewritten-app/robots.txt').exists(); + expectFile('./node_modules/.embroider/rewritten-app/package.json') + .json() + .get('ember-addon.assets') + .includes('robots.txt'); }); }); }); From 4435a34f3358149c4ef026b3e6609edc43962151 Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Sat, 17 Jun 2023 20:34:42 -0400 Subject: [PATCH 53/72] fix isDevelopingThisPackage --- packages/compat/src/compat-app.ts | 6 ++++++ packages/macros/src/babel/state.ts | 2 +- tests/scenarios/compat-stage2-test.ts | 19 ++++++++----------- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/packages/compat/src/compat-app.ts b/packages/compat/src/compat-app.ts index 7e967e453..5ac90c793 100644 --- a/packages/compat/src/compat-app.ts +++ b/packages/compat/src/compat-app.ts @@ -775,6 +775,12 @@ export default class CompatApp { if (this.env !== 'production') { this.macrosConfig.enablePackageDevelopment(this.root); this.macrosConfig.enableRuntimeMode(); + if (this.isDummy) { + // dummy apps automatically put their owning addon under development too + this.macrosConfig.enablePackageDevelopment( + dirname(pkgUpSync({ cwd: this.legacyEmberAppInstance.project.root })!) + ); + } } // this uses globalConfig because it's a way for packages to ask "is diff --git a/packages/macros/src/babel/state.ts b/packages/macros/src/babel/state.ts index 33faf7e80..04a49d0e9 100644 --- a/packages/macros/src/babel/state.ts +++ b/packages/macros/src/babel/state.ts @@ -86,7 +86,7 @@ function owningPackage(this: State): Package { if (!pkg) { throw new Error(`unable to determine which npm package owns the file ${this.sourceFile}`); } - return pkg; + return this.packageCache.original(pkg) || pkg; } function cloneDeep(this: State, node: Node): Node { diff --git a/tests/scenarios/compat-stage2-test.ts b/tests/scenarios/compat-stage2-test.ts index c6b03f967..2e82e8037 100644 --- a/tests/scenarios/compat-stage2-test.ts +++ b/tests/scenarios/compat-stage2-test.ts @@ -2,10 +2,9 @@ import { Options } from '@embroider/compat'; import { writeFileSync, unlinkSync } from 'fs'; import { PreparedApp, Project } from 'scenario-tester'; import { appScenarios, baseAddon, dummyAppScenarios, renameApp } from './scenarios'; -import { readFileSync } from 'fs'; -import { join } from 'path'; +import { join, resolve } from 'path'; import { Rebuilder, Transpiler } from '@embroider/test-support'; -import { expectFilesAt, expectRewrittenFilesAt, ExpectFile } from '@embroider/test-support/file-assertions/qunit'; +import { expectRewrittenFilesAt, ExpectFile } from '@embroider/test-support/file-assertions/qunit'; import { throwOnWarnings } from '@embroider/core'; import merge from 'lodash/merge'; import QUnit from 'qunit'; @@ -755,21 +754,19 @@ dummyAppScenarios }); hooks.beforeEach(assert => { - expectFile = expectFilesAt(readFileSync(join(app.dir, 'dist/.stage2-output'), 'utf8'), { qunit: assert }); - build = new Transpiler(expectFile.basePath); + expectFile = expectRewrittenFilesAt(resolve(app.dir, 'tests/dummy'), { qunit: assert }); + build = new Transpiler(resolve(app.dir, 'tests/dummy')); }); test('dummy app sees that its being developed', function () { - let assertFile = expectFile('components/inside-dummy-app.js').transform(build.transpile); + let assertFile = expectFile('node_modules/.embroider/rewritten-app/components/inside-dummy-app.js').transform( + build.transpile + ); assertFile.matches(/console\.log\(true\)/); }); test('addon within dummy app sees that its being developed', function () { - let assertFile = expectFile( - require.resolve('my-addon/components/hello-world', { - paths: [expectFile.basePath], - }) - ).transform(build.transpile); + let assertFile = expectFile('../../components/hello-world.js').transform(build.transpile); assertFile.matches(/console\.log\(true\)/); }); }); From d10569f6f007a69c7ab64b7ead8bfd995476c397 Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Sun, 18 Jun 2023 18:09:13 -0400 Subject: [PATCH 54/72] bugfix: using rewritten app root for v2 addon virtual peer deps --- packages/core/src/module-resolver.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/core/src/module-resolver.ts b/packages/core/src/module-resolver.ts index 6be62e3b3..73131fc97 100644 --- a/packages/core/src/module-resolver.ts +++ b/packages/core/src/module-resolver.ts @@ -772,7 +772,10 @@ export class Resolver { if (!this.options.activeAddons[packageName]) { throw new Error(`${pkg.name} is trying to import the app's ${packageName} package, but it seems to be missing`); } - let newHome = resolve(this.options.appRoot, 'package.json'); + let newHome = resolve( + this.packageCache.maybeMoved(this.packageCache.get(this.options.appRoot)).root, + 'package.json' + ); return logTransition(`emberVirtualPeerDeps in v2 addon`, request, request.rehome(newHome)); } From 270796f5c46a1c9ad7a5d32ce2c7c98dcdf516cb Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Mon, 19 Jun 2023 15:04:42 -0400 Subject: [PATCH 55/72] updating more tests --- tests/scenarios/stage1-test.ts | 183 +++++++++++++++++---------------- 1 file changed, 95 insertions(+), 88 deletions(-) diff --git a/tests/scenarios/stage1-test.ts b/tests/scenarios/stage1-test.ts index 7421ac808..6f1d58fa0 100644 --- a/tests/scenarios/stage1-test.ts +++ b/tests/scenarios/stage1-test.ts @@ -5,7 +5,7 @@ import { loadFromFixtureData } from './helpers'; import { dummyAppScenarios, baseAddon, appScenarios } from './scenarios'; import { PreparedApp } from 'scenario-tester'; import QUnit from 'qunit'; -import { expectFilesAt, ExpectFile } from '@embroider/test-support/file-assertions/qunit'; +import { expectRewrittenFilesAt, ExpectFile } from '@embroider/test-support/file-assertions/qunit'; const { module: Qmodule, test } = QUnit; @@ -28,95 +28,105 @@ appScenarios .forEachScenario(async scenario => { Qmodule(`${scenario.name}`, function (hooks) { let app: PreparedApp; - let workspaceDir: string; + let expectFile: ExpectFile; hooks.before(async assert => { process.env.THROW_UNLESS_PARALLELIZABLE = '1'; // see https://github.com/embroider-build/embroider/pull/924 app = await scenario.prepare(); let result = await app.execute('cross-env STAGE1_ONLY=true node ./node_modules/ember-cli/bin/ember b'); assert.equal(result.exitCode, 0, result.output); - workspaceDir = fs.readFileSync(join(app.dir, 'dist', '.stage1-output'), 'utf8'); }); hooks.after(async () => { delete process.env.THROW_UNLESS_PARALLELIZABLE; }); - test('component in app tree', function (assert) { - assert.ok(fs.existsSync(join(workspaceDir, 'node_modules/my-addon/_app_/components/hello-world.js'))); + hooks.beforeEach(assert => { + expectFile = expectRewrittenFilesAt(app.dir, { qunit: assert }); }); - test('addon metadata', function (assert) { - let assertMeta = fs.readJsonSync(join(workspaceDir, 'node_modules/my-addon/package.json'))['ember-addon']; - assert.deepEqual(assertMeta['app-js'], { './components/hello-world.js': './_app_/components/hello-world.js' }); - assert.ok( - JSON.stringify(assertMeta['implicit-modules']).includes('./components/hello-world'), - 'staticAddonTrees is off so we should include the component implicitly' - ); - assert.ok( - JSON.stringify(assertMeta['implicit-modules']).includes('./templates/components/hello-world'), - 'staticAddonTrees is off so we should include the template implicitly' - ); + test('component in app tree', function () { + expectFile('node_modules/my-addon/_app_/components/hello-world.js').exists(); + }); - assert.equal(assertMeta.version, 2); + test('addon metadata', function () { + let myAddonPkg = expectFile('node_modules/my-addon/package.json').json(); + myAddonPkg + .get('ember-addon.app-js') + .deepEquals({ './components/hello-world.js': './_app_/components/hello-world.js' }); + + myAddonPkg + .get('ember-addon.implicit-modules') + .includes( + './components/hello-world', + 'staticAddonTrees is off so we should include the component implicitly' + ); + + myAddonPkg + .get('ember-addon.implicit-modules') + .includes( + './templates/components/hello-world.hbs', + 'staticAddonTrees is off so we should include the template implicitly' + ); + + myAddonPkg.get('ember-addon.version').deepEquals(2); }); - test('component in addon tree', function (assert) { - let fileContents = fs.readFileSync(join(workspaceDir, 'node_modules/my-addon/components/hello-world.js')); + test('component in addon tree', function () { + expectFile('node_modules/my-addon/components/hello-world.js').matches( + 'getOwnConfig()', + 'JS macros have not run yet' + ); - assert.ok(fileContents.includes('getOwnConfig()'), 'JS macros have not run yet'); - assert.ok(fileContents.includes('embroider-sample-transforms-result'), 'custom babel plugins have run'); + expectFile('node_modules/my-addon/components/hello-world.js').matches( + 'embroider-sample-transforms-result', + 'custom babel plugins have run' + ); }); - test('component template in addon tree', function (assert) { - let fileContents = fs.readFileSync( - join(workspaceDir, 'node_modules/my-addon/templates/components/hello-world.hbs.js') - ); - assert.ok( - fileContents.includes('
hello world
'), + test('component template in addon tree', function () { + let fileContents = expectFile('node_modules/my-addon/templates/components/hello-world.hbs.js'); + + fileContents.matches( + '
hello world
', 'template is still hbs and custom transforms have run' ); - assert.ok( - fileContents.includes('{{macroDependencySatisfies \\"ember-source\\" \\">3\\"}}'), + + fileContents.matches( + '{{macroDependencySatisfies \\"ember-source\\" \\">3\\"}}', 'template macros have not run' ); }); - test('test module name added', function (assert) { - let fileContents = fs.readFileSync( - join(workspaceDir, 'node_modules/my-addon/templates/components/module-name.hbs.js') - ); + test('test module name added', function () { + let fileContents = expectFile('node_modules/my-addon/templates/components/module-name.hbs.js'); let expected = `
hello world
`; - assert.ok(fileContents.includes(expected), 'template is still hbs and module name transforms have run'); + fileContents.matches(expected, 'template is still hbs and module name transforms have run'); }); - test('component with inline template', function (assert) { - let fileContents = fs.readFileSync( - join(workspaceDir, 'node_modules/my-addon/components/has-inline-template.js') - ); - assert.ok( - fileContents.includes('hbs`
Inline
'), + test('component with inline template', function () { + let fileContents = expectFile('node_modules/my-addon/components/has-inline-template.js'); + + fileContents.matches( + 'hbs`
Inline
', 'tagged template is still hbs and custom transforms have run' ); - assert.ok( - /hbs\(["']
Extra<\/div>["']\)/.test(fileContents.toString()), + + fileContents.matches( + /hbs\(["']
Extra<\/div>["']\)/, 'called template is still hbs and custom transforms have run' ); - assert.ok( - /{{macroDependencySatisfies ['"]ember-source['"] ['"]>3['"]}}<\/span>/.test(fileContents.toString()), + + fileContents.matches( + /{{macroDependencySatisfies ['"]ember-source['"] ['"]>3['"]}}<\/span>/, 'template macros have not run' ); }); - test('in-repo-addon is available', function (assert) { - assert.ok(require.resolve('in-repo-addon/helpers/helper-from-in-repo-addon', { paths: [workspaceDir] })); - }); - - test('dynamic import is preserved', function (assert) { - let fileContents = fs.readFileSync( - join(workspaceDir, 'node_modules/my-addon/components/does-dynamic-import.js') + test('dynamic import is preserved', function () { + expectFile('node_modules/my-addon/components/does-dynamic-import.js').matches( + /return import\(['"]some-library['"]\)/ ); - assert.ok(/return import\(['"]some-library['"]\)/.test(fileContents.toString())); }); }); }); @@ -159,17 +169,15 @@ appScenarios .forEachScenario(async scenario => { Qmodule(`${scenario.name}`, function (hooks) { let app: PreparedApp; - let workspaceDir: string; + let expectFile: ExpectFile; hooks.before(async () => { app = await scenario.prepare(); await app.execute('cross-env STAGE1_ONLY=true node ./node_modules/ember-cli/bin/ember b'); - workspaceDir = fs.readFileSync(join(app.dir, 'dist', '.stage1-output'), 'utf8'); }); - let expectFile: ExpectFile; hooks.beforeEach(assert => { - expectFile = expectFilesAt(workspaceDir, { qunit: assert }); + expectFile = expectRewrittenFilesAt(app.dir, { qunit: assert }); }); test('component with inline template', function () { @@ -407,66 +415,65 @@ appScenarios .forEachScenario(async scenario => { Qmodule(`${scenario.name}`, function (hooks) { let app: PreparedApp; - let workspaceDir: string; + let expectFile: ExpectFile; hooks.before(async () => { app = await scenario.prepare(); await app.execute('cross-env STAGE1_ONLY=true node ./node_modules/ember-cli/bin/ember b'); - workspaceDir = fs.readFileSync(join(app.dir, 'dist', '.stage1-output'), 'utf8'); }); - test('real package.json wins', function (assert) { - let fileContents = fs.readFileSync(join(workspaceDir, 'node_modules/alpha/package.json')); - assert.ok(fileContents.includes('alpha')); + hooks.beforeEach(assert => { + expectFile = expectRewrittenFilesAt(app.dir, { qunit: assert }); + }); + + test('real package.json wins', function () { + expectFile('node_modules/alpha/package.json').matches('alpha'); }); - test('custom tree hooks are detected in addons that manually extend from Addon', function (assert) { - let fileContents = fs.readFileSync(join(workspaceDir, 'node_modules/has-custom-base/file.js')); - assert.ok(/weird-addon-path\/file\.js/.test(fileContents.toString())); + test('custom tree hooks are detected in addons that manually extend from Addon', function () { + expectFile('node_modules/has-custom-base/file.js').matches(/weird-addon-path\/file\.js/); }); - test('no fastboot-js is emitted', function (assert) { - let fileContents = fs.readJsonSync(join(workspaceDir, 'node_modules/undefined-fastboot/package.json')); - assert.equal(fileContents['ember-addon']['fastboot-js'], null); + test('no fastboot-js is emitted', function () { + expectFile('node_modules/undefined-fastboot/package.json').json().get('ember-addon.fastboot-js').equals(null); }); - test('custom tree hooks are detected when they have been patched into the addon instance', function (assert) { - assert.ok(fs.existsSync(join(workspaceDir, 'node_modules/externally-customized/public/hello/world.js'))); + test('custom tree hooks are detected when they have been patched into the addon instance', function () { + expectFile('node_modules/externally-customized/public/hello/world.js').exists(); }); - test('custom tree hooks are detected when they have been customized via treeForMethod names', function (assert) { - assert.ok(fs.existsSync(join(workspaceDir, 'node_modules/patches-method-name/hello/world.js'))); + test('custom tree hooks are detected when they have been customized via treeForMethod names', function () { + expectFile('node_modules/patches-method-name/hello/world.js').exists(); }); - test('addon with customized ember-addon.main can still use stock trees', function (assert) { - let fileContents = fs.readFileSync(join(workspaceDir, 'node_modules/moved-main/helpers/hello.js')); - assert.ok(/hello-world/.test(fileContents.toString())); + test('addon with customized ember-addon.main can still use stock trees', function () { + expectFile('node_modules/moved-main/helpers/hello.js').matches(/hello-world/); }); - test('addon with customized treeFor can suppress a stock tree', function (assert) { - assert.notOk(fs.existsSync(join(workspaceDir, 'node_modules/suppressed/_app_/app-example.js'))); + test('addon with customized treeFor can suppress a stock tree', function () { + expectFile('node_modules/suppressed/_app_/app-example.js').doesNotExist(); }); - test('addon with customized treeFor can pass through a stock tree', function (assert) { - assert.ok(fs.existsSync(join(workspaceDir, 'node_modules/suppressed/addon-example.js'))); + test('addon with customized treeFor can pass through a stock tree', function () { + expectFile('node_modules/suppressed/addon-example.js').exists(); }); - test('addon with customized treeFor can suppress a customized tree', function (assert) { - assert.notOk(fs.existsSync(join(workspaceDir, 'node_modules/suppressed-custom/_app_/app-example.js'))); + test('addon with customized treeFor can suppress a customized tree', function () { + expectFile('node_modules/suppressed-custom/_app_/app-example.js').doesNotExist(); }); - test('addon with customized treeFor can pass through a customized tree', function (assert) { - assert.ok(fs.existsSync(join(workspaceDir, 'node_modules/suppressed-custom/addon-example.js'))); + test('addon with customized treeFor can pass through a customized tree', function () { + expectFile('node_modules/suppressed-custom/addon-example.js').exists(); }); - test('blacklisted in-repo addon is present but empty', function (assert) { - assert.ok(fs.existsSync(join(workspaceDir, 'lib/blacklisted-in-repo-addon/package.json'))); - assert.notOk(fs.existsSync(join(workspaceDir, 'lib/blacklisted-in-repo-addon/example.js'))); + test('blacklisted in-repo addon is present but empty', function () { + expectFile('lib/blacklisted-in-repo-addon/package.json').exists(); + expectFile('lib/blacklisted-in-repo-addon/example.js').doesNotExist(); }); - test('disabled in-repo addon is present but empty', function (assert) { - assert.ok(fs.existsSync(join(workspaceDir, 'lib/disabled-in-repo-addon/package.json'))); - assert.notOk(fs.existsSync(join(workspaceDir, 'lib/disabled-in-repo-addon/example.js'))); + test('disabled in-repo addon is present but empty', function () { + expectFile('lib/disabled-in-repo-addon/package.json').exists(); + expectFile('lib/disabled-in-repo-addon/example.js').doesNotExist(); }); }); }); From 89aaee8109bec49f901fec3a270093a9931b1c7d Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Mon, 19 Jun 2023 18:50:15 -0400 Subject: [PATCH 56/72] updating more tests (This is not a stage1 concern now) --- tests/scenarios/stage1-test.ts | 37 +--------------------------------- 1 file changed, 1 insertion(+), 36 deletions(-) diff --git a/tests/scenarios/stage1-test.ts b/tests/scenarios/stage1-test.ts index 6f1d58fa0..01a302a06 100644 --- a/tests/scenarios/stage1-test.ts +++ b/tests/scenarios/stage1-test.ts @@ -1,8 +1,6 @@ -import { join } from 'path'; import merge from 'lodash/merge'; -import fs from 'fs-extra'; import { loadFromFixtureData } from './helpers'; -import { dummyAppScenarios, baseAddon, appScenarios } from './scenarios'; +import { baseAddon, appScenarios } from './scenarios'; import { PreparedApp } from 'scenario-tester'; import QUnit from 'qunit'; import { expectRewrittenFilesAt, ExpectFile } from '@embroider/test-support/file-assertions/qunit'; @@ -477,36 +475,3 @@ appScenarios }); }); }); - -dummyAppScenarios - .map('stage-1-dummy-addon', project => { - project.pkg.name = 'my-addon'; - - project.linkDependency('@embroider/webpack', { baseDir: __dirname }); - project.linkDependency('@embroider/core', { baseDir: __dirname }); - project.linkDependency('@embroider/compat', { baseDir: __dirname }); - - merge(project.files, { - addon: { - components: { - 'hello-world.js': '', - }, - }, - }); - }) - .forEachScenario(async scenario => { - Qmodule(`${scenario.name}`, function (hooks) { - let app: PreparedApp; - let workspaceDir: string; - - hooks.before(async () => { - app = await scenario.prepare(); - await app.execute('cross-env STAGE1_ONLY=true node ./node_modules/ember-cli/bin/ember b'); - workspaceDir = fs.readFileSync(join(app.dir, 'dist', '.stage1-output'), 'utf8'); - }); - - test('dummy app can resolve own addon', function (assert) { - assert.ok(require.resolve('my-addon/components/hello-world.js', { paths: [workspaceDir] })); - }); - }); - }); From 64318870b4055064100059a8318b64f470b59385 Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Mon, 19 Jun 2023 19:25:37 -0400 Subject: [PATCH 57/72] skipping some lts_4_4 tests due to ember-data --- tests/scenarios/fastboot-app-test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/scenarios/fastboot-app-test.ts b/tests/scenarios/fastboot-app-test.ts index f587a6b54..9f1633298 100644 --- a/tests/scenarios/fastboot-app-test.ts +++ b/tests/scenarios/fastboot-app-test.ts @@ -7,6 +7,7 @@ import { JSDOM } from 'jsdom'; const { module: Qmodule, test } = QUnit; appScenarios + .skip('lts_4_4') // @ember-data/debug 4.4 has an undeclared peerDep on @ember-data/store that wreaks havoc in our monorepo. .map('fastboot-app-test', project => { project.pkg.fastbootDependencies = ['crypto', 'node-fetch']; From 6d8e76f07534858ef3eae5af789ca466158d3020 Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Mon, 19 Jun 2023 19:31:02 -0400 Subject: [PATCH 58/72] unmoved packages can still resolve moved packages --- .../src/rewritten-package-cache.ts | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/packages/shared-internals/src/rewritten-package-cache.ts b/packages/shared-internals/src/rewritten-package-cache.ts index 248735b8e..f53cea9fe 100644 --- a/packages/shared-internals/src/rewritten-package-cache.ts +++ b/packages/shared-internals/src/rewritten-package-cache.ts @@ -51,18 +51,19 @@ export class RewrittenPackageCache implements PackageCacheTheGoodParts { } } + let resolveFromPkg: Package; let oldRoot = this.index.newToOld.get(fromPackage.root); - if (!oldRoot) { - // the fromPackage has not been moved, so we're just providing the plain - // behavior. - return this.plainCache.resolve(packageName, fromPackage); + if (oldRoot) { + // the requesting package has been moved, so do the resolving from the old location + resolveFromPkg = this.plainCache.get(oldRoot); + } else { + // the requesting package has not been moved + resolveFromPkg = fromPackage; } - // do the real resolving from the old location - let oldSrc = this.plainCache.get(oldRoot); - let oldDest = this.plainCache.resolve(packageName, oldSrc); + let oldDest = this.plainCache.resolve(packageName, resolveFromPkg); - // and if the package we found was itself moved return the moved one. + // if the package we found was itself moved return the moved one. return this.maybeMoved(oldDest); } From 0382837a2ccf74ad6e5f885bc399b9b31bdafd52 Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Tue, 20 Jun 2023 11:03:23 -0400 Subject: [PATCH 59/72] Compat adapters to deal with ember-data 4.4 undeclared dep @ember-data/debug 4.4 has an undeclared peerDep on @ember-data/store. This can cause it to resolve incorrect copies, which under Embroider may result in pulling the un-rewritten v1 addon into the build causing explosions. --- .../src/compat-adapters/@ember-data/debug.ts | 14 ++++++++++++++ .../src/compat-adapters/@ember-data/store.ts | 18 +++++++++++++++++- 2 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 packages/compat/src/compat-adapters/@ember-data/debug.ts diff --git a/packages/compat/src/compat-adapters/@ember-data/debug.ts b/packages/compat/src/compat-adapters/@ember-data/debug.ts new file mode 100644 index 000000000..064f4bfb1 --- /dev/null +++ b/packages/compat/src/compat-adapters/@ember-data/debug.ts @@ -0,0 +1,14 @@ +import { AddonMeta } from '@embroider/core'; +import V1Addon from '../../v1-addon'; + +export default class EmberDataDebug extends V1Addon { + get packageMeta(): Partial { + let meta = super.packageMeta; + + // See also the compat-adapter for @ember-data/store where we make this an + // implicit-module. + meta.externals = [...(meta.externals ?? []), '@ember-data/store']; + + return meta; + } +} diff --git a/packages/compat/src/compat-adapters/@ember-data/store.ts b/packages/compat/src/compat-adapters/@ember-data/store.ts index 32436f631..392e5ef06 100644 --- a/packages/compat/src/compat-adapters/@ember-data/store.ts +++ b/packages/compat/src/compat-adapters/@ember-data/store.ts @@ -1 +1,17 @@ -export { EmberDataBase as default } from '../ember-data'; +import { AddonMeta } from '@embroider/core'; +import { EmberDataBase } from '../ember-data'; + +export default class EmberDataStore extends EmberDataBase { + get packageMeta(): Partial { + let meta = super.packageMeta; + + // this is here because the compat-adapter for @ember-data/debug adds this + // to externals because it has an undeclared peerDep on us, and thus might + // resolve totally incorrect copies. By making it external we leave it up to + // runtime, where we will find this implicit-module for the actual copy of + // @ember-data/store that is active in app. + meta['implicit-modules'] = [...(meta['implicit-modules'] ?? []), './index.js']; + + return meta; + } +} From bd51bd7c5831bf6160a24dc9ba23406530a2a1c8 Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Tue, 20 Jun 2023 14:11:18 -0400 Subject: [PATCH 60/72] unskip test due to ember-data compat-adapter --- tests/scenarios/fastboot-app-test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/scenarios/fastboot-app-test.ts b/tests/scenarios/fastboot-app-test.ts index 9f1633298..f587a6b54 100644 --- a/tests/scenarios/fastboot-app-test.ts +++ b/tests/scenarios/fastboot-app-test.ts @@ -7,7 +7,6 @@ import { JSDOM } from 'jsdom'; const { module: Qmodule, test } = QUnit; appScenarios - .skip('lts_4_4') // @ember-data/debug 4.4 has an undeclared peerDep on @ember-data/store that wreaks havoc in our monorepo. .map('fastboot-app-test', project => { project.pkg.fastbootDependencies = ['crypto', 'node-fetch']; From e3ba9864d2d22a1a40ab3ea62d2eb769661c9aa4 Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Tue, 20 Jun 2023 14:31:58 -0400 Subject: [PATCH 61/72] update glimmer macros to be aware of rewritten-package-cache --- packages/macros/src/glimmer/ast-transform.ts | 4 ++-- packages/macros/src/glimmer/dependency-satisfies.ts | 4 ++-- packages/macros/src/glimmer/get-config.ts | 6 ++++-- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/packages/macros/src/glimmer/ast-transform.ts b/packages/macros/src/glimmer/ast-transform.ts index f6e4a57c5..b7d0a3aa9 100644 --- a/packages/macros/src/glimmer/ast-transform.ts +++ b/packages/macros/src/glimmer/ast-transform.ts @@ -4,7 +4,7 @@ import dependencySatisfies from './dependency-satisfies'; import { maybeAttrs } from './macro-maybe-attrs'; import { macroIfBlock, macroIfExpression, macroIfMustache } from './macro-condition'; import { failBuild } from './fail-build'; -import { PackageCache } from '@embroider/shared-internals'; +import { RewrittenPackageCache } from '@embroider/shared-internals'; export interface BuildPluginParams { // Glimmer requires this on ast transforms. @@ -53,7 +53,7 @@ export function makeFirstTransform(opts: FirstTransformParams) { throw new Error(`bug in @embroider/macros. Running without packageRoot but don't have filename.`); } - let packageCache = PackageCache.shared('embroider', opts.appRoot); + let packageCache = RewrittenPackageCache.shared('embroider', opts.appRoot); let scopeStack: string[][] = []; diff --git a/packages/macros/src/glimmer/dependency-satisfies.ts b/packages/macros/src/glimmer/dependency-satisfies.ts index 9202181a5..9d0006385 100644 --- a/packages/macros/src/glimmer/dependency-satisfies.ts +++ b/packages/macros/src/glimmer/dependency-satisfies.ts @@ -1,5 +1,5 @@ import { satisfies } from 'semver'; -import type { PackageCache } from '@embroider/shared-internals'; +import type { RewrittenPackageCache } from '@embroider/shared-internals'; export default function dependencySatisfies( node: any, @@ -9,7 +9,7 @@ export default function dependencySatisfies( // unconfigured and moduleName will be the full path to the source file. baseDir: string | undefined, moduleName: string, - packageCache: PackageCache + packageCache: RewrittenPackageCache ) { if (node.params.length !== 2) { throw new Error(`macroDependencySatisfies requires two arguments, you passed ${node.params.length}`); diff --git a/packages/macros/src/glimmer/get-config.ts b/packages/macros/src/glimmer/get-config.ts index 62e17209e..e1459aadc 100644 --- a/packages/macros/src/glimmer/get-config.ts +++ b/packages/macros/src/glimmer/get-config.ts @@ -1,4 +1,4 @@ -import type { PackageCache } from '@embroider/shared-internals'; +import type { RewrittenPackageCache } from '@embroider/shared-internals'; export default function getConfig( node: any, @@ -10,7 +10,7 @@ export default function getConfig( baseDir: string | undefined, moduleName: string, own: boolean, - packageCache: PackageCache + packageCache: RewrittenPackageCache ) { let targetConfig; let params = node.params.slice(); @@ -24,6 +24,7 @@ export default function getConfig( } if (own) { + us = packageCache.original(us) || us; targetConfig = userConfigs[us.root]; } else { let packageName = params.shift(); @@ -31,6 +32,7 @@ export default function getConfig( throw new Error(`macroGetConfig requires at least one argument`); } let targetPkg = packageCache.resolve(packageName.value, us); + targetPkg = packageCache.original(targetPkg) || targetPkg; targetConfig = userConfigs[targetPkg.root]; } while (typeof targetConfig === 'object' && targetConfig && params.length > 0) { From 4f021875065cc4caa211194e8c92778df1fc6b26 Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Tue, 20 Jun 2023 17:21:57 -0400 Subject: [PATCH 62/72] test setup for jest macro tests needs appPackageRoot now --- packages/macros/tests/babel/eval.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/macros/tests/babel/eval.test.ts b/packages/macros/tests/babel/eval.test.ts index 5f746a5ad..c1d136ee0 100644 --- a/packages/macros/tests/babel/eval.test.ts +++ b/packages/macros/tests/babel/eval.test.ts @@ -5,6 +5,7 @@ import type * as Babel from '@babel/core'; import { types as t } from '@babel/core'; import 'code-equality-assertions/jest'; import State, { initState } from '../../src/babel/state'; +import { resolve } from 'path'; describe('evaluation', function () { allBabelVersions({ @@ -122,7 +123,7 @@ describe('hasRuntimeImplementation', function () { allBabelVersions({ babelConfig() { return { - plugins: [testRuntime], + plugins: [[testRuntime, { appPackageRoot: resolve(__dirname, '..', '..') }]], }; }, createTests(transform) { From 9e078f07d94eb1291875912637da4e802ec1339e Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Wed, 21 Jun 2023 15:03:08 -0400 Subject: [PATCH 63/72] introduce explicit API for locating our (new, smaller) working dir --- packages/compat/src/audit.ts | 12 +++++-- .../compat/src/babel-plugin-adjust-imports.ts | 4 +-- packages/compat/src/compat-addons.ts | 14 ++++++--- packages/compat/src/compat-app-builder.ts | 3 +- packages/compat/src/compat-app.ts | 18 ++++++----- packages/compat/src/resolver-transform.ts | 4 +-- packages/shared-internals/src/index.ts | 1 + .../src/rewritten-package-cache.ts | 3 +- packages/shared-internals/src/working-dir.ts | 31 +++++++++++++++++++ packages/webpack/src/ember-webpack.ts | 4 +-- 10 files changed, 72 insertions(+), 22 deletions(-) create mode 100644 packages/shared-internals/src/working-dir.ts diff --git a/packages/compat/src/audit.ts b/packages/compat/src/audit.ts index 16ec301dc..e245e2300 100644 --- a/packages/compat/src/audit.ts +++ b/packages/compat/src/audit.ts @@ -1,6 +1,14 @@ import { readFileSync, readJSONSync } from 'fs-extra'; import { dirname, join, resolve as resolvePath } from 'path'; -import { AppMeta, explicitRelative, hbsToJS, Resolver, ResolverOptions, RewrittenPackageCache } from '@embroider/core'; +import { + AppMeta, + explicitRelative, + hbsToJS, + locateEmbroiderWorkingDir, + Resolver, + ResolverOptions, + RewrittenPackageCache, +} from '@embroider/core'; import { Memoize } from 'typescript-memoize'; import chalk from 'chalk'; import jsdom from 'jsdom'; @@ -259,7 +267,7 @@ export class Audit { } private get resolverParams(): ResolverOptions { - return readJSONSync(join(this.originAppRoot, 'node_modules', '.embroider', 'resolver.json')); + return readJSONSync(join(locateEmbroiderWorkingDir(this.originAppRoot), 'resolver.json')); } private resolver = new Resolver(this.resolverParams); diff --git a/packages/compat/src/babel-plugin-adjust-imports.ts b/packages/compat/src/babel-plugin-adjust-imports.ts index 91d4b86fc..1c527d4d3 100644 --- a/packages/compat/src/babel-plugin-adjust-imports.ts +++ b/packages/compat/src/babel-plugin-adjust-imports.ts @@ -5,7 +5,7 @@ import type { types as t } from '@babel/core'; import { ImportUtil } from 'babel-import-util'; import { readJSONSync } from 'fs-extra'; import { CompatResolverOptions } from './resolver-transform'; -import { Package, packageName, Resolver, unrelativize } from '@embroider/core'; +import { locateEmbroiderWorkingDir, Package, packageName, Resolver, unrelativize } from '@embroider/core'; import { snippetToDasherizedName } from './dasherize-component-name'; import { ActivePackageRules, appTreeRulesDir, ComponentRules, ModuleRules, TemplateRules } from './dependency-rules'; @@ -43,7 +43,7 @@ export default function main(babel: typeof Babel) { return cached; } let resolverOptions: CompatResolverOptions = readJSONSync( - join(appRoot, 'node_modules', '.embroider', 'resolver.json') + join(locateEmbroiderWorkingDir(appRoot), 'resolver.json') ); let resolver = new Resolver(resolverOptions); cached = { diff --git a/packages/compat/src/compat-addons.ts b/packages/compat/src/compat-addons.ts index 3ed436927..8095bdafa 100644 --- a/packages/compat/src/compat-addons.ts +++ b/packages/compat/src/compat-addons.ts @@ -1,6 +1,6 @@ import { Node } from 'broccoli-node-api'; import { resolve } from 'path'; -import { RewrittenPackageCache, Stage, WaitForTrees } from '@embroider/core'; +import { locateEmbroiderWorkingDir, RewrittenPackageCache, Stage, WaitForTrees } from '@embroider/core'; import TreeSync from 'tree-sync'; import CompatApp from './compat-app'; import { convertLegacyAddons } from './standalone-addon-build'; @@ -34,7 +34,7 @@ export default class CompatAddons implements Stage { async ready(): Promise<{ outputPath: string }> { return { - outputPath: resolve(this.compatApp.root, 'node_modules', '.embroider', 'rewritten-app'), + outputPath: resolve(locateEmbroiderWorkingDir(this.compatApp.root), 'rewritten-app'), }; } @@ -47,9 +47,13 @@ export default class CompatAddons implements Stage { changedMap: Map ) { if (!this.treeSync) { - this.treeSync = new TreeSync(addons, resolve(this.inputPath, 'node_modules/.embroider/rewritten-packages'), { - ignore: ['**/node_modules'], - }); + this.treeSync = new TreeSync( + addons, + resolve(locateEmbroiderWorkingDir(this.compatApp.root), 'rewritten-packages'), + { + ignore: ['**/node_modules'], + } + ); } if ( diff --git a/packages/compat/src/compat-app-builder.ts b/packages/compat/src/compat-app-builder.ts index 9f5bb1eaf..eb15307f3 100644 --- a/packages/compat/src/compat-app-builder.ts +++ b/packages/compat/src/compat-app-builder.ts @@ -16,6 +16,7 @@ import { cacheBustingPluginVersion, cacheBustingPluginPath, Resolver, + locateEmbroiderWorkingDir, } from '@embroider/core'; import walkSync from 'walk-sync'; import { resolve as resolvePath, posix } from 'path'; @@ -1057,7 +1058,7 @@ export class CompatAppBuilder { } private addResolverConfig(config: CompatResolverOptions) { - outputJSONSync(join(this.origAppPackage.root, 'node_modules', '.embroider', 'resolver.json'), config); + outputJSONSync(join(locateEmbroiderWorkingDir(this.compatApp.root), 'resolver.json'), config); } private shouldSplitRoute(routeName: string) { diff --git a/packages/compat/src/compat-app.ts b/packages/compat/src/compat-app.ts index 5ac90c793..cb878bd34 100644 --- a/packages/compat/src/compat-app.ts +++ b/packages/compat/src/compat-app.ts @@ -1,5 +1,12 @@ import { Node as BroccoliNode } from 'broccoli-node-api'; -import { PackageCache, WaitForTrees, Stage, RewrittenPackageCache, Package } from '@embroider/core'; +import { + PackageCache, + WaitForTrees, + Stage, + RewrittenPackageCache, + Package, + locateEmbroiderWorkingDir, +} from '@embroider/core'; import Options, { optionsWithDefaults } from './options'; import { Memoize } from 'typescript-memoize'; import { sync as pkgUpSync } from 'pkg-up'; @@ -829,6 +836,7 @@ export default class CompatApp { private async instantiate(root: string, packageCache: RewrittenPackageCache, configTree: V1Config) { let origAppPkg = this.appPackage(); let movedAppPkg = packageCache.withRewrittenDeps(origAppPkg); + let workingDir = locateEmbroiderWorkingDir(this.root); return new CompatAppBuilder( root, origAppPkg, @@ -836,12 +844,8 @@ export default class CompatApp { this.options, this, configTree, - packageCache.get( - join(origAppPkg.root, 'node_modules', '.embroider', 'rewritten-packages', '@embroider', 'synthesized-vendor') - ), - packageCache.get( - join(origAppPkg.root, 'node_modules', '.embroider', 'rewritten-packages', '@embroider', 'synthesized-styles') - ) + packageCache.get(join(workingDir, 'rewritten-packages', '@embroider', 'synthesized-vendor')), + packageCache.get(join(workingDir, 'rewritten-packages', '@embroider', 'synthesized-styles')) ); } diff --git a/packages/compat/src/resolver-transform.ts b/packages/compat/src/resolver-transform.ts index e0bec2dfd..77ec23a3d 100644 --- a/packages/compat/src/resolver-transform.ts +++ b/packages/compat/src/resolver-transform.ts @@ -14,7 +14,7 @@ import assertNever from 'assert-never'; import { join, sep } from 'path'; import { readJSONSync } from 'fs-extra'; import { dasherize, snippetToDasherizedName } from './dasherize-component-name'; -import { ResolverOptions as CoreResolverOptions, Resolver } from '@embroider/core'; +import { ResolverOptions as CoreResolverOptions, Resolver, locateEmbroiderWorkingDir } from '@embroider/core'; import CompatOptions from './options'; import { AuditMessage, Loc } from './audit'; import { camelCase, mergeWith } from 'lodash'; @@ -861,7 +861,7 @@ class TemplateResolver implements ASTPlugin { // This is the AST transform that resolves components, helpers and modifiers at build time export default function makeResolverTransform({ appRoot }: Options) { - let config: CompatResolverOptions = readJSONSync(join(appRoot, 'node_modules', '.embroider', 'resolver.json')); + let config: CompatResolverOptions = readJSONSync(join(locateEmbroiderWorkingDir(appRoot), 'resolver.json')); const resolverTransform: ASTPluginBuilder = env => { if (env.strictMode) { return { diff --git a/packages/shared-internals/src/index.ts b/packages/shared-internals/src/index.ts index 1f4a81aee..e1e33a281 100644 --- a/packages/shared-internals/src/index.ts +++ b/packages/shared-internals/src/index.ts @@ -22,3 +22,4 @@ export { pluginPath as cacheBustingPluginPath, version as cacheBustingPluginVersion, } from './babel-plugin-cache-busting'; +export { locateEmbroiderWorkingDir } from './working-dir'; diff --git a/packages/shared-internals/src/rewritten-package-cache.ts b/packages/shared-internals/src/rewritten-package-cache.ts index f53cea9fe..beedf8a71 100644 --- a/packages/shared-internals/src/rewritten-package-cache.ts +++ b/packages/shared-internals/src/rewritten-package-cache.ts @@ -3,6 +3,7 @@ import Package from './package'; import { existsSync, readJSONSync } from 'fs-extra'; import { resolve } from 'path'; import { getOrCreate } from './get-or-create'; +import { locateEmbroiderWorkingDir } from './working-dir'; export interface RewrittenPackageIndex { // keys are paths to original package root directories. @@ -136,7 +137,7 @@ export class RewrittenPackageCache implements PackageCacheTheGoodParts { } private loadIndex(): RewrittenPackageCache['index'] { - let addonsDir = resolve(this.appRoot, 'node_modules', '.embroider', 'rewritten-packages'); + let addonsDir = resolve(locateEmbroiderWorkingDir(this.appRoot), 'rewritten-packages'); let indexFile = resolve(addonsDir, 'index.json'); if (!existsSync(indexFile)) { return { diff --git a/packages/shared-internals/src/working-dir.ts b/packages/shared-internals/src/working-dir.ts new file mode 100644 index 000000000..b17bdd858 --- /dev/null +++ b/packages/shared-internals/src/working-dir.ts @@ -0,0 +1,31 @@ +import { resolve } from 'path'; +import { existsSync } from 'fs'; +import { readJSONSync } from 'fs-extra'; + +const cache = new Map(); + +// Most of this only exists because of classic dummy apps being weird. +export function locateEmbroiderWorkingDir(appRoot: string): string { + if (cache.has(appRoot)) { + return cache.get(appRoot); + } + if (existsSync(resolve(appRoot, 'package.json'))) { + // the normal case + let path = resolve(appRoot, 'node_modules', '.embroider'); + cache.set(appRoot, path); + return path; + } else { + // probably in a dummy app (sigh), but let's do a little checking to + // distinguish that case from someone pointing embroider at a nonsense + // location + if (existsSync(resolve(appRoot, '..', '..', 'package.json'))) { + let pkg = readJSONSync(resolve(appRoot, '..', '..', 'package.json')); + if (pkg.keywords?.includes('ember-addon')) { + let path = resolve(appRoot, '..', '..', 'node_modules', '.embroider'); + cache.set(appRoot, path); + return path; + } + } + throw new Error('unable to locate app'); + } +} diff --git a/packages/webpack/src/ember-webpack.ts b/packages/webpack/src/ember-webpack.ts index f7eca8c55..e93ec815b 100644 --- a/packages/webpack/src/ember-webpack.ts +++ b/packages/webpack/src/ember-webpack.ts @@ -21,7 +21,7 @@ import { getOrCreate, ResolverOptions, } from '@embroider/core'; -import { RewrittenPackageCache, tmpdir } from '@embroider/shared-internals'; +import { locateEmbroiderWorkingDir, RewrittenPackageCache, tmpdir } from '@embroider/shared-internals'; import webpack, { Configuration, RuleSetUseItem, WebpackPluginInstance } from 'webpack'; import { readFileSync, outputFileSync, copySync, Stats, statSync, readJSONSync } from 'fs-extra'; import { join, dirname, relative, sep } from 'path'; @@ -182,7 +182,7 @@ const Webpack: PackagerConstructor = class Webpack implements Packager } let resolverConfig: EmbroiderPluginOptions = readJSONSync( - join(this.appRoot, 'node_modules/.embroider/resolver.json') + join(locateEmbroiderWorkingDir(this.appRoot), 'resolver.json') ); return { entrypoints, otherAssets, babel, rootURL, resolverConfig, publicAssetURL, packageName: meta.name }; From 81abf212741d0922227227413c779ac8284ad94c Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Wed, 21 Jun 2023 15:09:21 -0400 Subject: [PATCH 64/72] cleaning up packageCache api --- packages/shared-internals/src/package-cache.ts | 15 +++------------ packages/shared-internals/src/package.ts | 4 ++-- .../src/rewritten-package-cache.ts | 6 +----- 3 files changed, 6 insertions(+), 19 deletions(-) diff --git a/packages/shared-internals/src/package-cache.ts b/packages/shared-internals/src/package-cache.ts index 085bc1677..c679cd929 100644 --- a/packages/shared-internals/src/package-cache.ts +++ b/packages/shared-internals/src/package-cache.ts @@ -11,7 +11,7 @@ export default class PackageCache { let cache = getOrCreate(this.resolutionCache, fromPackage, () => new Map() as Map); let result = getOrCreate(cache, packageName, () => { // the type cast is needed because resolvePackagePath itself is erroneously typed as `any`. - let packagePath = resolvePackagePath(packageName, this.basedir(fromPackage)) as string | null; + let packagePath = resolvePackagePath(packageName, fromPackage.root) as string | null; if (!packagePath) { // this gets our null into the cache so we don't keep trying to resolve // a thing that is not found @@ -27,12 +27,8 @@ export default class PackageCache { return result; } - protected rootCache: Map = new Map(); - protected resolutionCache: Map> = new Map(); - - basedir(pkg: Package): string { - return pkg.root; - } + private rootCache: Map = new Map(); + private resolutionCache: Map> = new Map(); get(packageRoot: string) { let root = realpathSync(packageRoot); @@ -65,11 +61,6 @@ export default class PackageCache { } } - // register to be shared as the per-process package cache with the given name - shareAs(identifier: string) { - shared.set(identifier, this); - } - static shared(identifier: string, appRoot: string) { let pk = getOrCreate(shared, identifier + appRoot, () => new PackageCache(appRoot)); diff --git a/packages/shared-internals/src/package.ts b/packages/shared-internals/src/package.ts index 775c0a722..ed463a6dc 100644 --- a/packages/shared-internals/src/package.ts +++ b/packages/shared-internals/src/package.ts @@ -137,7 +137,7 @@ export default class Package { // stop you. let pkg, main; try { - pkg = this.packageCache.get(join(this.packageCache.basedir(this), path)); + pkg = this.packageCache.get(join(this.root, path)); main = pkg.packageJSON['ember-addon']?.main || pkg.packageJSON['main']; } catch (err) { // package was missing or had invalid package.json @@ -150,7 +150,7 @@ export default class Package { main = `${main}.js`; } - let mainPath = join(this.packageCache.basedir(this), path, main); + let mainPath = join(this.root, path, main); if (!existsSync(mainPath)) { // package has no valid main return false; diff --git a/packages/shared-internals/src/rewritten-package-cache.ts b/packages/shared-internals/src/rewritten-package-cache.ts index beedf8a71..b014c8653 100644 --- a/packages/shared-internals/src/rewritten-package-cache.ts +++ b/packages/shared-internals/src/rewritten-package-cache.ts @@ -29,11 +29,7 @@ export interface RewrittenPackageIndex { // could see all of those) type PublicAPI = { [K in keyof T]: T[K] }; -// TODO: as our refactor lands we should be able to remove these things from -// PackageCache itself. -type PackageCacheTheGoodParts = Omit, 'basedir' | 'seed' | 'shareAs'>; - -export class RewrittenPackageCache implements PackageCacheTheGoodParts { +export class RewrittenPackageCache implements PublicAPI { constructor(private plainCache: PackageCache) {} get appRoot(): string { From cff8d4366e26f2965c14e7bd77638c9d075629c2 Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Wed, 21 Jun 2023 22:34:25 -0400 Subject: [PATCH 65/72] bugfixes that address failures in macro-sample-addon --- packages/compat/src/compat-app.ts | 4 +++- packages/macros/src/babel/evaluate-json.ts | 2 +- packages/macros/src/babel/get-config.ts | 10 +++------- packages/macros/src/babel/state.ts | 7 +++++++ 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/packages/compat/src/compat-app.ts b/packages/compat/src/compat-app.ts index cb878bd34..765527696 100644 --- a/packages/compat/src/compat-app.ts +++ b/packages/compat/src/compat-app.ts @@ -821,7 +821,9 @@ export default class CompatApp { @Memoize() appPackage(): Package { - let packageCache = RewrittenPackageCache.shared('embroider', this.root); + // this is deliberately not RewrittenPackageCache, because it's supposed to + // be the original copy of the app with all the original dependencies. + let packageCache = PackageCache.shared('embroider', this.root); if (this.isDummy) { return new DummyPackage( this.root, diff --git a/packages/macros/src/babel/evaluate-json.ts b/packages/macros/src/babel/evaluate-json.ts index 60555713f..255d6cccd 100644 --- a/packages/macros/src/babel/evaluate-json.ts +++ b/packages/macros/src/babel/evaluate-json.ts @@ -409,7 +409,7 @@ export class Evaluator { if (callee.referencesImport('@embroider/macros', 'isDevelopingThisPackage')) { return { confident: true, - value: this.state.opts.isDevelopingPackageRoots.includes(this.state.owningPackage().root), + value: this.state.opts.isDevelopingPackageRoots.includes(this.state.originalOwningPackage().root), hasRuntimeImplementation: false, }; } diff --git a/packages/macros/src/babel/get-config.ts b/packages/macros/src/babel/get-config.ts index b7d626613..733b47de4 100644 --- a/packages/macros/src/babel/get-config.ts +++ b/packages/macros/src/babel/get-config.ts @@ -28,7 +28,7 @@ function getPackage(path: NodePath, state: State, mode: 'own' } else { assertNever(mode); } - return targetPackage(state.sourceFile, packageName, state.packageCache); + return targetPackage(state.originalOwningPackage(), packageName, state.packageCache); } // this evaluates to the actual value of the config. It can be used directly by the Evaluator. @@ -75,16 +75,12 @@ export function insertConfig(path: NodePath, state: State, mod } function targetPackage( - fromPath: string, + us: Package, packageName: string | undefined, packageCache: RewrittenPackageCache ): Package | null { - let us = packageCache.ownerOfFile(fromPath); - if (!us) { - throw new Error(`unable to determine which npm package owns the file ${fromPath}`); - } if (!packageName) { - return packageCache.original(us) || us; + return us; } try { let target = packageCache.resolve(packageName, us); diff --git a/packages/macros/src/babel/state.ts b/packages/macros/src/babel/state.ts index 04a49d0e9..184c524e0 100644 --- a/packages/macros/src/babel/state.ts +++ b/packages/macros/src/babel/state.ts @@ -16,6 +16,7 @@ export default interface State { sourceFile: string; pathToOurAddon(moduleName: string): string; owningPackage(): Package; + originalOwningPackage(): Package; cloneDeep(node: Node): Node; opts: { @@ -59,6 +60,7 @@ export function initState(t: typeof Babel.types, path: NodePath Date: Wed, 21 Jun 2023 22:57:32 -0400 Subject: [PATCH 66/72] updating dummy app tests now that we put the working dir in the right place --- tests/scenarios/compat-dummy-app-test.ts | 4 ++-- tests/scenarios/compat-stage2-test.ts | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/scenarios/compat-dummy-app-test.ts b/tests/scenarios/compat-dummy-app-test.ts index 1c02fb7b4..fc30e7c25 100644 --- a/tests/scenarios/compat-dummy-app-test.ts +++ b/tests/scenarios/compat-dummy-app-test.ts @@ -65,8 +65,8 @@ dummyAppScenarios // "robots.txt" here because it thinks that file belongs to the // containing addon. By writing out the rewritten paths ourselves we // sidestep that problem≥ - expectFile('./node_modules/.embroider/rewritten-app/robots.txt').exists(); - expectFile('./node_modules/.embroider/rewritten-app/package.json') + expectFile('../../node_modules/.embroider/rewritten-app/robots.txt').exists(); + expectFile('../../node_modules/.embroider/rewritten-app/package.json') .json() .get('ember-addon.assets') .includes('robots.txt'); diff --git a/tests/scenarios/compat-stage2-test.ts b/tests/scenarios/compat-stage2-test.ts index 2e82e8037..e04a4fcff 100644 --- a/tests/scenarios/compat-stage2-test.ts +++ b/tests/scenarios/compat-stage2-test.ts @@ -759,9 +759,9 @@ dummyAppScenarios }); test('dummy app sees that its being developed', function () { - let assertFile = expectFile('node_modules/.embroider/rewritten-app/components/inside-dummy-app.js').transform( - build.transpile - ); + let assertFile = expectFile( + '../../node_modules/.embroider/rewritten-app/components/inside-dummy-app.js' + ).transform(build.transpile); assertFile.matches(/console\.log\(true\)/); }); From 346ad95b27a224afa32cdc1884ad884134e5f6a1 Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Wed, 21 Jun 2023 23:49:32 -0400 Subject: [PATCH 67/72] updating some more macro tests so they can find the app --- packages/macros/tests/babel/env-macros.test.ts | 2 +- packages/macros/tests/glimmer/helpers.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/macros/tests/babel/env-macros.test.ts b/packages/macros/tests/babel/env-macros.test.ts index 3ddca24af..90816a330 100644 --- a/packages/macros/tests/babel/env-macros.test.ts +++ b/packages/macros/tests/babel/env-macros.test.ts @@ -121,7 +121,7 @@ describe(`env macros`, function () { describe(`false cases`, function () { beforeEach(function () { - macrosConfig = MacrosConfig.for({}, '/nonexistent'); + macrosConfig = MacrosConfig.for({}, resolve(__dirname, '..', '..')); macrosConfig.setGlobalConfig(__filename, '@embroider/macros', { isTesting: false }); applyMode(macrosConfig); macrosConfig.finalize(); diff --git a/packages/macros/tests/glimmer/helpers.ts b/packages/macros/tests/glimmer/helpers.ts index 0a3bd741f..85ee5ae11 100644 --- a/packages/macros/tests/glimmer/helpers.ts +++ b/packages/macros/tests/glimmer/helpers.ts @@ -1,7 +1,7 @@ import { emberTemplateCompiler } from '@embroider/test-support'; import { Project } from 'scenario-tester'; import { MacrosConfig } from '../../src/node'; -import { join } from 'path'; +import { join, resolve } from 'path'; import { hbsToJS } from '@embroider/shared-internals'; import { transformSync } from '@babel/core'; import { Options as EtcOptions, Transform } from 'babel-plugin-ember-template-compilation'; @@ -19,7 +19,7 @@ export interface TemplateTransformOptions { export function templateTests(createTests: CreateTestsWithConfig | CreateTests) { let { plugins, setConfig } = MacrosConfig.transforms(); - let config = MacrosConfig.for({}, '/nonexistent'); + let config = MacrosConfig.for({}, resolve(__dirname, '..', '..')); setConfig(config); let transform = (templateContents: string, options: TemplateTransformOptions = {}) => { From c64ae91a285ea53ff95d6a9ebe327de1ea681f6c Mon Sep 17 00:00:00 2001 From: Chris Manson Date: Thu, 22 Jun 2023 12:03:58 +0100 Subject: [PATCH 68/72] temporarily skip engines tests to get CI green --- tests/scenarios/engines-test.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/scenarios/engines-test.ts b/tests/scenarios/engines-test.ts index 3b99c16f2..32dea7e67 100644 --- a/tests/scenarios/engines-test.ts +++ b/tests/scenarios/engines-test.ts @@ -79,6 +79,9 @@ let engineScenarios = appScenarios.map('engines', project => { }); engineScenarios + .skip('lts_3_28-engines') // this skip should be removed before https://github.com/embroider-build/embroider/pull/1435 is merged + .skip('lts_4_4-engines') // this skip should be removed before https://github.com/embroider-build/embroider/pull/1435 is merged + .skip('release-engines') // this skip should be removed before https://github.com/embroider-build/embroider/pull/1435 is merged .map('without-fastboot', () => {}) .forEachScenario(scenario => { Qmodule(scenario.name, function (hooks) { From 69a9516eaf931b5ca48bcadb544511f7f5d38503 Mon Sep 17 00:00:00 2001 From: Chris Manson Date: Thu, 22 Jun 2023 13:29:01 +0100 Subject: [PATCH 69/72] remove FS writing from app-differ --- packages/compat/src/compat-app-builder.ts | 36 ++++-------- packages/core/src/app-differ.ts | 72 ++--------------------- 2 files changed, 15 insertions(+), 93 deletions(-) diff --git a/packages/compat/src/compat-app-builder.ts b/packages/compat/src/compat-app-builder.ts index eb15307f3..be6ed8f9a 100644 --- a/packages/compat/src/compat-app-builder.ts +++ b/packages/compat/src/compat-app-builder.ts @@ -56,6 +56,7 @@ import SourceMapConcat from 'fast-sourcemap-concat'; import escapeRegExp from 'escape-string-regexp'; import type CompatApp from './compat-app'; +import TreeSync from 'tree-sync'; // This exists during the actual broccoli build step. As opposed to CompatApp, // which also exists during pipeline-construction time. @@ -63,6 +64,7 @@ import type CompatApp from './compat-app'; export class CompatAppBuilder { // for each relativePath, an Asset we have already emitted private assets: Map = new Map(); + private treeSync: TreeSync | undefined; constructor( private root: string, @@ -405,22 +407,6 @@ export class CompatAppBuilder { return result; } - // unlike our full config, this one just needs to know how to parse all the - // syntax our app can contain. - @Memoize() - private babelParserConfig(): TransformOptions { - let babel = cloneDeep(this.compatApp.babelConfig()); - - if (!babel.plugins) { - babel.plugins = []; - } - - // Our stage3 code is always allowed to use dynamic import. We may emit it - // ourself when splitting routes. - babel.plugins.push(require.resolve('@babel/plugin-syntax-dynamic-import')); - return babel; - } - @Memoize() private babelConfig(resolverConfig: CompatResolverOptions) { let babel = cloneDeep(this.compatApp.babelConfig()); @@ -721,16 +707,9 @@ export class CompatAppBuilder { this.appDiffers = engines.map(engine => { let differ: AppDiffer; if (this.activeFastboot) { - differ = new AppDiffer( - engine.destPath, - engine.sourcePath, - [...engine.addons], - true, - this.fastbootJSSrcDir(), - this.babelParserConfig() - ); + differ = new AppDiffer(engine.sourcePath, [...engine.addons], true, this.fastbootJSSrcDir()); } else { - differ = new AppDiffer(engine.destPath, engine.sourcePath, [...engine.addons]); + differ = new AppDiffer(engine.sourcePath, [...engine.addons]); } return { differ, @@ -745,6 +724,13 @@ export class CompatAppBuilder { .slice() .reverse() .forEach(a => a.differ.update()); + + if (!this.treeSync) { + this.treeSync = new TreeSync(appJSPath, this.root); + } + + this.treeSync.sync(); + return this.appDiffers.map(a => { return { ...a.engine, diff --git a/packages/core/src/app-differ.ts b/packages/core/src/app-differ.ts index eaf1d2eb0..8f5c8f64b 100644 --- a/packages/core/src/app-differ.ts +++ b/packages/core/src/app-differ.ts @@ -1,13 +1,9 @@ import { AddonPackage } from '@embroider/shared-internals'; import MultiTreeDiff, { InputTree } from './multi-tree-diff'; import walkSync from 'walk-sync'; -import { join, basename, dirname, resolve } from 'path'; -import { mkdirpSync, unlinkSync, rmdirSync, removeSync, copySync, writeFileSync, readFileSync } from 'fs-extra'; +import { resolve } from 'path'; import { debug } from './messages'; import assertNever from 'assert-never'; -import { describeExports } from './describe-exports'; -import { compile } from './js-handlebars'; -import { TransformOptions } from '@babel/core'; import { statSync } from 'fs'; import { format } from 'util'; @@ -23,14 +19,12 @@ export default class AppDiffer { isFastbootOnly: Map = new Map(); constructor( - private outputPath: string, ownAppJSDir: string, activeAddonDescendants: AddonPackage[], // arguments below this point are only needed in fastboot mode. Fastboot // makes this pretty messy because fastboot trees all merge into the app 🤮. fastbootEnabled = false, - ownFastbootJSDir?: string | undefined, - private babelParserConfig?: TransformOptions | undefined + ownFastbootJSDir?: string | undefined ) { this.sources = activeAddonDescendants.map(addon => maybeSource(addon, 'app-js')).filter(Boolean) as Source[]; @@ -76,21 +70,15 @@ export default class AppDiffer { let { ops, sources } = this.differ.update(); debug(`app-differ operations count: %s`, ops.length); for (let [operation, relativePath] of ops) { - let outputPath = join(this.outputPath, relativePath); switch (operation) { case 'unlink': - unlinkSync(outputPath); this.files.delete(relativePath); break; case 'rmdir': - rmdirSync(outputPath); - break; case 'mkdir': - mkdirpSync(outputPath); - break; case 'change': - removeSync(outputPath); - // deliberate fallthrough + // this used to actually write to disk but we have moved the functionality into the resolver code + break; case 'create': let sourceIndices = sources.get(relativePath)!; if (sourceIndices.length === 1) { @@ -100,33 +88,12 @@ export default class AppDiffer { // trying to import it anyway, because that would have already been // an error pre-embroider). this.isFastbootOnly.set(relativePath, sourceIndices[0] >= this.firstFastbootTree); - let source = this.sources[sourceIndices[0]]; - let sourceFile = source.locate(relativePath); - if (!source.isRelocated) { - copySync(sourceFile, outputPath, { dereference: true }); - } this.updateFiles(relativePath); } else { // we have both fastboot and non-fastboot files for this path. // Because of the way fastbootMerge is written, the first one is the // non-fastboot. this.isFastbootOnly.set(relativePath, false); - let [browserSrc, fastbootSrc] = sourceIndices.map(i => this.sources[i]); - let [browserSourceFile, fastbootSourceFile] = [browserSrc, fastbootSrc].map(src => - src.locate(relativePath) - ); - let dir = dirname(relativePath); - let base = basename(relativePath); - let browserDest = `_browser_${base}`; - let fastbootDest = `_fastboot_${base}`; - if (!browserSrc.isRelocated && !fastbootSrc.isRelocated) { - copySync(browserSourceFile, join(this.outputPath, dir, browserDest), { dereference: true }); - copySync(fastbootSourceFile, join(this.outputPath, dir, fastbootDest), { dereference: true }); - writeFileSync( - outputPath, - switcher(browserDest, fastbootDest, this.babelParserConfig!, readFileSync(browserSourceFile, 'utf8')) - ); - } this.updateFiles(relativePath); } break; @@ -167,37 +134,6 @@ function fastbootMerge(firstFastbootTree: number) { }; } -const switcherTemplate = compile(` -import { macroCondition, getGlobalConfig, importSync } from '@embroider/macros'; -let mod; -if (macroCondition(getGlobalConfig().fastboot?.isRunning)){ - mod = importSync("./{{js-string-escape fastbootDest}}"); -} else { - mod = importSync("./{{js-string-escape browserDest}}"); -} -{{#if hasDefaultExport}} -export default mod.default; -{{/if}} -{{#each names as |name|}} -export const {{name}} = mod.{{name}}; -{{/each}} -`) as (params: { fastbootDest: string; browserDest: string; names: string[]; hasDefaultExport: boolean }) => string; - -function switcher( - browserDest: string, - fastbootDest: string, - babelParserConfig: TransformOptions, - browserSource: string -): string { - let { names } = describeExports(browserSource, babelParserConfig); - return switcherTemplate({ - fastbootDest, - browserDest, - names: [...names].filter(name => name !== 'default'), - hasDefaultExport: names.has('default'), - }); -} - interface Source extends InputTree { // find the real on disk location of the file that is presented externally as // `relativePath` From da744d9ea68da0dfa4b21ed29e3e2d1e4bbee984 Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Tue, 27 Jun 2023 11:03:15 -0400 Subject: [PATCH 70/72] implementing engine-level fastboot overrides in resolver --- packages/compat/src/compat-app-builder.ts | 1 + packages/compat/tests/audit.test.ts | 1 + packages/core/src/module-resolver.ts | 110 +++++++++++++++++----- tests/scenarios/compat-resolver-test.ts | 1 + tests/scenarios/core-resolver-test.ts | 67 +++++++++++++ 5 files changed, 158 insertions(+), 22 deletions(-) diff --git a/packages/compat/src/compat-app-builder.ts b/packages/compat/src/compat-app-builder.ts index be6ed8f9a..6f3bcca77 100644 --- a/packages/compat/src/compat-app-builder.ts +++ b/packages/compat/src/compat-app-builder.ts @@ -277,6 +277,7 @@ export class CompatAppBuilder { engines: engines.map((engine, index) => ({ packageName: engine.package.name, root: index === 0 ? this.root : engine.package.root, // first engine is the app, which has been relocated to this.roto + fastbootFiles: {}, activeAddons: [...engine.addons] .map(a => ({ name: a.name, diff --git a/packages/compat/tests/audit.test.ts b/packages/compat/tests/audit.test.ts index c5c3030e8..ecebfe5a4 100644 --- a/packages/compat/tests/audit.test.ts +++ b/packages/compat/tests/audit.test.ts @@ -43,6 +43,7 @@ describe('audit', function () { engines: [ { packageName: 'audit-this-app', + fastbootFiles: {}, activeAddons: [], root: app.baseDir, }, diff --git a/packages/core/src/module-resolver.ts b/packages/core/src/module-resolver.ts index 73131fc97..9d62ef551 100644 --- a/packages/core/src/module-resolver.ts +++ b/packages/core/src/module-resolver.ts @@ -3,6 +3,7 @@ import { emberVirtualPeerDeps, extensionsPattern, packageName as getPackageName, + packageName, } from '@embroider/shared-internals'; import { dirname, resolve } from 'path'; import { Package, V2Package, explicitRelative, RewrittenPackageCache } from '@embroider/shared-internals'; @@ -65,6 +66,7 @@ export interface Options { interface EngineConfig { packageName: string; activeAddons: { name: string; root: string }[]; + fastbootFiles: { [appName: string]: { localFilename: string; shadowedFilename: string | undefined } }; root: string; } @@ -156,6 +158,8 @@ export class Resolver { request = this.handleFastbootCompat(request); request = this.handleGlobalsCompat(request); request = this.handleRenaming(request); + // we expect the specifier to be app relative at this point - must be after handleRenaming + request = this.handleAppFastboot(request); request = this.preHandleExternal(request); // this should probably stay the last step in beforeResolve, because it can @@ -295,6 +299,46 @@ export class Resolver { return owningPackage; } + private handleAppFastboot(request: R): R { + let pkg = this.owningPackage(request.fromFile); + + if (!pkg) { + return request; + } + + if (packageName(request.specifier)) { + // not a relative request, and we're assuming all within-engine requests + // are relative by this point due to `v1 self-import` which happens + // earlier + return request; + } + + let engineConfig = this.engineConfig(pkg.name); + if (engineConfig) { + for (let candidate of this.withResolvableExtensions(request.specifier)) { + let fastbootFile = engineConfig.fastbootFiles[candidate]; + if (fastbootFile) { + if (fastbootFile.shadowedFilename) { + let { names } = describeExports(readFileSync(resolve(pkg.root, fastbootFile.shadowedFilename), 'utf8'), {}); + return logTransition( + 'shadowed app fastboot', + request, + request.virtualize(fastbootSwitch(candidate, resolve(pkg.root, 'package.json'), names)) + ); + } else { + return logTransition( + 'unshadowed app fastboot', + request, + request.alias(fastbootFile.localFilename).rehome(resolve(pkg.root, 'package.json')) + ); + } + } + } + } + + return request; + } + private handleFastbootCompat(request: R): R { let match = decodeFastbootSwitch(request.fromFile); if (!match) { @@ -315,7 +359,22 @@ export class Resolver { let pkg = this.owningPackage(match.filename); if (pkg) { let rel = explicitRelative(pkg.root, match.filename); - let entry = this.getEntryFromMergeMap(rel, pkg.root); + + let engineConfig = this.engineConfig(pkg.name); + if (engineConfig) { + let fastbootFile = engineConfig.fastbootFiles[rel]; + if (fastbootFile && fastbootFile.shadowedFilename) { + let targetFile: string; + if (section === 'app-js') { + targetFile = fastbootFile.shadowedFilename; + } else { + targetFile = fastbootFile.localFilename; + } + return request.alias(targetFile).rehome(resolve(pkg.root, 'package.json')); + } + } + + let entry = this.getEntryFromMergeMap(rel, pkg.root)?.entry; if (entry?.type === 'both') { return request.alias(entry[section].localPath).rehome(resolve(entry[section].packageRoot, 'package.json')); } @@ -951,22 +1010,27 @@ export class Resolver { return logTransition('fallbackResolve final exit', request); } - private getEntryFromMergeMap(inEngineSpecifier: string, root: string): MergeEntry | undefined { + private getEntryFromMergeMap( + inEngineSpecifier: string, + root: string + ): { entry: MergeEntry; matched: string } | undefined { let entry: MergeEntry | undefined; + for (let candidate of this.withResolvableExtensions(inEngineSpecifier)) { + entry = this.mergeMap.get(root)?.get(candidate); + if (entry) { + return { entry, matched: candidate }; + } + } + } - if (inEngineSpecifier.match(/\.(hbs|js|hbs\.js)$/)) { - entry = this.mergeMap.get(root)?.get(inEngineSpecifier); + private *withResolvableExtensions(filename: string): Generator { + if (filename.match(/\.(hbs|js|hbs\.js)$/)) { + yield filename; } else { - // try looking up .hbs .js and .hbs.js in that order for specifiers without extenstions - ['.hbs', '.js', '.hbs.js'].forEach(ext => { - if (entry) { - return; - } - - entry = this.mergeMap.get(root)?.get(`${inEngineSpecifier}${ext}`); - }); + for (let ext of ['.hbs', '.js', '.hbs.js']) { + yield `${filename}${ext}`; + } } - return entry; } private searchAppTree( @@ -974,29 +1038,31 @@ export class Resolver { engine: EngineConfig, inEngineSpecifier: string ): R | undefined { - let entry = this.getEntryFromMergeMap(inEngineSpecifier, engine.root); + let matched = this.getEntryFromMergeMap(inEngineSpecifier, engine.root); - switch (entry?.type) { + switch (matched?.entry.type) { case undefined: return undefined; case 'app-only': - return request.alias(entry['app-js'].localPath).rehome(resolve(entry['app-js'].packageRoot, 'package.json')); + return request + .alias(matched.entry['app-js'].localPath) + .rehome(resolve(matched.entry['app-js'].packageRoot, 'package.json')); case 'fastboot-only': return request - .alias(entry['fastboot-js'].localPath) - .rehome(resolve(entry['fastboot-js'].packageRoot, 'package.json')); + .alias(matched.entry['fastboot-js'].localPath) + .rehome(resolve(matched.entry['fastboot-js'].packageRoot, 'package.json')); case 'both': let foundAppJS = this.nodeResolve( - entry['app-js'].localPath, - resolve(entry['app-js'].packageRoot, 'package.json') + matched.entry['app-js'].localPath, + resolve(matched.entry['app-js'].packageRoot, 'package.json') ); if (foundAppJS.type !== 'real') { throw new Error( - `${entry['app-js'].fromPackageName} declared ${inEngineSpecifier} in packageJSON.ember-addon.app-js, but that module does not exist` + `${matched.entry['app-js'].fromPackageName} declared ${inEngineSpecifier} in packageJSON.ember-addon.app-js, but that module does not exist` ); } let { names } = describeExports(readFileSync(foundAppJS.filename, 'utf8'), {}); - return request.virtualize(fastbootSwitch(request.specifier, request.fromFile, names)); + return request.virtualize(fastbootSwitch(matched.matched, resolve(engine.root, 'package.json'), names)); } } diff --git a/tests/scenarios/compat-resolver-test.ts b/tests/scenarios/compat-resolver-test.ts index f2390f275..5c18ef346 100644 --- a/tests/scenarios/compat-resolver-test.ts +++ b/tests/scenarios/compat-resolver-test.ts @@ -79,6 +79,7 @@ Scenarios.fromProject(() => new Project()) packageName: 'my-app', root: app.dir, activeAddons: [], + fastbootFiles: {}, }, ], modulePrefix: 'my-app', diff --git a/tests/scenarios/core-resolver-test.ts b/tests/scenarios/core-resolver-test.ts index 96bff154c..9f3646a46 100644 --- a/tests/scenarios/core-resolver-test.ts +++ b/tests/scenarios/core-resolver-test.ts @@ -52,6 +52,7 @@ Scenarios.fromProject(() => new Project()) podModulePrefix?: string; renamePackages?: Record; addonMeta?: Partial; + fastbootFiles?: { [appName: string]: { localFilename: string; shadowedFilename: string | undefined } }; } let configure: (opts?: ConfigureOpts) => Promise; @@ -92,6 +93,7 @@ Scenarios.fromProject(() => new Project()) { packageName: 'my-app', root: app.dir, + fastbootFiles: opts?.fastbootFiles ?? {}, activeAddons: [ { name: 'my-addon', @@ -569,6 +571,21 @@ Scenarios.fromProject(() => new Project()) .to('./node_modules/my-addon/_fastboot_/hello-world.js'); }); + test(`resolves app fastboot-js`, async function () { + givenFiles({ + './fastboot/hello-world.js': ``, + 'app.js': `import "my-app/hello-world"`, + }); + + await configure({ + fastbootFiles: { + './hello-world.js': { localFilename: './fastboot/hello-world.js', shadowedFilename: undefined }, + }, + }); + + expectAudit.module('./app.js').resolves('my-app/hello-world').to('./fastboot/hello-world.js'); + }); + test(`file exists in both app-js and fastboot-js`, async function () { givenFiles({ 'node_modules/my-addon/_fastboot_/hello-world.js': ` @@ -616,6 +633,56 @@ Scenarios.fromProject(() => new Project()) switcherModule.resolves('./fastboot').to('./node_modules/my-addon/_fastboot_/hello-world.js'); switcherModule.resolves('./browser').to('./node_modules/my-addon/_app_/hello-world.js'); }); + + test(`app and fastboot file exists`, async function () { + givenFiles({ + 'fastboot/hello-world.js': ` + export function hello() { return 'fastboot'; } + export class Bonjour {} + export default function() {} + const value = 1; + export { value }; + export const x = 2; + `, + 'app/hello-world.js': ` + export function hello() { return 'browser'; } + export class Bonjour {} + export default function() {} + const value = 1; + export { value }; + export const x = 2; + `, + 'app.js': `import "my-app/hello-world"`, + }); + + await configure({ + fastbootFiles: { + './hello-world.js': { + localFilename: './fastboot/hello-world.js', + shadowedFilename: './app/hello-world.js', + }, + }, + }); + + let switcherModule = expectAudit.module('./app.js').resolves('my-app/hello-world').toModule(); + switcherModule.codeEquals(` + import { macroCondition, getGlobalConfig, importSync } from '@embroider/macros'; + let mod; + if (macroCondition(getGlobalConfig().fastboot?.isRunning)) { + mod = importSync("./fastboot"); + } else { + mod = importSync("./browser"); + } + export default mod.default; + export const hello = mod.hello; + export const Bonjour = mod.Bonjour; + export const value = mod.value; + export const x = mod.x; + `); + + switcherModule.resolves('./fastboot').to('./fastboot/hello-world.js'); + switcherModule.resolves('./browser').to('./app/hello-world.js'); + }); }); Qmodule('legacy-addons', function () { From 4a22433d18d5cd7ed7d7ac1e137955502cd87fdc Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Tue, 27 Jun 2023 19:06:34 -0400 Subject: [PATCH 71/72] finished moving app's fastboot support into module-resolver --- packages/compat/src/compat-app-builder.ts | 63 ++++++++++++++++++++--- packages/core/src/app-files.ts | 1 + packages/core/src/module-resolver.ts | 42 ++++++++++----- 3 files changed, 85 insertions(+), 21 deletions(-) diff --git a/packages/compat/src/compat-app-builder.ts b/packages/compat/src/compat-app-builder.ts index 6f3bcca77..9dc1a4995 100644 --- a/packages/compat/src/compat-app-builder.ts +++ b/packages/compat/src/compat-app-builder.ts @@ -32,7 +32,7 @@ import mergeWith from 'lodash/mergeWith'; import cloneDeep from 'lodash/cloneDeep'; import { sync as resolveSync } from 'resolve'; import bind from 'bind-decorator'; -import { outputJSONSync, readJSONSync, statSync, unlinkSync, writeFileSync } from 'fs-extra'; +import { outputJSONSync, readJSONSync, rmSync, statSync, unlinkSync, writeFileSync } from 'fs-extra'; import type { Options as EtcOptions } from 'babel-plugin-ember-template-compilation'; import type { Options as ResolverTransformOptions } from './resolver-transform'; import type { Options as AdjustImportsOptions } from './babel-plugin-adjust-imports'; @@ -64,7 +64,8 @@ import TreeSync from 'tree-sync'; export class CompatAppBuilder { // for each relativePath, an Asset we have already emitted private assets: Map = new Map(); - private treeSync: TreeSync | undefined; + private appSync: { tree: TreeSync; files: Set } | undefined; + private fastbootSync: { tree: TreeSync; files: Set } | undefined; constructor( private root: string, @@ -276,8 +277,8 @@ export class CompatAppBuilder { appRoot: this.origAppPackage.root, engines: engines.map((engine, index) => ({ packageName: engine.package.name, - root: index === 0 ? this.root : engine.package.root, // first engine is the app, which has been relocated to this.roto - fastbootFiles: {}, + root: index === 0 ? this.root : engine.package.root, // first engine is the app, which has been relocated to this.root + fastbootFiles: engine.fastbootFiles, activeAddons: [...engine.addons] .map(a => ({ name: a.name, @@ -726,16 +727,45 @@ export class CompatAppBuilder { .reverse() .forEach(a => a.differ.update()); - if (!this.treeSync) { - this.treeSync = new TreeSync(appJSPath, this.root); + if (!this.appSync) { + // this rm is here so that we get a full list of the files on the first + // sync. If we didn't do it, TreeSync still produces the right on-disk + // output but it doesn't tell us the names of all the files that are in + // there on the first pass, if they were unchanged. + rmSync(this.root, { recursive: true, force: true }); + this.appSync = { tree: new TreeSync(appJSPath, this.root), files: new Set() }; } + syncTree(this.appSync); - this.treeSync.sync(); + let fastbootFiles: { [appName: string]: { localFilename: string; shadowedFilename: string | undefined } } = {}; + + if (this.activeFastboot) { + let fastbootDir = this.fastbootJSSrcDir(); + if (fastbootDir) { + if (!this.fastbootSync) { + this.fastbootSync = { + tree: new TreeSync(fastbootDir, resolvePath(this.root, '_fastboot_')), + files: new Set(), + }; + } + syncTree(this.fastbootSync); + fastbootFiles = Object.fromEntries( + [...this.fastbootSync.files].map(name => [ + `./${name}`, + { + localFilename: `./_fastboot_/${name}`, + shadowedFilename: this.appSync!.files.has(name) ? `./${name}` : undefined, + }, + ]) + ); + } + } return this.appDiffers.map(a => { return { ...a.engine, appFiles: new AppFiles(a.differ, this.resolvableExtensionsPattern, this.podModulePrefix()), + fastbootFiles, }; }); } @@ -1625,3 +1655,22 @@ class ConcatenatedAsset { return this.relativePath.replace(this.resolvableExtensions, '') + '.map'; } } + +function syncTree({ tree, files }: { tree: TreeSync; files: Set }): void { + for (let [operation, name] of tree.sync()) { + switch (operation) { + case 'rmdir': + case 'mkdir': + // we don't track directories, only files + break; + case 'unlink': + files.delete(name); + break; + case 'change': + // we only track existence, not contents + break; + case 'create': + files.add(name); + } + } +} diff --git a/packages/core/src/app-files.ts b/packages/core/src/app-files.ts index 912c58708..7d318a8ec 100644 --- a/packages/core/src/app-files.ts +++ b/packages/core/src/app-files.ts @@ -162,4 +162,5 @@ export interface EngineSummary { export interface Engine extends EngineSummary { appFiles: AppFiles; + fastbootFiles: { [appName: string]: { localFilename: string; shadowedFilename: string | undefined } }; } diff --git a/packages/core/src/module-resolver.ts b/packages/core/src/module-resolver.ts index 9d62ef551..c5f6d7074 100644 --- a/packages/core/src/module-resolver.ts +++ b/packages/core/src/module-resolver.ts @@ -155,11 +155,11 @@ export class Resolver { return logTransition('early exit', request); } - request = this.handleFastbootCompat(request); + request = this.handleFastbootSwitch(request); request = this.handleGlobalsCompat(request); request = this.handleRenaming(request); // we expect the specifier to be app relative at this point - must be after handleRenaming - request = this.handleAppFastboot(request); + request = this.generateFastbootSwitch(request); request = this.preHandleExternal(request); // this should probably stay the last step in beforeResolve, because it can @@ -299,7 +299,7 @@ export class Resolver { return owningPackage; } - private handleAppFastboot(request: R): R { + private generateFastbootSwitch(request: R): R { let pkg = this.owningPackage(request.fromFile); if (!pkg) { @@ -314,17 +314,19 @@ export class Resolver { } let engineConfig = this.engineConfig(pkg.name); + let appRelativePath = explicitRelative(pkg.root, resolve(dirname(request.fromFile), request.specifier)); if (engineConfig) { - for (let candidate of this.withResolvableExtensions(request.specifier)) { + for (let candidate of this.withResolvableExtensions(appRelativePath)) { let fastbootFile = engineConfig.fastbootFiles[candidate]; if (fastbootFile) { if (fastbootFile.shadowedFilename) { let { names } = describeExports(readFileSync(resolve(pkg.root, fastbootFile.shadowedFilename), 'utf8'), {}); - return logTransition( - 'shadowed app fastboot', - request, - request.virtualize(fastbootSwitch(candidate, resolve(pkg.root, 'package.json'), names)) - ); + let switchFile = fastbootSwitch(candidate, resolve(pkg.root, 'package.json'), names); + if (switchFile === request.fromFile) { + return logTransition('internal lookup from fastbootSwitch', request); + } else { + return logTransition('shadowed app fastboot', request, request.virtualize(switchFile)); + } } else { return logTransition( 'unshadowed app fastboot', @@ -339,7 +341,7 @@ export class Resolver { return request; } - private handleFastbootCompat(request: R): R { + private handleFastbootSwitch(request: R): R { let match = decodeFastbootSwitch(request.fromFile); if (!match) { return request; @@ -353,7 +355,7 @@ export class Resolver { } if (!section) { - return request; + return logTransition('non-special import in fastboot switch', request); } let pkg = this.owningPackage(match.filename); @@ -370,17 +372,29 @@ export class Resolver { } else { targetFile = fastbootFile.localFilename; } - return request.alias(targetFile).rehome(resolve(pkg.root, 'package.json')); + return logTransition( + 'matched app entry', + request, + // deliberately not using rehome because we want + // generateFastbootSwitch to see that this request is coming *from* + // a fastboot switch so it won't cycle back around. Instead we make + // the targetFile relative to the fromFile that we already have. + request.alias(explicitRelative(dirname(request.fromFile), resolve(pkg.root, targetFile))) + ); } } let entry = this.getEntryFromMergeMap(rel, pkg.root)?.entry; if (entry?.type === 'both') { - return request.alias(entry[section].localPath).rehome(resolve(entry[section].packageRoot, 'package.json')); + return logTransition( + 'matched addon entry', + request, + request.alias(entry[section].localPath).rehome(resolve(entry[section].packageRoot, 'package.json')) + ); } } - return request; + return logTransition('failed to match in fastboot switch', request); } private handleGlobalsCompat(request: R): R { From 55b592ce226bb598f92eb91894fd090e3806510b Mon Sep 17 00:00:00 2001 From: Edward Faulkner Date: Wed, 28 Jun 2023 07:39:37 -0400 Subject: [PATCH 72/72] replace tree-sync with a utility that only mirrors changes --- packages/compat/src/compat-app-builder.ts | 36 ++++-------------- packages/compat/src/sync-dir.ts | 46 +++++++++++++++++++++++ 2 files changed, 53 insertions(+), 29 deletions(-) create mode 100644 packages/compat/src/sync-dir.ts diff --git a/packages/compat/src/compat-app-builder.ts b/packages/compat/src/compat-app-builder.ts index 9dc1a4995..dbb9e3c3a 100644 --- a/packages/compat/src/compat-app-builder.ts +++ b/packages/compat/src/compat-app-builder.ts @@ -56,7 +56,7 @@ import SourceMapConcat from 'fast-sourcemap-concat'; import escapeRegExp from 'escape-string-regexp'; import type CompatApp from './compat-app'; -import TreeSync from 'tree-sync'; +import { SyncDir } from './sync-dir'; // This exists during the actual broccoli build step. As opposed to CompatApp, // which also exists during pipeline-construction time. @@ -64,8 +64,8 @@ import TreeSync from 'tree-sync'; export class CompatAppBuilder { // for each relativePath, an Asset we have already emitted private assets: Map = new Map(); - private appSync: { tree: TreeSync; files: Set } | undefined; - private fastbootSync: { tree: TreeSync; files: Set } | undefined; + private appSync: SyncDir | undefined; + private fastbootSync: SyncDir | undefined; constructor( private root: string, @@ -733,9 +733,9 @@ export class CompatAppBuilder { // output but it doesn't tell us the names of all the files that are in // there on the first pass, if they were unchanged. rmSync(this.root, { recursive: true, force: true }); - this.appSync = { tree: new TreeSync(appJSPath, this.root), files: new Set() }; + this.appSync = new SyncDir(appJSPath, this.root); } - syncTree(this.appSync); + this.appSync.update(); let fastbootFiles: { [appName: string]: { localFilename: string; shadowedFilename: string | undefined } } = {}; @@ -743,12 +743,9 @@ export class CompatAppBuilder { let fastbootDir = this.fastbootJSSrcDir(); if (fastbootDir) { if (!this.fastbootSync) { - this.fastbootSync = { - tree: new TreeSync(fastbootDir, resolvePath(this.root, '_fastboot_')), - files: new Set(), - }; + this.fastbootSync = new SyncDir(fastbootDir, resolvePath(this.root, '_fastboot_')); } - syncTree(this.fastbootSync); + this.fastbootSync.update(); fastbootFiles = Object.fromEntries( [...this.fastbootSync.files].map(name => [ `./${name}`, @@ -1655,22 +1652,3 @@ class ConcatenatedAsset { return this.relativePath.replace(this.resolvableExtensions, '') + '.map'; } } - -function syncTree({ tree, files }: { tree: TreeSync; files: Set }): void { - for (let [operation, name] of tree.sync()) { - switch (operation) { - case 'rmdir': - case 'mkdir': - // we don't track directories, only files - break; - case 'unlink': - files.delete(name); - break; - case 'change': - // we only track existence, not contents - break; - case 'create': - files.add(name); - } - } -} diff --git a/packages/compat/src/sync-dir.ts b/packages/compat/src/sync-dir.ts new file mode 100644 index 000000000..c3b655980 --- /dev/null +++ b/packages/compat/src/sync-dir.ts @@ -0,0 +1,46 @@ +import assertNever from 'assert-never'; +import FSTree from 'fs-tree-diff'; +import walkSync from 'walk-sync'; +import { resolve } from 'path'; +import { copySync, mkdirpSync, removeSync, rmdirSync, unlinkSync } from 'fs-extra'; + +// mirrors the changes in the src dir to the dest dir, while tracking the +// current set of files present. +export class SyncDir { + private prev: FSTree = new FSTree(); + readonly files: Set = new Set(); + + constructor(private src: string, private dest: string) {} + + update(): void { + let next = new FSTree({ + entries: walkSync.entries(this.src), + }); + for (let [operation, relativePath] of this.prev.calculatePatch(next)) { + let outputPath = resolve(this.dest, relativePath); + switch (operation) { + case 'unlink': + unlinkSync(outputPath); + this.files.delete(relativePath); + break; + case 'rmdir': + rmdirSync(outputPath); + break; + case 'mkdir': + mkdirpSync(outputPath); + break; + case 'change': + removeSync(outputPath); + copySync(resolve(this.src, relativePath), outputPath, { dereference: true }); + break; + case 'create': + copySync(resolve(this.src, relativePath), outputPath, { dereference: true }); + this.files.add(relativePath); + break; + default: + assertNever(operation); + } + this.prev = next; + } + } +}