Resolve ESM imports in HTML (#527)

* run optimized dependency install after build

* add test runner

* resolve esm imports inside of html

* replace triple with tuple

* update install options for more extensions

src/commands/build.ts

@@ -9,7 +9,7 @@ import npmRunPath from 'npm-run-path';
 import path from 'path';
 import rimraf from 'rimraf';
 import {BuildScript} from '../config';
-import {transformEsmImports} from '../rewrite-imports';
+import {transformFileImports} from '../rewrite-imports';
 import {printStats} from '../stats-formatter';
 import {CommandOptions} from '../util';
 import {
@@ -26,7 +26,7 @@ import {paint} from './paint';
 import srcFileExtensionMapping from './src-file-extension-mapping';
 
 async function installOptimizedDependencies(
-  allBuiltJsFiles: {outLoc: string; code: string}[],
+  allFilesToResolveImports: {outLoc: string; code: string}[],
   installDest: string,
   commandOptions: CommandOptions,
 ) {
@@ -41,7 +41,7 @@ async function installOptimizedDependencies(
   // 1. Scan imports from your final built JS files.
   const installTargets = await getInstallTargets(
     installConfig,
-    allBuiltJsFiles.map(({outLoc, code}) => [outLoc, code]),
+    allFilesToResolveImports.map(({outLoc, code}) => [outLoc, code]),
   );
   // 2. Install dependencies, based on the scan of your final build.
   const installResult = await installRunner(
@@ -243,7 +243,7 @@ export async function command(commandOptions: CommandOptions) {
   }
 
   const allBuiltFromFiles = new Set<string>();
-  const allBuiltJsFiles: {outLoc: string; code: string; fileLoc: string}[] = [];
+  const allFilesToResolveImports: {outLoc: string; code: string; fileLoc: string}[] = [];
   for (const workerConfig of relevantWorkers) {
     const {id, match, type} = workerConfig;
     if (type !== 'build' || match.length === 0) {
@@ -299,7 +299,9 @@ export async function command(commandOptions: CommandOptions) {
             code = `import './${path.basename(cssOutPath)}';\n` + code;
           }
           code = wrapImportMeta({code, env: true, hmr: false, config});
-          allBuiltJsFiles.push({outLoc, code, fileLoc});
+          allFilesToResolveImports.push({outLoc, code, fileLoc});
+        } else if (path.extname(outLoc) === '.html') {
+          allFilesToResolveImports.push({outLoc, code, fileLoc});
         } else {
           await fs.mkdir(path.dirname(outLoc), {recursive: true});
           await fs.writeFile(outLoc, code);
@@ -314,7 +316,7 @@ export async function command(commandOptions: CommandOptions) {
   const webModulesPath = installWorker.args.toUrl;
   const installDest = path.join(buildDirectoryLoc, webModulesPath);
   const installResult = await installOptimizedDependencies(
-    allBuiltJsFiles,
+    allFilesToResolveImports,
     installDest,
     commandOptions,
   );
@@ -323,7 +325,7 @@ export async function command(commandOptions: CommandOptions) {
   }
 
   const allProxiedFiles = new Set<string>();
-  for (const {outLoc, code, fileLoc} of allBuiltJsFiles) {
+  for (const {outLoc, code, fileLoc} of allFilesToResolveImports) {
     const resolveImportSpecifier = createImportResolver({
       fileLoc,
       webModulesPath,
@@ -332,7 +334,7 @@ export async function command(commandOptions: CommandOptions) {
       isBundled,
       config,
     });
-    const resolvedCode = await transformEsmImports(code, (spec) => {
+    const resolvedCode = await transformFileImports(code, path.extname(outLoc), (spec) => {
       // Try to resolve the specifier to a known URL in the project
       const resolvedImportUrl = resolveImportSpecifier(spec);
       if (resolvedImportUrl) {

src/commands/dev.ts

@@ -45,7 +45,11 @@ import url from 'url';
 import zlib from 'zlib';
 import {BuildScript, SnowpackPluginBuildResult} from '../config';
 import {EsmHmrEngine} from '../hmr-server-engine';
-import {scanCodeImportsExports, transformEsmImports} from '../rewrite-imports';
+import {
+  scanCodeImportsExports,
+  transformFileImports,
+  transformEsmImports,
+} from '../rewrite-imports';
 import {
   BUILD_CACHE,
   checkLockfileHash,
@@ -341,7 +345,12 @@ export async function command(commandOptions: CommandOptions) {
       }
     }
     const ext = path.extname(fileLoc).substr(1);
-    if (ext === 'js' || srcFileExtensionMapping[ext] === 'js') {
+    if (
+      ext === 'js' ||
+      srcFileExtensionMapping[ext] === 'js' ||
+      ext === 'html' ||
+      srcFileExtensionMapping[ext] === 'html'
+    ) {
       let missingWebModule: {spec: string; pkgName: string} | null = null;
       const webModulesScript = config.scripts.find((script) => script.id === 'mount:web_modules');
       const webModulesPath = webModulesScript ? webModulesScript.args.toUrl : '/web_modules';
@@ -353,28 +362,34 @@ export async function command(commandOptions: CommandOptions) {
         isBundled: false,
         config,
       });
-      builtFileResult.result = await transformEsmImports(builtFileResult.result, (spec) => {
-        // Try to resolve the specifier to a known URL in the project
-        const resolvedImportUrl = resolveImportSpecifier(spec);
-        if (resolvedImportUrl) {
-          return resolvedImportUrl;
-        }
-        // If that fails, return a placeholder import and attempt to resolve.
-        const packageName = getPackageNameFromSpecifier(spec);
-        const [depManifestLoc] = resolveDependencyManifest(packageName, cwd);
-        const doesPackageExist = !!depManifestLoc;
-        if (doesPackageExist) {
-          reinstallDependencies();
-        } else {
-          missingWebModule = {
-            spec: spec,
-            pkgName: packageName,
-          };
-        }
-        // Return a placeholder while Snowpack goes out and tries to re-install (or warn)
-        // on the missing package.
-        return spec;
-      });
+      builtFileResult.result = await transformFileImports(
+        builtFileResult.result,
+        // This is lame: because routes don't have a file extension, we create
+        // a fake file name with the correct extension, which is the important bit.
+        'file.' + (srcFileExtensionMapping[ext] || ext),
+        (spec) => {
+          // Try to resolve the specifier to a known URL in the project
+          const resolvedImportUrl = resolveImportSpecifier(spec);
+          if (resolvedImportUrl) {
+            return resolvedImportUrl;
+          }
+          // If that fails, return a placeholder import and attempt to resolve.
+          const packageName = getPackageNameFromSpecifier(spec);
+          const [depManifestLoc] = resolveDependencyManifest(packageName, cwd);
+          const doesPackageExist = !!depManifestLoc;
+          if (doesPackageExist) {
+            reinstallDependencies();
+          } else {
+            missingWebModule = {
+              spec: spec,
+              pkgName: packageName,
+            };
+          }
+          // Return a placeholder while Snowpack goes out and tries to re-install (or warn)
+          // on the missing package.
+          return spec;
+        },
+      );
       messageBus.emit('MISSING_WEB_MODULE', {
         id: fileLoc,
         data: missingWebModule,

src/commands/install.ts

@@ -494,6 +494,10 @@ export async function command(commandOptions: CommandOptions) {
   }
 }
 
+interface InstalllRunOptions extends CommandOptions {
+  installTargets: InstallTarget[];
+}
+
 interface InstallRunResult {
   success: boolean;
   hasError: boolean;
@@ -502,10 +506,12 @@ interface InstallRunResult {
   stats: DependencyStatsOutput | null;
 }
 
-export async function run(
-  {config, lockfile, pkgManifest}: CommandOptions,
-  installTargets: InstallTarget[],
-): Promise<InstallRunResult> {
+export async function run({
+  config,
+  lockfile,
+  pkgManifest,
+  installTargets,
+}: InstalllRunOptions): Promise<InstallRunResult> {
   const {
     installOptions: {dest},
     webDependencies,

src/rewrite-imports.ts

@@ -1,3 +1,5 @@
+import {HTML_JS_REGEX} from './util';
+
 const {parse} = require('es-module-lexer');
 
 function spliceString(source: string, withSlice: string, start: number, end: number) {
@@ -41,3 +43,35 @@ export async function transformEsmImports(
   }
   return rewrittenCode;
 }
+
+async function transformHtmlImports(code: string, replaceImport: (specifier: string) => string) {
+  let rewrittenCode = code;
+  let match;
+  const importRegex = new RegExp(HTML_JS_REGEX);
+  while ((match = importRegex.exec(rewrittenCode))) {
+    const [, scriptTag, scriptCode] = match;
+    rewrittenCode = spliceString(
+      rewrittenCode,
+      await transformEsmImports(scriptCode, replaceImport),
+      match.index + scriptTag.length,
+      match.index + scriptTag.length + scriptCode.length,
+    );
+  }
+  return rewrittenCode;
+}
+
+export async function transformFileImports(
+  code: string,
+  fileName: string,
+  replaceImport: (specifier: string) => string,
+) {
+  if (fileName.endsWith('.js')) {
+    return transformEsmImports(code, replaceImport);
+  }
+  if (fileName.endsWith('.html')) {
+    return transformHtmlImports(code, replaceImport);
+  }
+  throw new Error(
+    `Incompatible file: Cannot ESM imports for file "${fileName}". This is most likely an error within Snowpack.`,
+  );
+}

src/scan-imports.ts

@@ -7,7 +7,7 @@ import nodePath from 'path';
 import stripComments from 'strip-comments';
 import validatePackageName from 'validate-npm-package-name';
 import {SnowpackConfig} from './config';
-import {isTruthy, findMatchingMountScript} from './util';
+import {isTruthy, findMatchingMountScript, HTML_JS_REGEX} from './util';
 
 const WEB_MODULES_TOKEN = 'web_modules/';
 const WEB_MODULES_TOKEN_LENGTH = WEB_MODULES_TOKEN.length;
@@ -21,7 +21,6 @@ const ESM_DYNAMIC_IMPORT_REGEX = /import\((?:['"].+['"]|`[^$]+`)\)/gm;
 const HAS_NAMED_IMPORTS_REGEX = /^[\w\s\,]*\{(.*)\}/s;
 const SPLIT_NAMED_IMPORTS_REGEX = /\bas\s+\w+|,/s;
 const DEFAULT_IMPORT_REGEX = /import\s+(\w)+(,\s\{[\w\s]*\})?\s+from/s;
-const HTML_JS_REGEX = /<script.*?>(.*)<\/script>/gms;
 
 /**
  * An install target represents information about a dependency to install.
@@ -244,7 +243,10 @@ export async function scanImports(cwd: string, config: SnowpackConfig): Promise<
         while ((match = regex.exec(result))) {
           allMatches.push(match);
         }
-        return [filePath, allMatches.map(([full, code]) => code).join('\n')] as [string, string];
+        return [
+          filePath,
+          allMatches.map(([full, scriptTag, scriptCode]) => scriptCode).join('\n'),
+        ] as [string, string];
       }
       // If we don't recognize the file type, it could be source. Warn just in case.
       if (!mime.lookup(nodePath.extname(filePath))) {

src/util.ts

@@ -22,6 +22,8 @@ export const DEV_DEPENDENCIES_DIR = path.join(PROJECT_CACHE_DIR, 'dev');
 const LOCKFILE_HASH_FILE = '.hash';
 
 export const HAS_CDN_HASH_REGEX = /\-[a-zA-Z0-9]{16,}/;
+export const HTML_JS_REGEX = /(<script.*?>)(.+?)<\/script>/gms;
+
 export interface ImportMap {
   imports: {[packageName: string]: string};
 }

test/build/resolve-imports/expected-build/_dist_/index.html

@@ -0,0 +1,38 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <meta name="description" content="Web site created using create-snowpack-app" />
+    <title>Snowpack App</title>
+  </head>
+  <body>
+    <script type="module" src="/_dist_/index.js"></script>
+    <script type="module">
+      // Path aliases
+      import {flatten} from 'array-flatten';
+      console.log(flatten);
+
+      // Importing a file
+      import sort from './sort'; // relative import
+      import sort_ from 'src/sort'; // bare import using mount
+      import sort__ from 'src/sort.js'; // bare import using mount + extension
+      console.log(sort, sort_, sort__);
+
+      // Importing a directory index.js file
+      import components from './components'; // relative import 
+      import components_ from './components/index'; // relative import with index appended
+      import components__ from './components/index.js'; // relative import with index appended
+      import components___ from 'src/components'; // bare import using mount
+      import components____ from 'src/components/index'; // bare import using mount and index appended
+      import components_____ from 'src/components/index.js'; // bare import using mount and index.js appended
+      console.log(components, components_, components__, components___, components____, components_____);
+
+
+      // Importing something that isn't JS
+      import styles from './components/style.css'; // relative import 
+      import styles_ from 'src/components/style.css'; // relative import 
+      console.log(styles, styles_);
+    </script>
+  </body>
+</html>

test/build/resolve-imports/package.json

@@ -5,8 +5,10 @@
   },
   "snowpack": {
     "scripts": {
-      "mount:public": "mount public --to /",
       "mount:src": "mount src --to /_dist_"
+    },
+    "devOptions": {
+      "fallback": "_dist_/index.html"
     }
   },
   "dependencies": {