feat(compile): unstable npm and node specifier support (#19005)

This is the initial support for npm and node specifiers in `deno
compile`. The npm packages are included in the binary and read from it via
a virtual file system. This also supports the `--node-modules-dir` flag,
dependencies specified in a package.json, and npm binary commands (ex.
`deno compile --unstable npm:cowsay`)

Closes #16632

cli/Cargo.toml

@@ -84,7 +84,7 @@ lazy-regex.workspace = true
 libc.workspace = true
 log = { workspace = true, features = ["serde"] }
 lsp-types.workspace = true
-monch = "=0.4.1"
+monch = "=0.4.2"
 notify.workspace = true
 once_cell.workspace = true
 os_pipe.workspace = true

cli/args/flags.rs

@@ -527,8 +527,11 @@ impl Flags {
           .ok()
       }
       Task(_) | Check(_) | Coverage(_) | Cache(_) | Info(_) | Eval(_)
-      | Test(_) | Bench(_) | Repl(_) => std::env::current_dir().ok(),
-      _ => None,
+      | Test(_) | Bench(_) | Repl(_) | Compile(_) => {
+        std::env::current_dir().ok()
+      }
+      Bundle(_) | Completions(_) | Doc(_) | Fmt(_) | Init(_) | Install(_)
+      | Uninstall(_) | Lsp | Lint(_) | Types | Upgrade(_) | Vendor(_) => None,
     }
   }
 

cli/args/mod.rs

@@ -33,6 +33,7 @@ pub use config_file::TsTypeLib;
 pub use flags::*;
 pub use lockfile::Lockfile;
 pub use lockfile::LockfileError;
+pub use package_json::PackageJsonDepsProvider;
 
 use deno_ast::ModuleSpecifier;
 use deno_core::anyhow::anyhow;
@@ -556,7 +557,7 @@ struct CliOptionOverrides {
   import_map_specifier: Option<Option<ModuleSpecifier>>,
 }
 
-/// Holds the resolved options of many sources used by sub commands
+/// Holds the resolved options of many sources used by subcommands
 /// and provides some helper function for creating common objects.
 pub struct CliOptions {
   // the source of the options is a detail the rest of the
@@ -1303,6 +1304,16 @@ fn has_flag_env_var(name: &str) -> bool {
   matches!(value.as_ref().map(|s| s.as_str()), Ok("1"))
 }
 
+pub fn npm_pkg_req_ref_to_binary_command(
+  req_ref: &NpmPackageReqReference,
+) -> String {
+  let binary_name = req_ref
+    .sub_path
+    .as_deref()
+    .unwrap_or(req_ref.req.name.as_str());
+  binary_name.to_string()
+}
+
 #[cfg(test)]
 mod test {
   use super::*;

cli/args/package_json.rs

@@ -28,6 +28,33 @@ pub enum PackageJsonDepValueParseError {
 pub type PackageJsonDeps =
   BTreeMap<String, Result<NpmPackageReq, PackageJsonDepValueParseError>>;
 
+#[derive(Debug, Default)]
+pub struct PackageJsonDepsProvider(Option<PackageJsonDeps>);
+
+impl PackageJsonDepsProvider {
+  pub fn new(deps: Option<PackageJsonDeps>) -> Self {
+    Self(deps)
+  }
+
+  pub fn deps(&self) -> Option<&PackageJsonDeps> {
+    self.0.as_ref()
+  }
+
+  pub fn reqs(&self) -> Vec<&NpmPackageReq> {
+    match &self.0 {
+      Some(deps) => {
+        let mut package_reqs = deps
+          .values()
+          .filter_map(|r| r.as_ref().ok())
+          .collect::<Vec<_>>();
+        package_reqs.sort(); // deterministic resolution
+        package_reqs
+      }
+      None => Vec::new(),
+    }
+  }
+}
+
 /// Gets an application level package.json's npm package requirements.
 ///
 /// Note that this function is not general purpose. It is specifically for

cli/factory.rs

@@ -1,9 +1,11 @@
 // Copyright 2018-2023 the Deno authors. All rights reserved. MIT license.
 
+use crate::args::npm_pkg_req_ref_to_binary_command;
 use crate::args::CliOptions;
 use crate::args::DenoSubcommand;
 use crate::args::Flags;
 use crate::args::Lockfile;
+use crate::args::PackageJsonDepsProvider;
 use crate::args::StorageKeyResolver;
 use crate::args::TsConfigType;
 use crate::cache::Caches;
@@ -30,6 +32,7 @@ use crate::npm::NpmCache;
 use crate::npm::NpmResolution;
 use crate::npm::PackageJsonDepsInstaller;
 use crate::resolver::CliGraphResolver;
+use crate::standalone::DenoCompileBinaryWriter;
 use crate::tools::check::TypeChecker;
 use crate::util::progress_bar::ProgressBar;
 use crate::util::progress_bar::ProgressBarStyle;
@@ -151,6 +154,7 @@ struct CliFactoryServices {
   npm_cache: Deferred<Arc<NpmCache>>,
   npm_resolver: Deferred<Arc<CliNpmResolver>>,
   npm_resolution: Deferred<Arc<NpmResolution>>,
+  package_json_deps_provider: Deferred<Arc<PackageJsonDepsProvider>>,
   package_json_deps_installer: Deferred<Arc<PackageJsonDepsInstaller>>,
   text_only_progress_bar: Deferred<ProgressBar>,
   type_checker: Deferred<Arc<TypeChecker>>,
@@ -301,15 +305,17 @@ impl CliFactory {
       .npm_resolver
       .get_or_try_init_async(async {
         let npm_resolution = self.npm_resolution().await?;
+        let fs = self.fs().clone();
         let npm_fs_resolver = create_npm_fs_resolver(
-          self.fs().clone(),
+          fs.clone(),
           self.npm_cache()?.clone(),
           self.text_only_progress_bar(),
           CliNpmRegistryApi::default_url().to_owned(),
           npm_resolution.clone(),
           self.options.node_modules_dir_path(),
         );
         Ok(Arc::new(CliNpmResolver::new(
+          fs.clone(),
           npm_resolution.clone(),
           npm_fs_resolver,
           self.maybe_lockfile().as_ref().cloned(),
@@ -318,19 +324,25 @@ impl CliFactory {
       .await
   }
 
+  pub fn package_json_deps_provider(&self) -> &Arc<PackageJsonDepsProvider> {
+    self.services.package_json_deps_provider.get_or_init(|| {
+      Arc::new(PackageJsonDepsProvider::new(
+        self.options.maybe_package_json_deps(),
+      ))
+    })
+  }
+
   pub async fn package_json_deps_installer(
     &self,
   ) -> Result<&Arc<PackageJsonDepsInstaller>, AnyError> {
     self
       .services
       .package_json_deps_installer
       .get_or_try_init_async(async {
-        let npm_api = self.npm_api()?;
-        let npm_resolution = self.npm_resolution().await?;
         Ok(Arc::new(PackageJsonDepsInstaller::new(
-          npm_api.clone(),
-          npm_resolution.clone(),
-          self.options.maybe_package_json_deps(),
+          self.package_json_deps_provider().clone(),
+          self.npm_api()?.clone(),
+          self.npm_resolution().await?.clone(),
         )))
       })
       .await
@@ -365,6 +377,7 @@ impl CliFactory {
           self.options.no_npm(),
           self.npm_api()?.clone(),
           self.npm_resolution().await?.clone(),
+          self.package_json_deps_provider().clone(),
           self.package_json_deps_installer().await?.clone(),
         )))
       })
@@ -535,6 +548,21 @@ impl CliFactory {
     self.services.cjs_resolutions.get_or_init(Default::default)
   }
 
+  pub async fn create_compile_binary_writer(
+    &self,
+  ) -> Result<DenoCompileBinaryWriter, AnyError> {
+    Ok(DenoCompileBinaryWriter::new(
+      self.file_fetcher()?,
+      self.http_client(),
+      self.deno_dir()?,
+      self.npm_api()?,
+      self.npm_cache()?,
+      self.npm_resolver().await?,
+      self.npm_resolution().await?,
+      self.package_json_deps_provider(),
+    ))
+  }
+
   /// Gets a function that can be used to create a CliMainWorkerFactory
   /// for a file watcher.
   pub async fn create_cli_main_worker_factory_func(
@@ -572,6 +600,7 @@ impl CliFactory {
           NpmModuleLoader::new(
             cjs_resolutions.clone(),
             node_code_translator.clone(),
+            fs.clone(),
             node_resolver.clone(),
           ),
         )),
@@ -587,6 +616,7 @@ impl CliFactory {
     &self,
   ) -> Result<CliMainWorkerFactory, AnyError> {
     let node_resolver = self.node_resolver().await?;
+    let fs = self.fs();
     Ok(CliMainWorkerFactory::new(
       StorageKeyResolver::from_options(&self.options),
       self.npm_resolver().await?.clone(),
@@ -603,6 +633,7 @@ impl CliFactory {
         NpmModuleLoader::new(
           self.cjs_resolutions().clone(),
           self.node_code_translator().await?.clone(),
+          fs.clone(),
           node_resolver.clone(),
         ),
       )),
@@ -637,11 +668,8 @@ impl CliFactory {
           if let Ok(pkg_ref) = NpmPackageReqReference::from_str(&flags.script) {
             // if the user ran a binary command, we'll need to set process.argv[0]
             // to be the name of the binary command instead of deno
-            let binary_name = pkg_ref
-              .sub_path
-              .as_deref()
-              .unwrap_or(pkg_ref.req.name.as_str());
-            maybe_binary_command_name = Some(binary_name.to_string());
+            maybe_binary_command_name =
+              Some(npm_pkg_req_ref_to_binary_command(&pkg_ref));
           }
         }
         maybe_binary_command_name

cli/graph_util.rs

@@ -345,10 +345,10 @@ pub fn error_for_any_npm_specifier(
   for module in graph.modules() {
     match module {
       Module::Npm(module) => {
-        bail!("npm specifiers have not yet been implemented for this sub command (https://github.com/denoland/deno/issues/15960). Found: {}", module.specifier)
+        bail!("npm specifiers have not yet been implemented for this subcommand (https://github.com/denoland/deno/issues/15960). Found: {}", module.specifier)
       }
       Module::Node(module) => {
-        bail!("Node specifiers have not yet been implemented for this sub command (https://github.com/denoland/deno/issues/15960). Found: node:{}", module.module_name)
+        bail!("Node specifiers have not yet been implemented for this subcommand (https://github.com/denoland/deno/issues/15960). Found: node:{}", module.module_name)
       }
       Module::Esm(_) | Module::Json(_) | Module::External(_) => {}
     }

cli/lsp/documents.rs

@@ -46,6 +46,7 @@ use deno_semver::npm::NpmPackageReqReference;
 use indexmap::IndexMap;
 use lsp::Url;
 use once_cell::sync::Lazy;
+use package_json::PackageJsonDepsProvider;
 use std::collections::BTreeMap;
 use std::collections::HashMap;
 use std::collections::HashSet;
@@ -1218,17 +1219,20 @@ impl Documents {
       maybe_jsx_config.as_ref(),
       maybe_package_json_deps.as_ref(),
     );
+    let deps_provider =
+      Arc::new(PackageJsonDepsProvider::new(maybe_package_json_deps));
     let deps_installer = Arc::new(PackageJsonDepsInstaller::new(
+      deps_provider.clone(),
       npm_registry_api.clone(),
       npm_resolution.clone(),
-      maybe_package_json_deps,
     ));
     self.resolver = Arc::new(CliGraphResolver::new(
       maybe_jsx_config,
       maybe_import_map,
       false,
       npm_registry_api,
       npm_resolution,
+      deps_provider,
       deps_installer,
     ));
     self.imports = Arc::new(

cli/lsp/language_server.rs

@@ -457,8 +457,9 @@ fn create_lsp_structs(
   ));
   let resolution =
     Arc::new(NpmResolution::from_serialized(api.clone(), None, None));
+  let fs = Arc::new(deno_fs::RealFs);
   let fs_resolver = create_npm_fs_resolver(
-    Arc::new(deno_fs::RealFs),
+    fs.clone(),
     npm_cache.clone(),
     &progress_bar,
     registry_url.clone(),
@@ -468,7 +469,12 @@ fn create_lsp_structs(
   (
     api,
     npm_cache,
-    Arc::new(CliNpmResolver::new(resolution.clone(), fs_resolver, None)),
+    Arc::new(CliNpmResolver::new(
+      fs,
+      resolution.clone(),
+      fs_resolver,
+      None,
+    )),
     resolution,
   )
 }
@@ -711,6 +717,7 @@ impl Inner {
     ));
     let node_fs = Arc::new(deno_fs::RealFs);
     let npm_resolver = Arc::new(CliNpmResolver::new(
+      node_fs.clone(),
       npm_resolution.clone(),
       create_npm_fs_resolver(
         node_fs.clone(),

cli/main.rs

@@ -36,7 +36,6 @@ static GLOBAL: Jemalloc = Jemalloc;
 use crate::args::flags_from_vec;
 use crate::args::DenoSubcommand;
 use crate::args::Flags;
-use crate::resolver::CliGraphResolver;
 use crate::util::display;
 use crate::util::v8::get_v8_flags_from_env;
 use crate::util::v8::init_v8_flags;
@@ -97,7 +96,7 @@ async fn run_subcommand(flags: Flags) -> Result<i32, AnyError> {
       Ok(0)
     }
     DenoSubcommand::Compile(compile_flags) => {
-      tools::standalone::compile(flags, compile_flags).await?;
+      tools::compile::compile(flags, compile_flags).await?;
       Ok(0)
     }
     DenoSubcommand::Coverage(coverage_flags) => {