From f4ebdfb91f62fd660fff5776732678489356b4d9 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Wed, 13 Jul 2022 15:39:09 -0400 Subject: [PATCH 01/13] Start of work. --- cli/cache/emit.rs | 235 +++++++++++++++++++++++++++++++------- cli/cache/mod.rs | 19 +-- cli/deno_dir.rs | 8 +- cli/emit.rs | 111 ++++++------------ cli/graph_util.rs | 3 + cli/main.rs | 7 +- cli/module_loader.rs | 173 +++++++++++++++++++++++++++- cli/proc_state.rs | 192 ++++--------------------------- cli/tools/coverage/mod.rs | 34 +----- core/source_map.rs | 18 +++ 10 files changed, 455 insertions(+), 345 deletions(-) diff --git a/cli/cache/emit.rs b/cli/cache/emit.rs index e1469b862aee8e..1f18930f94c117 100644 --- a/cli/cache/emit.rs +++ b/cli/cache/emit.rs @@ -1,71 +1,220 @@ // Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. +use std::path::Path; + use deno_ast::ModuleSpecifier; use deno_core::error::AnyError; +use deno_runtime::deno_webstorage::rusqlite::params; +use deno_runtime::deno_webstorage::rusqlite::Connection; -use super::CacheType; -use super::Cacher; +use super::common::run_sqlite_pragma; /// Emit cache for a single file. #[derive(Debug, Clone, PartialEq)] pub struct SpecifierEmitCacheData { - pub source_hash: String, + pub source_hash: u64, pub text: String, pub map: Option, } -pub trait EmitCache { - /// Gets the emit data from the cache. - fn get_emit_data( - &self, - specifier: &ModuleSpecifier, - ) -> Option; - /// Sets the emit data in the cache. - fn set_emit_data( - &self, - specifier: ModuleSpecifier, - data: SpecifierEmitCacheData, - ) -> Result<(), AnyError>; - /// Gets the stored hash of the source of the provider specifier - /// to tell if the emit is out of sync with the source. - /// TODO(13302): this is actually not reliable and should be removed - /// once switching to an sqlite db - fn get_source_hash(&self, specifier: &ModuleSpecifier) -> Option; - /// Gets the emitted JavaScript of the TypeScript source. - /// TODO(13302): remove this once switching to an sqlite db - fn get_emit_text(&self, specifier: &ModuleSpecifier) -> Option; -} +/// The cache that stores previously emitted files. +pub struct EmitCache(Option); + +impl EmitCache { + pub fn new(db_file_path: &Path) -> Self { + match Self::try_new(db_file_path) { + Ok(cache) => cache, + Err(err) => { + log::debug!( + concat!( + "Failed creating internal emit cache. ", + "Recreating...\n\nError details:\n{:#}", + ), + err + ); + // Maybe the cache file is corrupt. Attempt to remove the cache file + // then attempt to recreate again. Otherwise, use null object pattern. + match std::fs::remove_file(db_file_path) { + Ok(_) => match Self::try_new(db_file_path) { + Ok(cache) => cache, + Err(err) => { + log::debug!( + concat!( + "Unable to create internal emit cache. ", + "This will reduce the performance of emitting.\n\n", + "Error details:\n{:#}", + ), + err + ); + Self(None) + } + }, + Err(_) => Self(None), + } + } + } + } -impl EmitCache for T { - fn get_emit_data( + fn try_new(db_file_path: &Path) -> Result { + let conn = Connection::open(db_file_path)?; + Self::from_connection(conn, crate::version::deno()) + } + + fn from_connection( + conn: Connection, + cli_version: String, + ) -> Result { + run_sqlite_pragma(&conn)?; + create_tables(&conn, cli_version)?; + + Ok(Self(Some(conn))) + } + + /// Gets the emit data from the cache. + pub fn get_emit_data( &self, specifier: &ModuleSpecifier, ) -> Option { + let conn = match &self.0 { + Some(conn) => conn, + None => return None, + }; + let mut stmt = conn + .prepare_cached("SELECT source_hash, emit_text, source_map FROM emitcache WHERE specifier=?1 LIMIT 1") + .ok()?; + let mut rows = stmt.query(params![specifier.to_string()]).ok()?; + let row = rows.next().ok().flatten()?; + Some(SpecifierEmitCacheData { - source_hash: self.get_source_hash(specifier)?, - text: self.get_emit_text(specifier)?, - map: self.get(CacheType::SourceMap, specifier), + source_hash: row.get(0).ok()?, + text: row.get(1).ok()?, + map: row.get(2).ok()?, }) } - fn get_source_hash(&self, specifier: &ModuleSpecifier) -> Option { - self.get(CacheType::Version, specifier) - } - - fn get_emit_text(&self, specifier: &ModuleSpecifier) -> Option { - self.get(CacheType::Emit, specifier) + /// Sets the emit data in the cache. + pub fn set_emit_data( + &self, + specifier: &ModuleSpecifier, + data: &SpecifierEmitCacheData, + ) { + if let Err(err) = self.set_emit_data_result(specifier, data) { + // should never error here, but if it ever does don't fail + if cfg!(debug_assertions) { + panic!("Error saving emit data: {}", err); + } else { + log::debug!("Error saving emit data: {}", err); + } + } } - fn set_emit_data( + fn set_emit_data_result( &self, - specifier: ModuleSpecifier, - data: SpecifierEmitCacheData, + specifier: &ModuleSpecifier, + data: &SpecifierEmitCacheData, ) -> Result<(), AnyError> { - self.set(CacheType::Version, &specifier, data.source_hash)?; - self.set(CacheType::Emit, &specifier, data.text)?; - if let Some(map) = data.map { - self.set(CacheType::SourceMap, &specifier, map)?; - } + let conn = match &self.0 { + Some(conn) => conn, + None => return Ok(()), + }; + let mut stmt = conn.prepare_cached( + "INSERT OR REPLACE INTO emitcache (specifier, source_hash, emit_text, source_map) VALUES (?1, ?2, ?3, ?4)", + )?; + stmt.execute(params![ + specifier.to_string(), + &data.source_hash, + &data.text, + &data.map, + ])?; Ok(()) } } + +fn create_tables( + conn: &Connection, + cli_version: String, +) -> Result<(), AnyError> { + // INT doesn't store up to u64, so use TEXT for source_hash + conn.execute( + "CREATE TABLE IF NOT EXISTS emitcache ( + specifier TEXT PRIMARY KEY, + source_hash TEXT NOT NULL, + emit_text TEXT NOT NULL, + source_map TEXT + )", + [], + )?; + conn.execute( + "CREATE TABLE IF NOT EXISTS info ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + )", + [], + )?; + + // delete the cache when the CLI version changes + let data_cli_version: Option = conn + .query_row( + "SELECT value FROM info WHERE key='CLI_VERSION' LIMIT 1", + [], + |row| row.get(0), + ) + .ok(); + if data_cli_version != Some(cli_version.to_string()) { + conn.execute("DELETE FROM emitcache", params![])?; + let mut stmt = conn + .prepare("INSERT OR REPLACE INTO info (key, value) VALUES (?1, ?2)")?; + stmt.execute(params!["CLI_VERSION", &cli_version])?; + } + + Ok(()) +} + +// #[cfg(test)] +// mod test { +// use super::*; + +// #[test] +// pub fn check_cache_general_use() { +// let conn = Connection::open_in_memory().unwrap(); +// let cache = EmitCache::from_connection(conn, "1.0.0".to_string()).unwrap(); + +// assert!(!cache.has_check_hash(1)); +// cache.add_check_hash(1); +// assert!(cache.has_check_hash(1)); +// assert!(!cache.has_check_hash(2)); + +// let specifier1 = ModuleSpecifier::parse("file:///test.json").unwrap(); +// assert_eq!(cache.get_tsbuildinfo(&specifier1), None); +// cache.set_tsbuildinfo(&specifier1, "test"); +// assert_eq!(cache.get_tsbuildinfo(&specifier1), Some("test".to_string())); + +// // try changing the cli version (should clear) +// let conn = cache.0.unwrap(); +// let cache = +// TypeCheckCache::from_connection(conn, "2.0.0".to_string()).unwrap(); +// assert!(!cache.has_check_hash(1)); +// cache.add_check_hash(1); +// assert!(cache.has_check_hash(1)); +// assert_eq!(cache.get_tsbuildinfo(&specifier1), None); +// cache.set_tsbuildinfo(&specifier1, "test"); +// assert_eq!(cache.get_tsbuildinfo(&specifier1), Some("test".to_string())); + +// // recreating the cache should not remove the data because the CLI version and state hash is the same +// let conn = cache.0.unwrap(); +// let cache = +// TypeCheckCache::from_connection(conn, "2.0.0".to_string()).unwrap(); +// assert!(cache.has_check_hash(1)); +// assert!(!cache.has_check_hash(2)); +// assert_eq!(cache.get_tsbuildinfo(&specifier1), Some("test".to_string())); + +// // adding when already exists should not cause issue +// cache.add_check_hash(1); +// assert!(cache.has_check_hash(1)); +// cache.set_tsbuildinfo(&specifier1, "other"); +// assert_eq!( +// cache.get_tsbuildinfo(&specifier1), +// Some("other".to_string()) +// ); +// } +// } diff --git a/cli/cache/mod.rs b/cli/cache/mod.rs index f363d8fa8743ad..0cbc019044a433 100644 --- a/cli/cache/mod.rs +++ b/cli/cache/mod.rs @@ -59,7 +59,6 @@ pub trait Cacher { /// A "wrapper" for the FileFetcher and DiskCache for the Deno CLI that provides /// a concise interface to the DENO_DIR when building module graphs. pub struct FetchCacher { - disk_cache: DiskCache, dynamic_permissions: Permissions, file_fetcher: Arc, root_permissions: Permissions, @@ -67,7 +66,6 @@ pub struct FetchCacher { impl FetchCacher { pub fn new( - disk_cache: DiskCache, file_fetcher: FileFetcher, root_permissions: Permissions, dynamic_permissions: Permissions, @@ -75,7 +73,6 @@ impl FetchCacher { let file_fetcher = Arc::new(file_fetcher); Self { - disk_cache, dynamic_permissions, file_fetcher, root_permissions, @@ -87,21 +84,11 @@ impl Loader for FetchCacher { fn get_cache_info(&self, specifier: &ModuleSpecifier) -> Option { let local = self.file_fetcher.get_local_path(specifier)?; if local.is_file() { - let location = &self.disk_cache.location; - let emit = self - .disk_cache - .get_cache_filename_with_extension(specifier, "js") - .map(|p| location.join(p)) - .filter(|p| p.is_file()); - let map = self - .disk_cache - .get_cache_filename_with_extension(specifier, "js.map") - .map(|p| location.join(p)) - .filter(|p| p.is_file()); Some(CacheInfo { local: Some(local), - emit, - map, + // we no longer store the emit and map in their own files + emit: None, + map: None, }) } else { None diff --git a/cli/deno_dir.rs b/cli/deno_dir.rs index 0a1864f2ce000a..cb19a8a23fbb6f 100644 --- a/cli/deno_dir.rs +++ b/cli/deno_dir.rs @@ -46,6 +46,12 @@ impl DenoDir { Ok(deno_dir) } + /// Path for the cache used for emits. + pub fn emit_cache_db_file_path(&self) -> PathBuf { + // bump this version name to invalidate the entire cache + self.root.join("emit_cache_v1") + } + /// Path for the incremental cache used for formatting. pub fn fmt_incremental_cache_db_file_path(&self) -> PathBuf { // bump this version name to invalidate the entire cache @@ -58,7 +64,7 @@ impl DenoDir { self.root.join("lint_incremental_cache_v1") } - /// Path for the incremental cache used for linting. + /// Path for the cache used for type checking. pub fn type_checking_cache_db_file_path(&self) -> PathBuf { // bump this version name to invalidate the entire cache self.root.join("check_cache_v1") diff --git a/cli/emit.rs b/cli/emit.rs index a530dbcb913c84..b609231c98a761 100644 --- a/cli/emit.rs +++ b/cli/emit.rs @@ -22,6 +22,7 @@ use crate::version; use deno_ast::swc::bundler::Hook; use deno_ast::swc::bundler::ModuleRecord; use deno_ast::swc::common::Span; +use deno_ast::ParsedSource; use deno_core::error::AnyError; use deno_core::parking_lot::RwLock; use deno_core::serde::Deserialize; @@ -32,15 +33,12 @@ use deno_core::serde_json; use deno_core::serde_json::json; use deno_core::ModuleSpecifier; use deno_graph::MediaType; -use deno_graph::ModuleGraph; use deno_graph::ModuleGraphError; use deno_graph::ModuleKind; use deno_graph::ResolutionError; -use std::collections::HashSet; use std::fmt; use std::result; use std::sync::Arc; -use std::time::Instant; /// A structure representing stats from an emit operation for a graph. #[derive(Clone, Debug, Default, Eq, PartialEq)] @@ -259,15 +257,44 @@ fn get_tsc_roots( } } +pub fn emit_parsed_source( + cache: &EmitCache, + specifier: &ModuleSpecifier, + parsed_source: &ParsedSource, + emit_options: &deno_ast::EmitOptions, + emit_config_hash: &str, +) -> Result { + let source_hash = get_source_hash_for_emit( + parsed_source.text_info().text_str(), + emit_config_hash, + ); + if let Some(emit_data) = cache.get_emit_data(specifier) { + if emit_data.source_hash == source_hash { + return Ok(emit_data.text); + } + } + let transpiled_source = parsed_source.transpile(&emit_options)?; + let cache_data = SpecifierEmitCacheData { + source_hash, + text: transpiled_source.text, + map: transpiled_source.source_map, + }; + cache.set_emit_data(specifier, &cache_data); + Ok(cache_data.text) +} + /// A hashing function that takes the source code, version and optionally a /// user provided config and generates a string hash which can be stored to /// determine if the cached emit is valid or not. -fn get_version(source_bytes: &[u8], config_bytes: &[u8]) -> String { - crate::checksum::gen(&[ - source_bytes, - version::deno().as_bytes(), - config_bytes, - ]) +fn get_source_hash_for_emit(source_text: &str, emit_config_hash: &str) -> u64 { + // twox hash is insecure, but fast so it works for our purposes + use std::hash::Hasher; + use twox_hash::XxHash64; + + let mut hasher = XxHash64::default(); + hasher.write(source_text.as_bytes()); + hasher.write(emit_config_hash.as_bytes()); + hasher.finish() } /// Determine if a given module kind and media type is emittable or not. @@ -404,72 +431,6 @@ pub fn check( }) } -pub struct EmitOptions { - pub ts_config: TsConfig, - pub reload: bool, - pub reload_exclusions: HashSet, -} - -/// Given a module graph, emit any appropriate modules and cache them. -// TODO(nayeemrmn): This would ideally take `GraphData` like -// `check()`, but the AST isn't stored in that. Cleanup. -pub fn emit( - graph: &ModuleGraph, - cache: &dyn EmitCache, - options: EmitOptions, -) -> Result { - let start = Instant::now(); - let config_bytes = options.ts_config.as_bytes(); - let include_js = options.ts_config.get_check_js(); - let emit_options = options.ts_config.into(); - - let mut emit_count = 0_u32; - let mut file_count = 0_u32; - for module in graph.modules() { - file_count += 1; - if !is_emittable(&module.kind, &module.media_type, include_js) { - continue; - } - let needs_reload = - options.reload && !options.reload_exclusions.contains(&module.specifier); - let version = get_version( - module.maybe_source.as_ref().map(|s| s.as_bytes()).unwrap(), - &config_bytes, - ); - let is_valid = cache - .get_source_hash(&module.specifier) - .map_or(false, |v| v == version); - if is_valid && !needs_reload { - continue; - } - let transpiled_source = module - .maybe_parsed_source - .as_ref() - .map(|source| source.transpile(&emit_options)) - .unwrap()?; - emit_count += 1; - cache.set_emit_data( - module.specifier.clone(), - SpecifierEmitCacheData { - source_hash: version, - text: transpiled_source.text, - map: transpiled_source.source_map, - }, - )?; - } - - let stats = Stats(vec![ - ("Files".to_string(), file_count), - ("Emitted".to_string(), emit_count), - ("Total time".to_string(), start.elapsed().as_millis() as u32), - ]); - - Ok(CheckResult { - diagnostics: Diagnostics::default(), - stats, - }) -} - enum CheckHashResult { Hash(u64), NoFiles, diff --git a/cli/graph_util.rs b/cli/graph_util.rs index de418edd7a654f..4f9c66138599c2 100644 --- a/cli/graph_util.rs +++ b/cli/graph_util.rs @@ -4,6 +4,7 @@ use crate::colors; use crate::emit::TsTypeLib; use crate::errors::get_error_class_name; +use deno_ast::ParsedSource; use deno_core::error::custom_error; use deno_core::error::AnyError; use deno_core::ModuleSpecifier; @@ -38,6 +39,7 @@ pub fn contains_specifier( pub enum ModuleEntry { Module { code: Arc, + maybe_parsed_source: Option, dependencies: BTreeMap, media_type: MediaType, /// Whether or not this is a JS/JSX module with a `@ts-check` directive. @@ -146,6 +148,7 @@ impl GraphData { }; let module_entry = ModuleEntry::Module { code, + maybe_parsed_source: module.maybe_parsed_source.clone(), dependencies: module.dependencies.clone(), ts_check, media_type, diff --git a/cli/main.rs b/cli/main.rs index fbbfc77d24ea6b..6c666dc578fe87 100644 --- a/cli/main.rs +++ b/cli/main.rs @@ -168,11 +168,11 @@ fn create_web_worker_callback( .map(ToOwned::to_owned), root_cert_store: Some(ps.root_cert_store.clone()), seed: ps.options.seed(), - module_loader, create_web_worker_cb, preload_module_cb, format_js_error_fn: Some(Arc::new(format_js_error)), - source_map_getter: Some(Box::new(ps.clone())), + source_map_getter: Some(Box::new(module_loader.clone())), + module_loader, worker_type: args.worker_type, maybe_inspector_server, get_error_class_fn: Some(&crate::errors::get_error_class_name), @@ -248,7 +248,7 @@ pub fn create_main_worker( .map(ToOwned::to_owned), root_cert_store: Some(ps.root_cert_store.clone()), seed: ps.options.seed(), - source_map_getter: Some(Box::new(ps.clone())), + source_map_getter: Some(Box::new(module_loader.clone())), format_js_error_fn: Some(Arc::new(format_js_error)), create_web_worker_cb, web_worker_preload_module_cb, @@ -609,7 +609,6 @@ async fn create_graph_and_maybe_check( debug: bool, ) -> Result, AnyError> { let mut cache = cache::FetchCacher::new( - ps.dir.gen_cache.clone(), ps.file_fetcher.clone(), Permissions::allow_all(), Permissions::allow_all(), diff --git a/cli/module_loader.rs b/cli/module_loader.rs index cf94a47671d9e8..db6ba8abac6957 100644 --- a/cli/module_loader.rs +++ b/cli/module_loader.rs @@ -1,20 +1,35 @@ // Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. +use crate::cache::EmitCache; +use crate::emit::emit_parsed_source; use crate::emit::TsTypeLib; +use crate::graph_util::ModuleEntry; use crate::proc_state::ProcState; +use deno_ast::MediaType; +use deno_core::anyhow::anyhow; use deno_core::error::AnyError; use deno_core::futures::future::FutureExt; use deno_core::futures::Future; +use deno_core::resolve_url; use deno_core::ModuleLoader; +use deno_core::ModuleSource; use deno_core::ModuleSpecifier; +use deno_core::ModuleType; use deno_core::OpState; +use deno_core::SourceMapGetter; use deno_runtime::permissions::Permissions; use std::cell::RefCell; use std::pin::Pin; use std::rc::Rc; use std::str; +pub struct ModuleCodeSource { + pub code: String, + pub found_url: ModuleSpecifier, + pub media_type: MediaType, +} + pub struct CliModuleLoader { pub lib: TsTypeLib, /// The initial set of permissions used to resolve the static imports in the @@ -22,24 +37,91 @@ pub struct CliModuleLoader { /// read access errors must be raised based on the parent thread permissions. pub root_permissions: Permissions, pub ps: ProcState, + // the emit cache can only be used on one thread due to sqlite constraints + emit_cache: EmitCache, + emit_options: deno_ast::EmitOptions, + emit_options_hash: String, } impl CliModuleLoader { pub fn new(ps: ProcState) -> Rc { + let emit_cache = EmitCache::new(&ps.dir.emit_cache_db_file_path()); Rc::new(CliModuleLoader { lib: ps.options.ts_type_lib_window(), root_permissions: Permissions::allow_all(), ps, + emit_cache, }) } pub fn new_for_worker(ps: ProcState, permissions: Permissions) -> Rc { + let emit_cache = EmitCache::new(&ps.dir.emit_cache_db_file_path()); Rc::new(CliModuleLoader { lib: ps.options.ts_type_lib_worker(), root_permissions: permissions, ps, + emit_cache, }) } + + pub fn load_prepared_module( + &self, + specifier: &ModuleSpecifier, + ) -> Result { + let graph_data = self.ps.graph_data.read(); + let found_url = graph_data.follow_redirect(&specifier); + match graph_data.get(&found_url) { + Some(ModuleEntry::Module { + code, + media_type, + maybe_parsed_source, + .. + }) => { + let code = match media_type { + MediaType::JavaScript + | MediaType::Unknown + | MediaType::Cjs + | MediaType::Mjs + | MediaType::Json => { + if let Some(source) = graph_data.get_cjs_esm_translation(&specifier) + { + source.to_owned() + } else { + code.to_string() + } + } + MediaType::Dts | MediaType::Dcts | MediaType::Dmts => "".to_string(), + MediaType::TypeScript + | MediaType::Mts + | MediaType::Cts + | MediaType::Jsx + | MediaType::Tsx => { + // get emit text + let parsed_source = maybe_parsed_source.as_ref().unwrap(); // should always be set + emit_parsed_source( + &self.emit_cache, + &specifier, + parsed_source, + &self.ps.emit_options, + &self.ps.emit_options_hash, + )? + } + MediaType::TsBuildInfo | MediaType::Wasm | MediaType::SourceMap => { + panic!("Unexpected media type {} for {}", media_type, found_url) + } + }; + Ok(ModuleCodeSource { + code, + found_url, + media_type: *media_type, + }) + } + _ => Err(anyhow!( + "Loading unprepared module: {}", + specifier.to_string() + )), + } + } } impl ModuleLoader for CliModuleLoader { @@ -54,18 +136,37 @@ impl ModuleLoader for CliModuleLoader { fn load( &self, - module_specifier: &ModuleSpecifier, + specifier: &ModuleSpecifier, maybe_referrer: Option, is_dynamic: bool, ) -> Pin> { - let module_specifier = module_specifier.clone(); - let ps = self.ps.clone(); + log::debug!( + "specifier: {} maybe_referrer: {} is_dynamic: {}", + specifier, + maybe_referrer + .as_ref() + .map(|s| s.to_string()) + .unwrap_or_else(|| "".to_string()), + is_dynamic + ); // NOTE: this block is async only because of `deno_core` interface // requirements; module was already loaded when constructing module graph - // during call to `prepare_load`. - async move { ps.load(module_specifier, maybe_referrer, is_dynamic) } - .boxed_local() + // during call to `prepare_load` so we can load it synchronously. + let result = + self + .load_prepared_module(specifier) + .map(|code_source| ModuleSource { + code: code_source.code.into_bytes().into_boxed_slice(), + module_url_specified: specifier.to_string(), + module_url_found: code_source.found_url.to_string(), + module_type: match code_source.media_type { + MediaType::Json => ModuleType::Json, + _ => ModuleType::JavaScript, + }, + }); + + Box::pin(deno_core::futures::future::ready(result)) } fn prepare_load( @@ -103,3 +204,63 @@ impl ModuleLoader for CliModuleLoader { .boxed_local() } } + +impl SourceMapGetter for CliModuleLoader { + fn get_source_map(&self, file_name: &str) -> Option> { + if let Ok(specifier) = resolve_url(file_name) { + match specifier.scheme() { + // we should only be looking for emits for schemes that denote external + // modules, which the disk_cache supports + "wasm" | "file" | "http" | "https" | "data" | "blob" => (), + _ => return None, + } + if let Some(cache_data) = self.emit_cache.get_emit_data(&specifier) { + source_map_from_code(&cache_data.text) + .or_else(|| cache_data.map.map(|t| t.into_bytes())) + } else if let Ok(source) = self.load_prepared_module(&specifier) { + source_map_from_code(&source.code) + } else { + None + } + } else { + None + } + } + + fn get_source_line( + &self, + file_name: &str, + line_number: usize, + ) -> Option { + let graph_data = self.ps.graph_data.read(); + let specifier = graph_data.follow_redirect(&resolve_url(file_name).ok()?); + let code = match graph_data.get(&specifier) { + Some(ModuleEntry::Module { code, .. }) => code, + _ => return None, + }; + // Do NOT use .lines(): it skips the terminating empty line. + // (due to internally using_terminator() instead of .split()) + let lines: Vec<&str> = code.split('\n').collect(); + if line_number >= lines.len() { + Some(format!( + "{} Couldn't format source line: Line {} is out of bounds (source may have changed at runtime)", + crate::colors::yellow("Warning"), line_number + 1, + )) + } else { + Some(lines[line_number].to_string()) + } + } +} + +fn source_map_from_code(code: &str) -> Option> { + static PREFIX: &str = "//# sourceMappingURL=data:application/json;base64,"; + let last_line = code.rsplitn(2, |u| u == '\n').next().unwrap(); + if last_line.starts_with(PREFIX) { + let input = last_line.split_at(PREFIX.len()).1; + let decoded_map = base64::decode(input) + .expect("Unable to decode source map from emitted file."); + Some(decoded_map) + } else { + None + } +} diff --git a/cli/proc_state.rs b/cli/proc_state.rs index fac2b2418cd014..0402dd1be1801f 100644 --- a/cli/proc_state.rs +++ b/cli/proc_state.rs @@ -5,7 +5,6 @@ use crate::args::DenoSubcommand; use crate::args::Flags; use crate::args::TypeCheckMode; use crate::cache; -use crate::cache::EmitCache; use crate::cache::TypeCheckCache; use crate::compat; use crate::compat::NodeEsmResolver; @@ -70,7 +69,7 @@ pub struct Inner { pub coverage_dir: Option, pub file_fetcher: FileFetcher, pub options: Arc, - graph_data: Arc>, + pub graph_data: Arc>, pub lockfile: Option>>, pub maybe_import_map: Option>, pub maybe_inspector_server: Option>, @@ -300,7 +299,6 @@ impl ProcState { } } let mut cache = cache::FetchCacher::new( - self.dir.gen_cache.clone(), self.file_fetcher.clone(), root_permissions.clone(), dynamic_permissions.clone(), @@ -427,45 +425,28 @@ impl ProcState { log::warn!("{}", ignored_options); } - // start type checking if necessary - let type_checking_task = - if self.options.type_check_mode() != TypeCheckMode::None { - let maybe_config_specifier = self.options.maybe_config_file_specifier(); - let roots = roots.clone(); - let options = emit::CheckOptions { - type_check_mode: self.options.type_check_mode(), - debug: self.options.log_level() == Some(log::Level::Debug), - maybe_config_specifier, - ts_config: ts_config_result.ts_config.clone(), - log_checks: true, - reload: self.options.reload_flag() - && !roots.iter().all(|r| reload_exclusions.contains(&r.0)), - }; - // todo(THIS PR): don't use a cache on failure - let check_cache = - TypeCheckCache::new(&self.dir.type_checking_cache_db_file_path()); - let graph_data = self.graph_data.clone(); - Some(tokio::task::spawn_blocking(move || { - emit::check(&roots, graph_data, &check_cache, options) - })) - } else { - None + // type check if necessary + if self.options.type_check_mode() != TypeCheckMode::None { + let maybe_config_specifier = self.options.maybe_config_file_specifier(); + let roots = roots.clone(); + let options = emit::CheckOptions { + type_check_mode: self.options.type_check_mode(), + debug: self.options.log_level() == Some(log::Level::Debug), + maybe_config_specifier, + ts_config: ts_config_result.ts_config, + log_checks: true, + reload: self.options.reload_flag() + && !roots.iter().all(|r| reload_exclusions.contains(&r.0)), }; - - let options = emit::EmitOptions { - ts_config: ts_config_result.ts_config, - reload: self.options.reload_flag(), - reload_exclusions, - }; - let emit_result = emit::emit(&graph, &self.dir.gen_cache, options)?; - log::debug!("{}", emit_result.stats); - - if let Some(type_checking_task) = type_checking_task { - let type_check_result = type_checking_task.await??; - if !type_check_result.diagnostics.is_empty() { - return Err(anyhow!(type_check_result.diagnostics)); + let check_cache = + TypeCheckCache::new(&self.dir.type_checking_cache_db_file_path()); + let graph_data = self.graph_data.clone(); + let check_result = + emit::check(&roots, graph_data, &check_cache, options)?; + if !check_result.diagnostics.is_empty() { + return Err(anyhow!(check_result.diagnostics)); } - log::debug!("{}", type_check_result.stats); + log::debug!("{}", check_result.stats); } if self.options.type_check_mode() != TypeCheckMode::None { @@ -534,80 +515,11 @@ impl ProcState { } } - pub fn load( - &self, - specifier: ModuleSpecifier, - maybe_referrer: Option, - is_dynamic: bool, - ) -> Result { - log::debug!( - "specifier: {} maybe_referrer: {} is_dynamic: {}", - specifier, - maybe_referrer - .as_ref() - .map(|s| s.to_string()) - .unwrap_or_else(|| "".to_string()), - is_dynamic - ); - - let graph_data = self.graph_data.read(); - let found_url = graph_data.follow_redirect(&specifier); - match graph_data.get(&found_url) { - Some(ModuleEntry::Module { - code, media_type, .. - }) => { - let code = match media_type { - MediaType::JavaScript - | MediaType::Unknown - | MediaType::Cjs - | MediaType::Mjs - | MediaType::Json => { - if let Some(source) = graph_data.get_cjs_esm_translation(&specifier) - { - source.to_owned() - } else { - code.to_string() - } - } - MediaType::Dts | MediaType::Dcts | MediaType::Dmts => "".to_string(), - MediaType::TypeScript - | MediaType::Mts - | MediaType::Cts - | MediaType::Jsx - | MediaType::Tsx => { - let cached_text = self.dir.gen_cache.get_emit_text(&found_url); - match cached_text { - Some(text) => text, - None => unreachable!("Unexpected missing emit: {}\n\nTry reloading with the --reload CLI flag or deleting your DENO_DIR.", found_url), - } - } - MediaType::TsBuildInfo | MediaType::Wasm | MediaType::SourceMap => { - panic!("Unexpected media type {} for {}", media_type, found_url) - } - }; - Ok(ModuleSource { - code: code.into_bytes().into_boxed_slice(), - module_url_specified: specifier.to_string(), - module_url_found: found_url.to_string(), - module_type: match media_type { - MediaType::Json => ModuleType::Json, - _ => ModuleType::JavaScript, - }, - }) - } - _ => Err(anyhow!( - "Loading unprepared module: {}", - specifier.to_string() - )), - } - } - pub async fn create_graph( &self, roots: Vec<(ModuleSpecifier, ModuleKind)>, ) -> Result { let mut cache = cache::FetchCacher::new( - self.dir.gen_cache.clone(), self.file_fetcher.clone(), Permissions::allow_all(), Permissions::allow_all(), @@ -644,55 +556,6 @@ impl ProcState { } } -// TODO(@kitsonk) this is only temporary, but should be refactored to somewhere -// else, like a refactored file_fetcher. -impl SourceMapGetter for ProcState { - fn get_source_map(&self, file_name: &str) -> Option> { - if let Ok(specifier) = resolve_url(file_name) { - match specifier.scheme() { - // we should only be looking for emits for schemes that denote external - // modules, which the disk_cache supports - "wasm" | "file" | "http" | "https" | "data" | "blob" => (), - _ => return None, - } - if let Some(cache_data) = self.dir.gen_cache.get_emit_data(&specifier) { - source_map_from_code(cache_data.text.as_bytes()) - .or_else(|| cache_data.map.map(|t| t.into_bytes())) - } else if let Ok(source) = self.load(specifier, None, false) { - source_map_from_code(&source.code) - } else { - None - } - } else { - None - } - } - - fn get_source_line( - &self, - file_name: &str, - line_number: usize, - ) -> Option { - let graph_data = self.graph_data.read(); - let specifier = graph_data.follow_redirect(&resolve_url(file_name).ok()?); - let code = match graph_data.get(&specifier) { - Some(ModuleEntry::Module { code, .. }) => code, - _ => return None, - }; - // Do NOT use .lines(): it skips the terminating empty line. - // (due to internally using_terminator() instead of .split()) - let lines: Vec<&str> = code.split('\n').collect(); - if line_number >= lines.len() { - Some(format!( - "{} Couldn't format source line: Line {} is out of bounds (source may have changed at runtime)", - crate::colors::yellow("Warning"), line_number + 1, - )) - } else { - Some(lines[line_number].to_string()) - } - } -} - pub fn import_map_from_text( specifier: &Url, json_text: &str, @@ -717,19 +580,6 @@ pub fn import_map_from_text( Ok(result.import_map) } -fn source_map_from_code(code: &[u8]) -> Option> { - static PREFIX: &[u8] = b"//# sourceMappingURL=data:application/json;base64,"; - let last_line = code.rsplitn(2, |u| u == &b'\n').next().unwrap(); - if last_line.starts_with(PREFIX) { - let input = last_line.split_at(PREFIX.len()).1; - let decoded_map = base64::decode(input) - .expect("Unable to decode source map from emitted file."); - Some(decoded_map) - } else { - None - } -} - #[derive(Debug)] struct FileWatcherReporter { sender: tokio::sync::mpsc::UnboundedSender>, diff --git a/cli/tools/coverage/mod.rs b/cli/tools/coverage/mod.rs index d2c6c189433578..3140a4bb3fbbf1 100644 --- a/cli/tools/coverage/mod.rs +++ b/cli/tools/coverage/mod.rs @@ -5,6 +5,7 @@ use crate::args::Flags; use crate::cache::EmitCache; use crate::colors; use crate::fs_util::collect_files; +use crate::module_loader::CliModuleLoader; use crate::proc_state::ProcState; use crate::tools::fmt::format_json; @@ -605,6 +606,7 @@ pub async fn cover_files( coverage_flags: CoverageFlags, ) -> Result<(), AnyError> { let ps = ProcState::build(flags).await?; + let module_loader = CliModuleLoader::new(ps.clone()); let script_coverages = collect_coverages(coverage_flags.files, coverage_flags.ignore)?; @@ -665,36 +667,10 @@ pub async fn cover_files( })?; // Check if file was transpiled - let transpiled_source = match file.media_type { - MediaType::JavaScript - | MediaType::Unknown - | MediaType::Cjs - | MediaType::Mjs - | MediaType::Json => file.source.as_ref().to_string(), - MediaType::Dts | MediaType::Dmts | MediaType::Dcts => "".to_string(), - MediaType::TypeScript - | MediaType::Jsx - | MediaType::Mts - | MediaType::Cts - | MediaType::Tsx => { - match ps.dir.gen_cache.get_emit_text(&file.specifier) { - Some(source) => source, - None => { - return Err(anyhow!( - "Missing transpiled source code for: \"{}\". - Before generating coverage report, run `deno test --coverage` to ensure consistent state.", - file.specifier, - )) - } - } - } - MediaType::Wasm | MediaType::TsBuildInfo | MediaType::SourceMap => { - unreachable!() - } - }; - + let transpiled_source = + module_loader.load_prepared_module(&file.specifier)?.code; let original_source = &file.source; - let maybe_source_map = ps.get_source_map(&script_coverage.url); + let maybe_source_map = module_loader.get_source_map(&script_coverage.url); let coverage_report = generate_coverage_report( &script_coverage, diff --git a/core/source_map.rs b/core/source_map.rs index 6a261fa7d6f82d..0df58c4be496c8 100644 --- a/core/source_map.rs +++ b/core/source_map.rs @@ -5,6 +5,7 @@ use crate::resolve_url; pub use sourcemap::SourceMap; use std::collections::HashMap; +use std::rc::Rc; use std::str; pub trait SourceMapGetter { @@ -17,6 +18,23 @@ pub trait SourceMapGetter { ) -> Option; } +impl SourceMapGetter for Rc +where + T: SourceMapGetter, +{ + fn get_source_map(&self, file_name: &str) -> Option> { + (**self).get_source_map(file_name) + } + + fn get_source_line( + &self, + file_name: &str, + line_number: usize, + ) -> Option { + (**self).get_source_line(file_name, line_number) + } +} + #[derive(Debug, Default)] pub struct SourceMapCache { maps: HashMap>, From 6e9707bbdc05dbadb7910d916a09dbb6bb677d61 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Wed, 13 Jul 2022 16:35:18 -0400 Subject: [PATCH 02/13] More changes. --- cli/cache/common.rs | 35 ++++++++++++++---- cli/cache/disk_cache.rs | 78 --------------------------------------- cli/cache/incremental.rs | 15 +++++--- cli/cache/mod.rs | 33 +---------------- cli/emit.rs | 50 +++++-------------------- cli/module_loader.rs | 4 +- cli/proc_state.rs | 37 +++++++++---------- cli/tools/coverage/mod.rs | 36 ++++++++++++++---- 8 files changed, 95 insertions(+), 193 deletions(-) diff --git a/cli/cache/common.rs b/cli/cache/common.rs index c01c1ab9aaf016..6a806a70fe6eed 100644 --- a/cli/cache/common.rs +++ b/cli/cache/common.rs @@ -1,16 +1,37 @@ // Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. +use std::hash::Hasher; + use deno_core::error::AnyError; use deno_runtime::deno_webstorage::rusqlite::Connection; -/// Very fast non-cryptographically secure hash. -pub fn fast_insecure_hash(bytes: &[u8]) -> u64 { - use std::hash::Hasher; - use twox_hash::XxHash64; +/// A very fast insecure hash that uses the xxHash algorithm. +#[derive(Default)] +pub struct FastInsecureHash(twox_hash::XxHash64); + +impl FastInsecureHash { + pub fn new() -> Self { + Self::default() + } + + pub fn write_str(&mut self, text: &str) -> &mut Self { + self.write(text.as_bytes()); + self + } + + pub fn write(&mut self, bytes: &[u8]) -> &mut Self { + self.0.write(bytes); + self + } + + pub fn write_u64(&mut self, value: u64) -> &mut Self { + self.0.write_u64(value); + self + } - let mut hasher = XxHash64::default(); - hasher.write(bytes); - hasher.finish() + pub fn finish(&self) -> u64 { + self.0.finish() + } } /// Runs the common sqlite pragma. diff --git a/cli/cache/disk_cache.rs b/cli/cache/disk_cache.rs index 01352c3989bbc7..5a2f11e3c3a004 100644 --- a/cli/cache/disk_cache.rs +++ b/cli/cache/disk_cache.rs @@ -3,13 +3,6 @@ use crate::fs_util; use crate::http_cache::url_to_filename; -use super::CacheType; -use super::Cacher; -use super::EmitMetadata; - -use deno_ast::ModuleSpecifier; -use deno_core::error::AnyError; -use deno_core::serde_json; use deno_core::url::Host; use deno_core::url::Url; use std::ffi::OsStr; @@ -154,77 +147,6 @@ impl DiskCache { fs_util::atomic_write_file(&path, data, crate::http_cache::CACHE_PERM) .map_err(|e| with_io_context(&e, format!("{:#?}", &path))) } - - fn get_emit_metadata( - &self, - specifier: &ModuleSpecifier, - ) -> Option { - let filename = self.get_cache_filename_with_extension(specifier, "meta")?; - let bytes = self.get(&filename).ok()?; - serde_json::from_slice(&bytes).ok() - } - - fn set_emit_metadata( - &self, - specifier: &ModuleSpecifier, - data: EmitMetadata, - ) -> Result<(), AnyError> { - let filename = self - .get_cache_filename_with_extension(specifier, "meta") - .unwrap(); - let bytes = serde_json::to_vec(&data)?; - self.set(&filename, &bytes).map_err(|e| e.into()) - } -} - -// todo(13302): remove and replace with sqlite database -impl Cacher for DiskCache { - fn get( - &self, - cache_type: CacheType, - specifier: &ModuleSpecifier, - ) -> Option { - let extension = match cache_type { - CacheType::Emit => "js", - CacheType::SourceMap => "js.map", - CacheType::Version => { - return self.get_emit_metadata(specifier).map(|d| d.version_hash) - } - }; - let filename = - self.get_cache_filename_with_extension(specifier, extension)?; - self - .get(&filename) - .ok() - .and_then(|b| String::from_utf8(b).ok()) - } - - fn set( - &self, - cache_type: CacheType, - specifier: &ModuleSpecifier, - value: String, - ) -> Result<(), AnyError> { - let extension = match cache_type { - CacheType::Emit => "js", - CacheType::SourceMap => "js.map", - CacheType::Version => { - let data = if let Some(mut data) = self.get_emit_metadata(specifier) { - data.version_hash = value; - data - } else { - EmitMetadata { - version_hash: value, - } - }; - return self.set_emit_metadata(specifier, data); - } - }; - let filename = self - .get_cache_filename_with_extension(specifier, extension) - .unwrap(); - self.set(&filename, value.as_bytes()).map_err(|e| e.into()) - } } #[cfg(test)] diff --git a/cli/cache/incremental.rs b/cli/cache/incremental.rs index b5fff0734b3b97..e87b47295e8a7b 100644 --- a/cli/cache/incremental.rs +++ b/cli/cache/incremental.rs @@ -12,8 +12,8 @@ use deno_runtime::deno_webstorage::rusqlite::Connection; use serde::Serialize; use tokio::task::JoinHandle; -use super::common::fast_insecure_hash; use super::common::run_sqlite_pragma; +use super::common::FastInsecureHash; /// Cache used to skip formatting/linting a file again when we /// know it is already formatted or has no lint diagnostics. @@ -79,8 +79,9 @@ impl IncrementalCacheInner { state: &TState, initial_file_paths: &[PathBuf], ) -> Result { - let state_hash = - fast_insecure_hash(serde_json::to_string(state).unwrap().as_bytes()); + let state_hash = FastInsecureHash::new() + .write_str(&serde_json::to_string(state).unwrap()) + .finish(); let sql_cache = SqlIncrementalCache::new(db_file_path, state_hash)?; Ok(Self::from_sql_incremental_cache( sql_cache, @@ -123,13 +124,15 @@ impl IncrementalCacheInner { pub fn is_file_same(&self, file_path: &Path, file_text: &str) -> bool { match self.previous_hashes.get(file_path) { - Some(hash) => *hash == fast_insecure_hash(file_text.as_bytes()), + Some(hash) => { + *hash == FastInsecureHash::new().write_str(file_text).finish() + } None => false, } } pub fn update_file(&self, file_path: &Path, file_text: &str) { - let hash = fast_insecure_hash(file_text.as_bytes()); + let hash = FastInsecureHash::new().write_str(file_text).finish(); if let Some(previous_hash) = self.previous_hashes.get(file_path) { if *previous_hash == hash { return; // do not bother updating the db file because nothing has changed @@ -334,7 +337,7 @@ mod test { .unwrap(); let file_path = PathBuf::from("/mod.ts"); let file_text = "test"; - let file_hash = fast_insecure_hash(file_text.as_bytes()); + let file_hash = FastInsecureHash::new().write_str(file_text).finish(); sql_cache.set_source_hash(&file_path, file_hash).unwrap(); let cache = IncrementalCacheInner::from_sql_incremental_cache( sql_cache, diff --git a/cli/cache/mod.rs b/cli/cache/mod.rs index 0cbc019044a433..d535edb688d873 100644 --- a/cli/cache/mod.rs +++ b/cli/cache/mod.rs @@ -3,10 +3,7 @@ use crate::errors::get_error_class_name; use crate::file_fetcher::FileFetcher; -use deno_core::error::AnyError; use deno_core::futures::FutureExt; -use deno_core::serde::Deserialize; -use deno_core::serde::Serialize; use deno_core::ModuleSpecifier; use deno_graph::source::CacheInfo; use deno_graph::source::LoadFuture; @@ -22,40 +19,12 @@ mod emit; mod incremental; pub use check::TypeCheckCache; +pub use common::FastInsecureHash; pub use disk_cache::DiskCache; pub use emit::EmitCache; pub use emit::SpecifierEmitCacheData; pub use incremental::IncrementalCache; -#[derive(Debug, Deserialize, Serialize)] -pub struct EmitMetadata { - pub version_hash: String, -} - -pub enum CacheType { - Emit, - SourceMap, - Version, -} - -/// A trait which provides a concise implementation to getting and setting -/// values in a cache. -pub trait Cacher { - /// Get a value from the cache. - fn get( - &self, - cache_type: CacheType, - specifier: &ModuleSpecifier, - ) -> Option; - /// Set a value in the cache. - fn set( - &self, - cache_type: CacheType, - specifier: &ModuleSpecifier, - value: String, - ) -> Result<(), AnyError>; -} - /// A "wrapper" for the FileFetcher and DiskCache for the Deno CLI that provides /// a concise interface to the DENO_DIR when building module graphs. pub struct FetchCacher { diff --git a/cli/emit.rs b/cli/emit.rs index 0436c7dc46e06e..cabef3a925592e 100644 --- a/cli/emit.rs +++ b/cli/emit.rs @@ -10,6 +10,7 @@ use crate::args::EmitConfigOptions; use crate::args::TsConfig; use crate::args::TypeCheckMode; use crate::cache::EmitCache; +use crate::cache::FastInsecureHash; use crate::cache::SpecifierEmitCacheData; use crate::cache::TypeCheckCache; use crate::colors; @@ -115,8 +116,8 @@ pub enum TsConfigType { /// Return a configuration for bundling, using swc to emit the bundle. This is /// independent of type checking. Bundle, - /// Return a configuration to use tsc to type check and optionally emit. This - /// is independent of either bundling or just emitting via swc + /// Return a configuration to use tsc to type check. This + /// is independent of either bundling or emitting via swc. Check { lib: TsTypeLib }, /// Return a configuration to use swc to emit single module files. Emit, @@ -248,12 +249,13 @@ pub fn emit_parsed_source( specifier: &ModuleSpecifier, parsed_source: &ParsedSource, emit_options: &deno_ast::EmitOptions, - emit_config_hash: &str, + emit_config_hash: u64, ) -> Result { - let source_hash = get_source_hash_for_emit( - parsed_source.text_info().text_str(), - emit_config_hash, - ); + let source_hash = FastInsecureHash::new() + .write_str(parsed_source.text_info().text_str()) + .write_u64(emit_config_hash) + .finish(); + if let Some(emit_data) = cache.get_emit_data(specifier) { if emit_data.source_hash == source_hash { return Ok(emit_data.text); @@ -269,40 +271,6 @@ pub fn emit_parsed_source( Ok(cache_data.text) } -/// A hashing function that takes the source code, version and optionally a -/// user provided config and generates a string hash which can be stored to -/// determine if the cached emit is valid or not. -fn get_source_hash_for_emit(source_text: &str, emit_config_hash: &str) -> u64 { - // twox hash is insecure, but fast so it works for our purposes - use std::hash::Hasher; - use twox_hash::XxHash64; - - let mut hasher = XxHash64::default(); - hasher.write(source_text.as_bytes()); - hasher.write(emit_config_hash.as_bytes()); - hasher.finish() -} - -/// Determine if a given module kind and media type is emittable or not. -pub fn is_emittable( - kind: &ModuleKind, - media_type: &MediaType, - include_js: bool, -) -> bool { - if matches!(kind, ModuleKind::Synthetic) { - return false; - } - match &media_type { - MediaType::TypeScript - | MediaType::Mts - | MediaType::Cts - | MediaType::Tsx - | MediaType::Jsx => true, - MediaType::JavaScript | MediaType::Mjs | MediaType::Cjs => include_js, - _ => false, - } -} - /// Options for performing a check of a module graph. Note that the decision to /// emit or not is determined by the `ts_config` settings. pub struct CheckOptions { diff --git a/cli/module_loader.rs b/cli/module_loader.rs index db6ba8abac6957..147cdcc0ab49c8 100644 --- a/cli/module_loader.rs +++ b/cli/module_loader.rs @@ -39,8 +39,6 @@ pub struct CliModuleLoader { pub ps: ProcState, // the emit cache can only be used on one thread due to sqlite constraints emit_cache: EmitCache, - emit_options: deno_ast::EmitOptions, - emit_options_hash: String, } impl CliModuleLoader { @@ -103,7 +101,7 @@ impl CliModuleLoader { &specifier, parsed_source, &self.ps.emit_options, - &self.ps.emit_options_hash, + self.ps.emit_options_hash, )? } MediaType::TsBuildInfo | MediaType::Wasm | MediaType::SourceMap => { diff --git a/cli/proc_state.rs b/cli/proc_state.rs index a5c20f9897064e..2758cac5413d0f 100644 --- a/cli/proc_state.rs +++ b/cli/proc_state.rs @@ -5,6 +5,7 @@ use crate::args::DenoSubcommand; use crate::args::Flags; use crate::args::TypeCheckMode; use crate::cache; +use crate::cache::FastInsecureHash; use crate::cache::TypeCheckCache; use crate::compat; use crate::compat::NodeEsmResolver; @@ -22,7 +23,6 @@ use crate::lockfile::Lockfile; use crate::resolver::ImportMapResolver; use crate::resolver::JsxResolver; -use deno_ast::MediaType; use deno_core::anyhow::anyhow; use deno_core::anyhow::Context; use deno_core::error::custom_error; @@ -30,14 +30,10 @@ use deno_core::error::AnyError; use deno_core::futures; use deno_core::parking_lot::Mutex; use deno_core::parking_lot::RwLock; -use deno_core::resolve_url; use deno_core::url::Url; use deno_core::CompiledWasmModuleStore; -use deno_core::ModuleSource; use deno_core::ModuleSpecifier; -use deno_core::ModuleType; use deno_core::SharedArrayBufferStore; -use deno_core::SourceMapGetter; use deno_graph::create_graph; use deno_graph::source::CacheInfo; use deno_graph::source::LoadFuture; @@ -69,6 +65,8 @@ pub struct Inner { pub coverage_dir: Option, pub file_fetcher: FileFetcher, pub options: Arc, + pub emit_options: deno_ast::EmitOptions, + pub emit_options_hash: u64, pub graph_data: Arc>, pub lockfile: Option>>, pub maybe_import_map: Option>, @@ -210,10 +208,21 @@ impl ProcState { file_paths: Arc::new(Mutex::new(vec![])), }); + let ts_config_result = + cli_options.resolve_ts_config_for_emit(TsConfigType::Emit)?; + if let Some(ignored_options) = ts_config_result.maybe_ignored_options { + log::warn!("{}", ignored_options); + } + Ok(ProcState(Arc::new(Inner { dir, coverage_dir, options: cli_options, + emit_options_hash: FastInsecureHash::new() + // todo: use hash of emit options instead as it's more specific + .write(&ts_config_result.ts_config.as_bytes()) + .finish(), + emit_options: ts_config_result.ts_config.into(), file_fetcher, graph_data: Default::default(), lockfile, @@ -409,19 +418,6 @@ impl ProcState { .unwrap()?; } - let config_type = if self.options.type_check_mode() == TypeCheckMode::None { - TsConfigType::Emit - } else { - TsConfigType::Check { lib } - }; - - let ts_config_result = - self.options.resolve_ts_config_for_emit(config_type)?; - - if let Some(ignored_options) = ts_config_result.maybe_ignored_options { - log::warn!("{}", ignored_options); - } - // type check if necessary if self.options.type_check_mode() != TypeCheckMode::None { let maybe_config_specifier = self.options.maybe_config_file_specifier(); @@ -430,7 +426,10 @@ impl ProcState { type_check_mode: self.options.type_check_mode(), debug: self.options.log_level() == Some(log::Level::Debug), maybe_config_specifier, - ts_config: ts_config_result.ts_config, + ts_config: self + .options + .resolve_ts_config_for_emit(TsConfigType::Check { lib })? + .ts_config, log_checks: true, reload: self.options.reload_flag() && !roots.iter().all(|r| reload_exclusions.contains(&r.0)), diff --git a/cli/tools/coverage/mod.rs b/cli/tools/coverage/mod.rs index 3140a4bb3fbbf1..97e3b587464b0c 100644 --- a/cli/tools/coverage/mod.rs +++ b/cli/tools/coverage/mod.rs @@ -5,7 +5,6 @@ use crate::args::Flags; use crate::cache::EmitCache; use crate::colors; use crate::fs_util::collect_files; -use crate::module_loader::CliModuleLoader; use crate::proc_state::ProcState; use crate::tools::fmt::format_json; @@ -18,7 +17,6 @@ use deno_core::serde_json; use deno_core::sourcemap::SourceMap; use deno_core::url::Url; use deno_core::LocalInspectorSession; -use deno_core::SourceMapGetter; use regex::Regex; use std::fs; use std::fs::File; @@ -606,7 +604,7 @@ pub async fn cover_files( coverage_flags: CoverageFlags, ) -> Result<(), AnyError> { let ps = ProcState::build(flags).await?; - let module_loader = CliModuleLoader::new(ps.clone()); + let emit_cache = EmitCache::new(&ps.dir.emit_cache_db_file_path()); let script_coverages = collect_coverages(coverage_flags.files, coverage_flags.ignore)?; @@ -667,15 +665,39 @@ pub async fn cover_files( })?; // Check if file was transpiled - let transpiled_source = - module_loader.load_prepared_module(&file.specifier)?.code; + let (transpiled_source, maybe_source_map) = match file.media_type { + MediaType::JavaScript + | MediaType::Unknown + | MediaType::Cjs + | MediaType::Mjs + | MediaType::Json => (file.source.as_ref().to_string(), None), + MediaType::Dts | MediaType::Dmts | MediaType::Dcts => ("".to_string(), None), + MediaType::TypeScript + | MediaType::Jsx + | MediaType::Mts + | MediaType::Cts + | MediaType::Tsx => { + match emit_cache.get_emit_data(&file.specifier) { + Some(data) => (data.text, data.map), + None => { + return Err(anyhow!( + "Missing transpiled source code for: \"{}\". + Before generating coverage report, run `deno test --coverage` to ensure consistent state.", + file.specifier, + )) + } + } + } + MediaType::Wasm | MediaType::TsBuildInfo | MediaType::SourceMap => { + unreachable!() + } + }; let original_source = &file.source; - let maybe_source_map = module_loader.get_source_map(&script_coverage.url); let coverage_report = generate_coverage_report( &script_coverage, &transpiled_source, - &maybe_source_map, + &maybe_source_map.map(|t| t.into_bytes()), &out_mode, ); From 6a0c19104c05a93ce80c70ec4d0d64f88da0da1f Mon Sep 17 00:00:00 2001 From: David Sherret Date: Wed, 13 Jul 2022 16:53:41 -0400 Subject: [PATCH 03/13] Fixes. --- cli/cache/emit.rs | 4 ++-- cli/emit.rs | 33 ---------------------------- cli/tests/integration/info_tests.rs | 34 ----------------------------- 3 files changed, 2 insertions(+), 69 deletions(-) diff --git a/cli/cache/emit.rs b/cli/cache/emit.rs index 1f18930f94c117..b4cffb3babe290 100644 --- a/cli/cache/emit.rs +++ b/cli/cache/emit.rs @@ -86,7 +86,7 @@ impl EmitCache { let row = rows.next().ok().flatten()?; Some(SpecifierEmitCacheData { - source_hash: row.get(0).ok()?, + source_hash: row.get::(0).ok()?.parse::().ok()?, text: row.get(1).ok()?, map: row.get(2).ok()?, }) @@ -122,7 +122,7 @@ impl EmitCache { )?; stmt.execute(params![ specifier.to_string(), - &data.source_hash, + &data.source_hash.to_string(), &data.text, &data.map, ])?; diff --git a/cli/emit.rs b/cli/emit.rs index cabef3a925592e..55d0fb32e9f7a0 100644 --- a/cli/emit.rs +++ b/cli/emit.rs @@ -567,36 +567,3 @@ impl From for deno_ast::EmitOptions { } } } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_is_emittable() { - assert!(is_emittable( - &ModuleKind::Esm, - &MediaType::TypeScript, - false - )); - assert!(!is_emittable( - &ModuleKind::Synthetic, - &MediaType::TypeScript, - false - )); - assert!(!is_emittable(&ModuleKind::Esm, &MediaType::Dts, false)); - assert!(!is_emittable(&ModuleKind::Esm, &MediaType::Dcts, false)); - assert!(!is_emittable(&ModuleKind::Esm, &MediaType::Dmts, false)); - assert!(is_emittable(&ModuleKind::Esm, &MediaType::Tsx, false)); - assert!(!is_emittable( - &ModuleKind::Esm, - &MediaType::JavaScript, - false - )); - assert!(!is_emittable(&ModuleKind::Esm, &MediaType::Cjs, false)); - assert!(!is_emittable(&ModuleKind::Esm, &MediaType::Mjs, false)); - assert!(is_emittable(&ModuleKind::Esm, &MediaType::JavaScript, true)); - assert!(is_emittable(&ModuleKind::Esm, &MediaType::Jsx, false)); - assert!(!is_emittable(&ModuleKind::Esm, &MediaType::Json, false)); - } -} diff --git a/cli/tests/integration/info_tests.rs b/cli/tests/integration/info_tests.rs index e02bb8ff22e10b..eb4a73b6223d8f 100644 --- a/cli/tests/integration/info_tests.rs +++ b/cli/tests/integration/info_tests.rs @@ -1,40 +1,6 @@ // Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. use crate::itest; -use test_util as util; -use test_util::TempDir; - -#[test] -fn info_with_compiled_source() { - let _g = util::http_server(); - let module_path = "http://127.0.0.1:4545/048_media_types_jsx.ts"; - let t = TempDir::new(); - - let mut deno = util::deno_cmd() - .env("DENO_DIR", t.path()) - .current_dir(util::testdata_path()) - .arg("cache") - .arg(&module_path) - .spawn() - .unwrap(); - let status = deno.wait().unwrap(); - assert!(status.success()); - - let output = util::deno_cmd() - .env("DENO_DIR", t.path()) - .env("NO_COLOR", "1") - .current_dir(util::testdata_path()) - .arg("info") - .arg(&module_path) - .output() - .unwrap(); - - let str_output = std::str::from_utf8(&output.stdout).unwrap().trim(); - eprintln!("{}", str_output); - // check the output of the test.ts program. - assert!(str_output.contains("emit: ")); - assert_eq!(output.stderr, b""); -} itest!(_022_info_flag_script { args: "info http://127.0.0.1:4545/019_media_types.ts", From 250722f8edad27eb75d0eee1ddff79dc42235c0e Mon Sep 17 00:00:00 2001 From: David Sherret Date: Wed, 13 Jul 2022 17:05:32 -0400 Subject: [PATCH 04/13] Clippy. --- cli/emit.rs | 2 +- cli/module_loader.rs | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cli/emit.rs b/cli/emit.rs index 55d0fb32e9f7a0..4ae42333ab5345 100644 --- a/cli/emit.rs +++ b/cli/emit.rs @@ -261,7 +261,7 @@ pub fn emit_parsed_source( return Ok(emit_data.text); } } - let transpiled_source = parsed_source.transpile(&emit_options)?; + let transpiled_source = parsed_source.transpile(emit_options)?; let cache_data = SpecifierEmitCacheData { source_hash, text: transpiled_source.text, diff --git a/cli/module_loader.rs b/cli/module_loader.rs index 147cdcc0ab49c8..10d3c206197bfe 100644 --- a/cli/module_loader.rs +++ b/cli/module_loader.rs @@ -67,7 +67,7 @@ impl CliModuleLoader { specifier: &ModuleSpecifier, ) -> Result { let graph_data = self.ps.graph_data.read(); - let found_url = graph_data.follow_redirect(&specifier); + let found_url = graph_data.follow_redirect(specifier); match graph_data.get(&found_url) { Some(ModuleEntry::Module { code, @@ -81,7 +81,7 @@ impl CliModuleLoader { | MediaType::Cjs | MediaType::Mjs | MediaType::Json => { - if let Some(source) = graph_data.get_cjs_esm_translation(&specifier) + if let Some(source) = graph_data.get_cjs_esm_translation(specifier) { source.to_owned() } else { @@ -98,7 +98,7 @@ impl CliModuleLoader { let parsed_source = maybe_parsed_source.as_ref().unwrap(); // should always be set emit_parsed_source( &self.emit_cache, - &specifier, + specifier, parsed_source, &self.ps.emit_options, self.ps.emit_options_hash, @@ -252,7 +252,7 @@ impl SourceMapGetter for CliModuleLoader { fn source_map_from_code(code: &str) -> Option> { static PREFIX: &str = "//# sourceMappingURL=data:application/json;base64,"; - let last_line = code.rsplitn(2, |u| u == '\n').next().unwrap(); + let last_line = code.rsplit(|u| u == '\n').next().unwrap(); if last_line.starts_with(PREFIX) { let input = last_line.split_at(PREFIX.len()).1; let decoded_map = base64::decode(input) From 85fc0be453b8afa12159f35735e58ffbc643aacd Mon Sep 17 00:00:00 2001 From: David Sherret Date: Wed, 13 Jul 2022 17:23:51 -0400 Subject: [PATCH 05/13] Verify the source hash in the cache in order to encourage consumers to verify they're getting the right source --- cli/cache/emit.rs | 21 +++++++++++++++++---- cli/emit.rs | 27 ++++++++++++++++----------- cli/module_loader.rs | 7 ++++++- cli/tools/coverage/mod.rs | 11 ++++++++--- 4 files changed, 47 insertions(+), 19 deletions(-) diff --git a/cli/cache/emit.rs b/cli/cache/emit.rs index b4cffb3babe290..f42baa0987206b 100644 --- a/cli/cache/emit.rs +++ b/cli/cache/emit.rs @@ -12,7 +12,6 @@ use super::common::run_sqlite_pragma; /// Emit cache for a single file. #[derive(Debug, Clone, PartialEq)] pub struct SpecifierEmitCacheData { - pub source_hash: u64, pub text: String, pub map: Option, } @@ -71,9 +70,14 @@ impl EmitCache { } /// Gets the emit data from the cache. + /// + /// Ideally, you SHOULD provide an expected source hash in order + /// to verify that you're getting a value from the cache + /// that is relevant to the source. pub fn get_emit_data( &self, specifier: &ModuleSpecifier, + maybe_expected_source_hash: Option, ) -> Option { let conn = match &self.0 { Some(conn) => conn, @@ -85,8 +89,15 @@ impl EmitCache { let mut rows = stmt.query(params![specifier.to_string()]).ok()?; let row = rows.next().ok().flatten()?; + if let Some(expected_hash) = maybe_expected_source_hash { + // verify that the emit is for the source + let saved_hash = row.get::(0).ok()?; + if saved_hash != expected_hash.to_string() { + return None; + } + } + Some(SpecifierEmitCacheData { - source_hash: row.get::(0).ok()?.parse::().ok()?, text: row.get(1).ok()?, map: row.get(2).ok()?, }) @@ -96,9 +107,10 @@ impl EmitCache { pub fn set_emit_data( &self, specifier: &ModuleSpecifier, + source_hash: u64, data: &SpecifierEmitCacheData, ) { - if let Err(err) = self.set_emit_data_result(specifier, data) { + if let Err(err) = self.set_emit_data_result(specifier, source_hash, data) { // should never error here, but if it ever does don't fail if cfg!(debug_assertions) { panic!("Error saving emit data: {}", err); @@ -111,6 +123,7 @@ impl EmitCache { fn set_emit_data_result( &self, specifier: &ModuleSpecifier, + source_hash: u64, data: &SpecifierEmitCacheData, ) -> Result<(), AnyError> { let conn = match &self.0 { @@ -122,7 +135,7 @@ impl EmitCache { )?; stmt.execute(params![ specifier.to_string(), - &data.source_hash.to_string(), + source_hash.to_string(), &data.text, &data.map, ])?; diff --git a/cli/emit.rs b/cli/emit.rs index 4ae42333ab5345..032da9f9595285 100644 --- a/cli/emit.rs +++ b/cli/emit.rs @@ -244,6 +244,16 @@ fn get_tsc_roots( } } +/// A hashing function that takes the source code, version and optionally a +/// user provided config and generates a string hash which can be stored to +/// determine if the cached emit is valid or not. +pub fn get_source_hash(source_text: &str, emit_options_hash: u64) -> u64 { + FastInsecureHash::new() + .write_str(source_text) + .write_u64(emit_options_hash) + .finish() +} + pub fn emit_parsed_source( cache: &EmitCache, specifier: &ModuleSpecifier, @@ -251,23 +261,18 @@ pub fn emit_parsed_source( emit_options: &deno_ast::EmitOptions, emit_config_hash: u64, ) -> Result { - let source_hash = FastInsecureHash::new() - .write_str(parsed_source.text_info().text_str()) - .write_u64(emit_config_hash) - .finish(); - - if let Some(emit_data) = cache.get_emit_data(specifier) { - if emit_data.source_hash == source_hash { - return Ok(emit_data.text); - } + let source_hash = + get_source_hash(parsed_source.text_info().text_str(), emit_config_hash); + + if let Some(emit_data) = cache.get_emit_data(specifier, Some(source_hash)) { + return Ok(emit_data.text); } let transpiled_source = parsed_source.transpile(emit_options)?; let cache_data = SpecifierEmitCacheData { - source_hash, text: transpiled_source.text, map: transpiled_source.source_map, }; - cache.set_emit_data(specifier, &cache_data); + cache.set_emit_data(specifier, source_hash, &cache_data); Ok(cache_data.text) } diff --git a/cli/module_loader.rs b/cli/module_loader.rs index 10d3c206197bfe..6681357385cc7b 100644 --- a/cli/module_loader.rs +++ b/cli/module_loader.rs @@ -212,7 +212,12 @@ impl SourceMapGetter for CliModuleLoader { "wasm" | "file" | "http" | "https" | "data" | "blob" => (), _ => return None, } - if let Some(cache_data) = self.emit_cache.get_emit_data(&specifier) { + if let Some(cache_data) = self.emit_cache.get_emit_data( + &specifier, + // ideally we would provide a source hash here in order to verify that + // we're getting a source map for the file currently in the cache + None, + ) { source_map_from_code(&cache_data.text) .or_else(|| cache_data.map.map(|t| t.into_bytes())) } else if let Ok(source) = self.load_prepared_module(&specifier) { diff --git a/cli/tools/coverage/mod.rs b/cli/tools/coverage/mod.rs index 97e3b587464b0c..d847fe4281586a 100644 --- a/cli/tools/coverage/mod.rs +++ b/cli/tools/coverage/mod.rs @@ -4,6 +4,7 @@ use crate::args::CoverageFlags; use crate::args::Flags; use crate::cache::EmitCache; use crate::colors; +use crate::emit::get_source_hash; use crate::fs_util::collect_files; use crate::proc_state::ProcState; use crate::tools::fmt::format_json; @@ -665,19 +666,24 @@ pub async fn cover_files( })?; // Check if file was transpiled + let original_source = &file.source; let (transpiled_source, maybe_source_map) = match file.media_type { MediaType::JavaScript | MediaType::Unknown | MediaType::Cjs | MediaType::Mjs | MediaType::Json => (file.source.as_ref().to_string(), None), - MediaType::Dts | MediaType::Dmts | MediaType::Dcts => ("".to_string(), None), + MediaType::Dts | MediaType::Dmts | MediaType::Dcts => { + ("".to_string(), None) + } MediaType::TypeScript | MediaType::Jsx | MediaType::Mts | MediaType::Cts | MediaType::Tsx => { - match emit_cache.get_emit_data(&file.specifier) { + let source_hash = + get_source_hash(original_source, ps.emit_options_hash); + match emit_cache.get_emit_data(&file.specifier, Some(source_hash)) { Some(data) => (data.text, data.map), None => { return Err(anyhow!( @@ -692,7 +698,6 @@ pub async fn cover_files( unreachable!() } }; - let original_source = &file.source; let coverage_report = generate_coverage_report( &script_coverage, From a4e06151b4ee9a7ea7414e240292a44027bc10a3 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Thu, 14 Jul 2022 11:16:01 -0400 Subject: [PATCH 06/13] Fix coverage code. --- cli/module_loader.rs | 14 +------------- cli/text_encoding.rs | 13 +++++++++++++ cli/tools/coverage/mod.rs | 8 ++++++-- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/cli/module_loader.rs b/cli/module_loader.rs index 6681357385cc7b..75408dc7ab231e 100644 --- a/cli/module_loader.rs +++ b/cli/module_loader.rs @@ -5,6 +5,7 @@ use crate::emit::emit_parsed_source; use crate::emit::TsTypeLib; use crate::graph_util::ModuleEntry; use crate::proc_state::ProcState; +use crate::text_encoding::source_map_from_code; use deno_ast::MediaType; use deno_core::anyhow::anyhow; @@ -254,16 +255,3 @@ impl SourceMapGetter for CliModuleLoader { } } } - -fn source_map_from_code(code: &str) -> Option> { - static PREFIX: &str = "//# sourceMappingURL=data:application/json;base64,"; - let last_line = code.rsplit(|u| u == '\n').next().unwrap(); - if last_line.starts_with(PREFIX) { - let input = last_line.split_at(PREFIX.len()).1; - let decoded_map = base64::decode(input) - .expect("Unable to decode source map from emitted file."); - Some(decoded_map) - } else { - None - } -} diff --git a/cli/text_encoding.rs b/cli/text_encoding.rs index 392bab7b832146..f4a29a5edc9423 100644 --- a/cli/text_encoding.rs +++ b/cli/text_encoding.rs @@ -54,6 +54,19 @@ pub fn strip_bom(text: &str) -> &str { } } +pub fn source_map_from_code(code: &str) -> Option> { + static PREFIX: &str = "//# sourceMappingURL=data:application/json;base64,"; + let last_line = code.rsplit(|u| u == '\n').next().unwrap(); + if last_line.starts_with(PREFIX) { + let input = last_line.split_at(PREFIX.len()).1; + let decoded_map = base64::decode(input) + .expect("Unable to decode source map from emitted file."); + Some(decoded_map) + } else { + None + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/cli/tools/coverage/mod.rs b/cli/tools/coverage/mod.rs index d847fe4281586a..94e9bffae2da14 100644 --- a/cli/tools/coverage/mod.rs +++ b/cli/tools/coverage/mod.rs @@ -7,6 +7,7 @@ use crate::colors; use crate::emit::get_source_hash; use crate::fs_util::collect_files; use crate::proc_state::ProcState; +use crate::text_encoding::source_map_from_code; use crate::tools::fmt::format_json; use deno_ast::MediaType; @@ -684,7 +685,10 @@ pub async fn cover_files( let source_hash = get_source_hash(original_source, ps.emit_options_hash); match emit_cache.get_emit_data(&file.specifier, Some(source_hash)) { - Some(data) => (data.text, data.map), + Some(data) => { + let map = source_map_from_code(&data.text).or_else(|| data.map.map(|t| t.into_bytes())); + (data.text, map) + }, None => { return Err(anyhow!( "Missing transpiled source code for: \"{}\". @@ -702,7 +706,7 @@ pub async fn cover_files( let coverage_report = generate_coverage_report( &script_coverage, &transpiled_source, - &maybe_source_map.map(|t| t.into_bytes()), + &maybe_source_map, &out_mode, ); From e83113bfd2efaaaad892b5a8952ab0809f0cfc59 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Thu, 14 Jul 2022 11:47:36 -0400 Subject: [PATCH 07/13] Emit cache unit tests --- cli/cache/check.rs | 2 +- cli/cache/emit.rs | 104 ++++++++++++++++++++++++--------------------- 2 files changed, 57 insertions(+), 49 deletions(-) diff --git a/cli/cache/check.rs b/cli/cache/check.rs index 4e0f8d9123eedf..4f8acadc259b8f 100644 --- a/cli/cache/check.rs +++ b/cli/cache/check.rs @@ -233,7 +233,7 @@ mod test { cache.set_tsbuildinfo(&specifier1, "test"); assert_eq!(cache.get_tsbuildinfo(&specifier1), Some("test".to_string())); - // recreating the cache should not remove the data because the CLI version and state hash is the same + // recreating the cache should not remove the data because the CLI version is the same let conn = cache.0.unwrap(); let cache = TypeCheckCache::from_connection(conn, "2.0.0".to_string()).unwrap(); diff --git a/cli/cache/emit.rs b/cli/cache/emit.rs index f42baa0987206b..fbf385e90e83f6 100644 --- a/cli/cache/emit.rs +++ b/cli/cache/emit.rs @@ -183,51 +183,59 @@ fn create_tables( Ok(()) } -// #[cfg(test)] -// mod test { -// use super::*; - -// #[test] -// pub fn check_cache_general_use() { -// let conn = Connection::open_in_memory().unwrap(); -// let cache = EmitCache::from_connection(conn, "1.0.0".to_string()).unwrap(); - -// assert!(!cache.has_check_hash(1)); -// cache.add_check_hash(1); -// assert!(cache.has_check_hash(1)); -// assert!(!cache.has_check_hash(2)); - -// let specifier1 = ModuleSpecifier::parse("file:///test.json").unwrap(); -// assert_eq!(cache.get_tsbuildinfo(&specifier1), None); -// cache.set_tsbuildinfo(&specifier1, "test"); -// assert_eq!(cache.get_tsbuildinfo(&specifier1), Some("test".to_string())); - -// // try changing the cli version (should clear) -// let conn = cache.0.unwrap(); -// let cache = -// TypeCheckCache::from_connection(conn, "2.0.0".to_string()).unwrap(); -// assert!(!cache.has_check_hash(1)); -// cache.add_check_hash(1); -// assert!(cache.has_check_hash(1)); -// assert_eq!(cache.get_tsbuildinfo(&specifier1), None); -// cache.set_tsbuildinfo(&specifier1, "test"); -// assert_eq!(cache.get_tsbuildinfo(&specifier1), Some("test".to_string())); - -// // recreating the cache should not remove the data because the CLI version and state hash is the same -// let conn = cache.0.unwrap(); -// let cache = -// TypeCheckCache::from_connection(conn, "2.0.0".to_string()).unwrap(); -// assert!(cache.has_check_hash(1)); -// assert!(!cache.has_check_hash(2)); -// assert_eq!(cache.get_tsbuildinfo(&specifier1), Some("test".to_string())); - -// // adding when already exists should not cause issue -// cache.add_check_hash(1); -// assert!(cache.has_check_hash(1)); -// cache.set_tsbuildinfo(&specifier1, "other"); -// assert_eq!( -// cache.get_tsbuildinfo(&specifier1), -// Some("other".to_string()) -// ); -// } -// } +#[cfg(test)] +mod test { + use super::*; + + #[test] + pub fn emit_cache_general_use() { + let conn = Connection::open_in_memory().unwrap(); + let cache = EmitCache::from_connection(conn, "1.0.0".to_string()).unwrap(); + + let specifier1 = ModuleSpecifier::parse("file:///test.json").unwrap(); + assert_eq!(cache.get_emit_data(&specifier1, None), None); + let cache_data1 = SpecifierEmitCacheData { + text: "text".to_string(), + map: Some("map".to_string()), + }; + cache.set_emit_data(&specifier1, 10, &cache_data1); + // providing no source hash + assert_eq!( + cache.get_emit_data(&specifier1, None), + Some(cache_data1.clone()) + ); + // providing the incorrect source hash + assert_eq!(cache.get_emit_data(&specifier1, Some(5)), None); + // providing the correct source hash + assert_eq!( + cache.get_emit_data(&specifier1, Some(10)), + Some(cache_data1.clone()), + ); + + // try changing the cli version (should clear) + let conn = cache.0.unwrap(); + let cache = EmitCache::from_connection(conn, "2.0.0".to_string()).unwrap(); + assert_eq!(cache.get_emit_data(&specifier1, None), None); + cache.set_emit_data(&specifier1, 5, &cache_data1); + + // recreating the cache should not remove the data because the CLI version is the same + let conn = cache.0.unwrap(); + let cache = EmitCache::from_connection(conn, "2.0.0".to_string()).unwrap(); + assert_eq!( + cache.get_emit_data(&specifier1, Some(5)), + Some(cache_data1.clone()) + ); + + // adding when already exists should not cause issue + let cache_data2 = SpecifierEmitCacheData { + text: "asdf".to_string(), + map: None, + }; + cache.set_emit_data(&specifier1, 20, &cache_data2); + assert_eq!(cache.get_emit_data(&specifier1, Some(5)), None); + assert_eq!( + cache.get_emit_data(&specifier1, Some(20)), + Some(cache_data2.clone()) + ); + } +} From ceede092d864b6344752a743954d36052949b70f Mon Sep 17 00:00:00 2001 From: David Sherret Date: Thu, 14 Jul 2022 15:33:24 -0400 Subject: [PATCH 08/13] Reduce memory usage by storing source map separately. --- cli/args/mod.rs | 5 ++++ cli/cache/emit.rs | 24 +++++++++---------- cli/emit.rs | 49 +++++++++++++++++++++++++++------------ cli/module_loader.rs | 40 +++++++++++++++++++------------- cli/text_encoding.rs | 19 +++++++++++---- cli/tools/coverage/mod.rs | 10 ++++---- 6 files changed, 95 insertions(+), 52 deletions(-) diff --git a/cli/args/mod.rs b/cli/args/mod.rs index accdeae7ff1c37..2aac47a863b0a2 100644 --- a/cli/args/mod.rs +++ b/cli/args/mod.rs @@ -277,6 +277,11 @@ impl CliOptions { self.flags.enable_testing_features } + /// If the --inspect or --inspect-brk flags are used. + pub fn is_inspecting(&self) -> bool { + self.flags.inspect.is_some() || self.flags.inspect_brk.is_some() + } + pub fn inspect_brk(&self) -> Option { self.flags.inspect_brk } diff --git a/cli/cache/emit.rs b/cli/cache/emit.rs index fbf385e90e83f6..7f02630e8dc230 100644 --- a/cli/cache/emit.rs +++ b/cli/cache/emit.rs @@ -12,8 +12,8 @@ use super::common::run_sqlite_pragma; /// Emit cache for a single file. #[derive(Debug, Clone, PartialEq)] pub struct SpecifierEmitCacheData { - pub text: String, - pub map: Option, + pub code: String, + pub map: String, } /// The cache that stores previously emitted files. @@ -84,7 +84,7 @@ impl EmitCache { None => return None, }; let mut stmt = conn - .prepare_cached("SELECT source_hash, emit_text, source_map FROM emitcache WHERE specifier=?1 LIMIT 1") + .prepare_cached("SELECT source_hash, code, source_map FROM emitcache WHERE specifier=?1 LIMIT 1") .ok()?; let mut rows = stmt.query(params![specifier.to_string()]).ok()?; let row = rows.next().ok().flatten()?; @@ -98,7 +98,7 @@ impl EmitCache { } Some(SpecifierEmitCacheData { - text: row.get(1).ok()?, + code: row.get(1).ok()?, map: row.get(2).ok()?, }) } @@ -131,12 +131,12 @@ impl EmitCache { None => return Ok(()), }; let mut stmt = conn.prepare_cached( - "INSERT OR REPLACE INTO emitcache (specifier, source_hash, emit_text, source_map) VALUES (?1, ?2, ?3, ?4)", + "INSERT OR REPLACE INTO emitcache (specifier, source_hash, code, source_map) VALUES (?1, ?2, ?3, ?4)", )?; stmt.execute(params![ specifier.to_string(), source_hash.to_string(), - &data.text, + &data.code, &data.map, ])?; Ok(()) @@ -152,8 +152,8 @@ fn create_tables( "CREATE TABLE IF NOT EXISTS emitcache ( specifier TEXT PRIMARY KEY, source_hash TEXT NOT NULL, - emit_text TEXT NOT NULL, - source_map TEXT + code TEXT NOT NULL, + source_map TEXT NOT NULL )", [], )?; @@ -195,8 +195,8 @@ mod test { let specifier1 = ModuleSpecifier::parse("file:///test.json").unwrap(); assert_eq!(cache.get_emit_data(&specifier1, None), None); let cache_data1 = SpecifierEmitCacheData { - text: "text".to_string(), - map: Some("map".to_string()), + code: "text".to_string(), + map: "map".to_string(), }; cache.set_emit_data(&specifier1, 10, &cache_data1); // providing no source hash @@ -228,8 +228,8 @@ mod test { // adding when already exists should not cause issue let cache_data2 = SpecifierEmitCacheData { - text: "asdf".to_string(), - map: None, + code: "asdf".to_string(), + map: "map2".to_string(), }; cache.set_emit_data(&specifier1, 20, &cache_data2); assert_eq!(cache.get_emit_data(&specifier1, Some(5)), None); diff --git a/cli/emit.rs b/cli/emit.rs index 44844d97f806eb..7203e2dff29153 100644 --- a/cli/emit.rs +++ b/cli/emit.rs @@ -17,6 +17,7 @@ use crate::colors; use crate::diagnostics::Diagnostics; use crate::graph_util::GraphData; use crate::graph_util::ModuleEntry; +use crate::text_encoding::code_with_source_map; use crate::tsc; use crate::version; @@ -158,8 +159,6 @@ pub fn get_ts_config_for_emit( "jsxFactory": "React.createElement", "jsxFragmentFactory": "React.Fragment", "importsNotUsedAsValues": "remove", - "inlineSourceMap": true, - "inlineSources": true, "isolatedModules": true, "lib": lib, "module": "esnext", @@ -178,9 +177,9 @@ pub fn get_ts_config_for_emit( "checkJs": false, "emitDecoratorMetadata": false, "importsNotUsedAsValues": "remove", - "inlineSourceMap": true, - "inlineSources": true, - "sourceMap": false, + "inlineSourceMap": false, + "inlineSources": false, + "sourceMap": true, "jsx": "react", "jsxFactory": "React.createElement", "jsxFragmentFactory": "React.Fragment", @@ -241,26 +240,46 @@ pub fn get_source_hash(source_text: &str, emit_options_hash: u64) -> u64 { .finish() } +pub struct EmittedParsedSource { + pub code: String, + pub map: String, +} + +impl EmittedParsedSource { + pub fn into_text_with_embedded_source_map(self) -> String { + code_with_source_map(self.code, &self.map) + } +} + pub fn emit_parsed_source( cache: &EmitCache, specifier: &ModuleSpecifier, parsed_source: &ParsedSource, emit_options: &deno_ast::EmitOptions, emit_config_hash: u64, -) -> Result { +) -> Result { let source_hash = get_source_hash(parsed_source.text_info().text_str(), emit_config_hash); - if let Some(emit_data) = cache.get_emit_data(specifier, Some(source_hash)) { - return Ok(emit_data.text); - } - let transpiled_source = parsed_source.transpile(emit_options)?; - let cache_data = SpecifierEmitCacheData { - text: transpiled_source.text, - map: transpiled_source.source_map, + let cache_data = if let Some(cache_data) = + cache.get_emit_data(specifier, Some(source_hash)) + { + cache_data + } else { + let transpiled_source = parsed_source.transpile(emit_options)?; + let cache_data = SpecifierEmitCacheData { + code: transpiled_source.text, + map: transpiled_source + .source_map + .expect("should have source map"), + }; + cache.set_emit_data(specifier, source_hash, &cache_data); + cache_data }; - cache.set_emit_data(specifier, source_hash, &cache_data); - Ok(cache_data.text) + Ok(EmittedParsedSource { + code: cache_data.code, + map: cache_data.map, + }) } /// Options for performing a check of a module graph. Note that the decision to diff --git a/cli/module_loader.rs b/cli/module_loader.rs index 75408dc7ab231e..48a6a17c492789 100644 --- a/cli/module_loader.rs +++ b/cli/module_loader.rs @@ -27,6 +27,7 @@ use std::str; pub struct ModuleCodeSource { pub code: String, + pub map: Option, pub found_url: ModuleSpecifier, pub media_type: MediaType, } @@ -76,20 +77,24 @@ impl CliModuleLoader { maybe_parsed_source, .. }) => { - let code = match media_type { + let (code, map) = match media_type { MediaType::JavaScript | MediaType::Unknown | MediaType::Cjs | MediaType::Mjs | MediaType::Json => { - if let Some(source) = graph_data.get_cjs_esm_translation(specifier) + let code = if let Some(source) = + graph_data.get_cjs_esm_translation(specifier) { source.to_owned() } else { code.to_string() - } + }; + (code, None) + } + MediaType::Dts | MediaType::Dcts | MediaType::Dmts => { + ("".to_string(), None) } - MediaType::Dts | MediaType::Dcts | MediaType::Dmts => "".to_string(), MediaType::TypeScript | MediaType::Mts | MediaType::Cts @@ -97,13 +102,20 @@ impl CliModuleLoader { | MediaType::Tsx => { // get emit text let parsed_source = maybe_parsed_source.as_ref().unwrap(); // should always be set - emit_parsed_source( + let emit = emit_parsed_source( &self.emit_cache, specifier, parsed_source, &self.ps.emit_options, self.ps.emit_options_hash, - )? + )?; + if self.ps.options.is_inspecting() { + // when inspecting, embed the source map + // so the user can see the original file + (emit.into_text_with_embedded_source_map(), None) + } else { + (emit.code, Some(emit.map)) + } } MediaType::TsBuildInfo | MediaType::Wasm | MediaType::SourceMap => { panic!("Unexpected media type {} for {}", media_type, found_url) @@ -111,6 +123,7 @@ impl CliModuleLoader { }; Ok(ModuleCodeSource { code, + map, found_url, media_type: *media_type, }) @@ -213,16 +226,11 @@ impl SourceMapGetter for CliModuleLoader { "wasm" | "file" | "http" | "https" | "data" | "blob" => (), _ => return None, } - if let Some(cache_data) = self.emit_cache.get_emit_data( - &specifier, - // ideally we would provide a source hash here in order to verify that - // we're getting a source map for the file currently in the cache - None, - ) { - source_map_from_code(&cache_data.text) - .or_else(|| cache_data.map.map(|t| t.into_bytes())) - } else if let Ok(source) = self.load_prepared_module(&specifier) { - source_map_from_code(&source.code) + if let Ok(source) = self.load_prepared_module(&specifier) { + source + .map + .map(|m| m.into_bytes()) + .or_else(|| source_map_from_code(&source.code)) } else { None } diff --git a/cli/text_encoding.rs b/cli/text_encoding.rs index f4a29a5edc9423..2cc071f9d3c42a 100644 --- a/cli/text_encoding.rs +++ b/cli/text_encoding.rs @@ -54,11 +54,13 @@ pub fn strip_bom(text: &str) -> &str { } } +static SOURCE_MAP_PREFIX: &str = + "//# sourceMappingURL=data:application/json;base64,"; + pub fn source_map_from_code(code: &str) -> Option> { - static PREFIX: &str = "//# sourceMappingURL=data:application/json;base64,"; - let last_line = code.rsplit(|u| u == '\n').next().unwrap(); - if last_line.starts_with(PREFIX) { - let input = last_line.split_at(PREFIX.len()).1; + let last_line = code.rsplit(|u| u == '\n').next()?; + if last_line.starts_with(SOURCE_MAP_PREFIX) { + let input = last_line.split_at(SOURCE_MAP_PREFIX.len()).1; let decoded_map = base64::decode(input) .expect("Unable to decode source map from emitted file."); Some(decoded_map) @@ -67,6 +69,15 @@ pub fn source_map_from_code(code: &str) -> Option> { } } +pub fn code_with_source_map(mut code: String, source_map: &str) -> String { + if !code.ends_with('\n') { + code.push('\n'); + } + code.push_str(SOURCE_MAP_PREFIX); + code.push_str(&base64::encode(&source_map)); + code +} + #[cfg(test)] mod tests { use super::*; diff --git a/cli/tools/coverage/mod.rs b/cli/tools/coverage/mod.rs index 94e9bffae2da14..b477ab645ec008 100644 --- a/cli/tools/coverage/mod.rs +++ b/cli/tools/coverage/mod.rs @@ -673,7 +673,10 @@ pub async fn cover_files( | MediaType::Unknown | MediaType::Cjs | MediaType::Mjs - | MediaType::Json => (file.source.as_ref().to_string(), None), + | MediaType::Json => ( + file.source.as_ref().to_string(), + source_map_from_code(file.source.as_ref()), + ), MediaType::Dts | MediaType::Dmts | MediaType::Dcts => { ("".to_string(), None) } @@ -685,10 +688,7 @@ pub async fn cover_files( let source_hash = get_source_hash(original_source, ps.emit_options_hash); match emit_cache.get_emit_data(&file.specifier, Some(source_hash)) { - Some(data) => { - let map = source_map_from_code(&data.text).or_else(|| data.map.map(|t| t.into_bytes())); - (data.text, map) - }, + Some(data) => (data.code, Some(data.map.into_bytes())), None => { return Err(anyhow!( "Missing transpiled source code for: \"{}\". From adecc49a76cc8b420715753827cd031046fce4ea Mon Sep 17 00:00:00 2001 From: David Sherret Date: Thu, 14 Jul 2022 15:51:03 -0400 Subject: [PATCH 09/13] Lint. --- cli/cache/emit.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/cli/cache/emit.rs b/cli/cache/emit.rs index 7f02630e8dc230..b287b389c12fd9 100644 --- a/cli/cache/emit.rs +++ b/cli/cache/emit.rs @@ -221,10 +221,7 @@ mod test { // recreating the cache should not remove the data because the CLI version is the same let conn = cache.0.unwrap(); let cache = EmitCache::from_connection(conn, "2.0.0".to_string()).unwrap(); - assert_eq!( - cache.get_emit_data(&specifier1, Some(5)), - Some(cache_data1.clone()) - ); + assert_eq!(cache.get_emit_data(&specifier1, Some(5)), Some(cache_data1)); // adding when already exists should not cause issue let cache_data2 = SpecifierEmitCacheData { @@ -235,7 +232,7 @@ mod test { assert_eq!(cache.get_emit_data(&specifier1, Some(5)), None); assert_eq!( cache.get_emit_data(&specifier1, Some(20)), - Some(cache_data2.clone()) + Some(cache_data2) ); } } From 87fb86e2b9bb328be22930d66f7a355aa3a321b8 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Thu, 14 Jul 2022 16:23:07 -0400 Subject: [PATCH 10/13] Updates. --- cli/cache/check.rs | 4 ++-- cli/cache/emit.rs | 8 ++++---- cli/proc_state.rs | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cli/cache/check.rs b/cli/cache/check.rs index 4f8acadc259b8f..6f3c41950a5426 100644 --- a/cli/cache/check.rs +++ b/cli/cache/check.rs @@ -22,7 +22,7 @@ impl TypeCheckCache { Err(err) => { log::debug!( concat!( - "Failed creating internal type checking cache. ", + "Failed loading internal type checking cache. ", "Recreating...\n\nError details:\n{:#}", ), err @@ -35,7 +35,7 @@ impl TypeCheckCache { Err(err) => { log::debug!( concat!( - "Unable to create internal cache for type checking. ", + "Unable to load internal cache for type checking. ", "This will reduce the performance of type checking.\n\n", "Error details:\n{:#}", ), diff --git a/cli/cache/emit.rs b/cli/cache/emit.rs index b287b389c12fd9..e39f41ebc3208f 100644 --- a/cli/cache/emit.rs +++ b/cli/cache/emit.rs @@ -26,7 +26,7 @@ impl EmitCache { Err(err) => { log::debug!( concat!( - "Failed creating internal emit cache. ", + "Failed loading internal emit cache. ", "Recreating...\n\nError details:\n{:#}", ), err @@ -39,7 +39,7 @@ impl EmitCache { Err(err) => { log::debug!( concat!( - "Unable to create internal emit cache. ", + "Unable to load internal emit cache. ", "This will reduce the performance of emitting.\n\n", "Error details:\n{:#}", ), @@ -72,8 +72,8 @@ impl EmitCache { /// Gets the emit data from the cache. /// /// Ideally, you SHOULD provide an expected source hash in order - /// to verify that you're getting a value from the cache - /// that is relevant to the source. + /// to verify that you're getting a value from the cache that + /// is for the provided source. pub fn get_emit_data( &self, specifier: &ModuleSpecifier, diff --git a/cli/proc_state.rs b/cli/proc_state.rs index 2758cac5413d0f..adf3dfdc185181 100644 --- a/cli/proc_state.rs +++ b/cli/proc_state.rs @@ -219,7 +219,7 @@ impl ProcState { coverage_dir, options: cli_options, emit_options_hash: FastInsecureHash::new() - // todo: use hash of emit options instead as it's more specific + // todo(dsherret): use hash of emit options instead as it's more specific .write(&ts_config_result.ts_config.as_bytes()) .finish(), emit_options: ts_config_result.ts_config.into(), From 43c090bc8caa68a1399e9318a3c650402d8dc727 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Thu, 14 Jul 2022 16:48:49 -0400 Subject: [PATCH 11/13] Remove logic for optional source hash as it's now unnecessary. --- cli/cache/emit.rs | 51 +++++++++++++++------------------------ cli/emit.rs | 27 ++++++++++----------- cli/tools/coverage/mod.rs | 2 +- 3 files changed, 34 insertions(+), 46 deletions(-) diff --git a/cli/cache/emit.rs b/cli/cache/emit.rs index e39f41ebc3208f..e28270ec74c173 100644 --- a/cli/cache/emit.rs +++ b/cli/cache/emit.rs @@ -71,35 +71,32 @@ impl EmitCache { /// Gets the emit data from the cache. /// - /// Ideally, you SHOULD provide an expected source hash in order - /// to verify that you're getting a value from the cache that - /// is for the provided source. + /// The expected source hash is used in order to verify + /// that you're getting a value from the cache that is + /// for the provided source. pub fn get_emit_data( &self, specifier: &ModuleSpecifier, - maybe_expected_source_hash: Option, + expected_source_hash: u64, ) -> Option { let conn = match &self.0 { Some(conn) => conn, None => return None, }; let mut stmt = conn - .prepare_cached("SELECT source_hash, code, source_map FROM emitcache WHERE specifier=?1 LIMIT 1") + .prepare_cached("SELECT code, source_map FROM emitcache WHERE specifier=?1 AND source_hash=?2 LIMIT 1") + .ok()?; + let mut rows = stmt + .query(params![ + specifier.to_string(), + expected_source_hash.to_string() + ]) .ok()?; - let mut rows = stmt.query(params![specifier.to_string()]).ok()?; let row = rows.next().ok().flatten()?; - if let Some(expected_hash) = maybe_expected_source_hash { - // verify that the emit is for the source - let saved_hash = row.get::(0).ok()?; - if saved_hash != expected_hash.to_string() { - return None; - } - } - Some(SpecifierEmitCacheData { - code: row.get(1).ok()?, - map: row.get(2).ok()?, + code: row.get(0).ok()?, + map: row.get(1).ok()?, }) } @@ -193,35 +190,30 @@ mod test { let cache = EmitCache::from_connection(conn, "1.0.0".to_string()).unwrap(); let specifier1 = ModuleSpecifier::parse("file:///test.json").unwrap(); - assert_eq!(cache.get_emit_data(&specifier1, None), None); + assert_eq!(cache.get_emit_data(&specifier1, 1), None); let cache_data1 = SpecifierEmitCacheData { code: "text".to_string(), map: "map".to_string(), }; cache.set_emit_data(&specifier1, 10, &cache_data1); - // providing no source hash - assert_eq!( - cache.get_emit_data(&specifier1, None), - Some(cache_data1.clone()) - ); // providing the incorrect source hash - assert_eq!(cache.get_emit_data(&specifier1, Some(5)), None); + assert_eq!(cache.get_emit_data(&specifier1, 5), None); // providing the correct source hash assert_eq!( - cache.get_emit_data(&specifier1, Some(10)), + cache.get_emit_data(&specifier1, 10), Some(cache_data1.clone()), ); // try changing the cli version (should clear) let conn = cache.0.unwrap(); let cache = EmitCache::from_connection(conn, "2.0.0".to_string()).unwrap(); - assert_eq!(cache.get_emit_data(&specifier1, None), None); + assert_eq!(cache.get_emit_data(&specifier1, 10), None); cache.set_emit_data(&specifier1, 5, &cache_data1); // recreating the cache should not remove the data because the CLI version is the same let conn = cache.0.unwrap(); let cache = EmitCache::from_connection(conn, "2.0.0".to_string()).unwrap(); - assert_eq!(cache.get_emit_data(&specifier1, Some(5)), Some(cache_data1)); + assert_eq!(cache.get_emit_data(&specifier1, 5), Some(cache_data1)); // adding when already exists should not cause issue let cache_data2 = SpecifierEmitCacheData { @@ -229,10 +221,7 @@ mod test { map: "map2".to_string(), }; cache.set_emit_data(&specifier1, 20, &cache_data2); - assert_eq!(cache.get_emit_data(&specifier1, Some(5)), None); - assert_eq!( - cache.get_emit_data(&specifier1, Some(20)), - Some(cache_data2) - ); + assert_eq!(cache.get_emit_data(&specifier1, 5), None); + assert_eq!(cache.get_emit_data(&specifier1, 20), Some(cache_data2)); } } diff --git a/cli/emit.rs b/cli/emit.rs index 7203e2dff29153..18831bd513a3e3 100644 --- a/cli/emit.rs +++ b/cli/emit.rs @@ -261,21 +261,20 @@ pub fn emit_parsed_source( let source_hash = get_source_hash(parsed_source.text_info().text_str(), emit_config_hash); - let cache_data = if let Some(cache_data) = - cache.get_emit_data(specifier, Some(source_hash)) - { - cache_data - } else { - let transpiled_source = parsed_source.transpile(emit_options)?; - let cache_data = SpecifierEmitCacheData { - code: transpiled_source.text, - map: transpiled_source - .source_map - .expect("should have source map"), + let cache_data = + if let Some(cache_data) = cache.get_emit_data(specifier, source_hash) { + cache_data + } else { + let transpiled_source = parsed_source.transpile(emit_options)?; + let cache_data = SpecifierEmitCacheData { + code: transpiled_source.text, + map: transpiled_source + .source_map + .expect("should have source map"), + }; + cache.set_emit_data(specifier, source_hash, &cache_data); + cache_data }; - cache.set_emit_data(specifier, source_hash, &cache_data); - cache_data - }; Ok(EmittedParsedSource { code: cache_data.code, map: cache_data.map, diff --git a/cli/tools/coverage/mod.rs b/cli/tools/coverage/mod.rs index b477ab645ec008..f1e67d7a6f6274 100644 --- a/cli/tools/coverage/mod.rs +++ b/cli/tools/coverage/mod.rs @@ -687,7 +687,7 @@ pub async fn cover_files( | MediaType::Tsx => { let source_hash = get_source_hash(original_source, ps.emit_options_hash); - match emit_cache.get_emit_data(&file.specifier, Some(source_hash)) { + match emit_cache.get_emit_data(&file.specifier, source_hash) { Some(data) => (data.code, Some(data.map.into_bytes())), None => { return Err(anyhow!( From 0d53b1dc56d492b17eba9f4b3ee9af321216f5b9 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Sat, 16 Jul 2022 18:41:25 -0400 Subject: [PATCH 12/13] This should actually be the opposite way --- cli/module_loader.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cli/module_loader.rs b/cli/module_loader.rs index 48a6a17c492789..5ff8dc4e47f778 100644 --- a/cli/module_loader.rs +++ b/cli/module_loader.rs @@ -227,10 +227,10 @@ impl SourceMapGetter for CliModuleLoader { _ => return None, } if let Ok(source) = self.load_prepared_module(&specifier) { - source - .map - .map(|m| m.into_bytes()) - .or_else(|| source_map_from_code(&source.code)) + // always check the code first as someone might be running + // with `--inspect`, which embeds a source file in the code + source_map_from_code(&source.code) + .or_else(|| source.map.map(|m| m.into_bytes())) } else { None } From 4fb035e98cf51d7ae67a8ed7dd1d14655dc5a3a0 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Sat, 16 Jul 2022 18:49:57 -0400 Subject: [PATCH 13/13] Hash -> Hasher --- cli/cache/common.rs | 6 +++--- cli/cache/incremental.rs | 10 +++++----- cli/cache/mod.rs | 2 +- cli/emit.rs | 4 ++-- cli/proc_state.rs | 4 ++-- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/cli/cache/common.rs b/cli/cache/common.rs index 6a806a70fe6eed..b536d6cb2138b8 100644 --- a/cli/cache/common.rs +++ b/cli/cache/common.rs @@ -5,11 +5,11 @@ use std::hash::Hasher; use deno_core::error::AnyError; use deno_runtime::deno_webstorage::rusqlite::Connection; -/// A very fast insecure hash that uses the xxHash algorithm. +/// A very fast insecure hasher that uses the xxHash algorithm. #[derive(Default)] -pub struct FastInsecureHash(twox_hash::XxHash64); +pub struct FastInsecureHasher(twox_hash::XxHash64); -impl FastInsecureHash { +impl FastInsecureHasher { pub fn new() -> Self { Self::default() } diff --git a/cli/cache/incremental.rs b/cli/cache/incremental.rs index e87b47295e8a7b..da832a8b5a4578 100644 --- a/cli/cache/incremental.rs +++ b/cli/cache/incremental.rs @@ -13,7 +13,7 @@ use serde::Serialize; use tokio::task::JoinHandle; use super::common::run_sqlite_pragma; -use super::common::FastInsecureHash; +use super::common::FastInsecureHasher; /// Cache used to skip formatting/linting a file again when we /// know it is already formatted or has no lint diagnostics. @@ -79,7 +79,7 @@ impl IncrementalCacheInner { state: &TState, initial_file_paths: &[PathBuf], ) -> Result { - let state_hash = FastInsecureHash::new() + let state_hash = FastInsecureHasher::new() .write_str(&serde_json::to_string(state).unwrap()) .finish(); let sql_cache = SqlIncrementalCache::new(db_file_path, state_hash)?; @@ -125,14 +125,14 @@ impl IncrementalCacheInner { pub fn is_file_same(&self, file_path: &Path, file_text: &str) -> bool { match self.previous_hashes.get(file_path) { Some(hash) => { - *hash == FastInsecureHash::new().write_str(file_text).finish() + *hash == FastInsecureHasher::new().write_str(file_text).finish() } None => false, } } pub fn update_file(&self, file_path: &Path, file_text: &str) { - let hash = FastInsecureHash::new().write_str(file_text).finish(); + let hash = FastInsecureHasher::new().write_str(file_text).finish(); if let Some(previous_hash) = self.previous_hashes.get(file_path) { if *previous_hash == hash { return; // do not bother updating the db file because nothing has changed @@ -337,7 +337,7 @@ mod test { .unwrap(); let file_path = PathBuf::from("/mod.ts"); let file_text = "test"; - let file_hash = FastInsecureHash::new().write_str(file_text).finish(); + let file_hash = FastInsecureHasher::new().write_str(file_text).finish(); sql_cache.set_source_hash(&file_path, file_hash).unwrap(); let cache = IncrementalCacheInner::from_sql_incremental_cache( sql_cache, diff --git a/cli/cache/mod.rs b/cli/cache/mod.rs index d535edb688d873..bb754c64bb4d0f 100644 --- a/cli/cache/mod.rs +++ b/cli/cache/mod.rs @@ -19,7 +19,7 @@ mod emit; mod incremental; pub use check::TypeCheckCache; -pub use common::FastInsecureHash; +pub use common::FastInsecureHasher; pub use disk_cache::DiskCache; pub use emit::EmitCache; pub use emit::SpecifierEmitCacheData; diff --git a/cli/emit.rs b/cli/emit.rs index caac328eb32b70..f24842a9456394 100644 --- a/cli/emit.rs +++ b/cli/emit.rs @@ -10,7 +10,7 @@ use crate::args::EmitConfigOptions; use crate::args::TsConfig; use crate::args::TypeCheckMode; use crate::cache::EmitCache; -use crate::cache::FastInsecureHash; +use crate::cache::FastInsecureHasher; use crate::cache::SpecifierEmitCacheData; use crate::cache::TypeCheckCache; use crate::colors; @@ -233,7 +233,7 @@ fn get_tsc_roots( /// user provided config and generates a string hash which can be stored to /// determine if the cached emit is valid or not. pub fn get_source_hash(source_text: &str, emit_options_hash: u64) -> u64 { - FastInsecureHash::new() + FastInsecureHasher::new() .write_str(source_text) .write_u64(emit_options_hash) .finish() diff --git a/cli/proc_state.rs b/cli/proc_state.rs index 0143d079b55deb..504fcfa1b8f3f1 100644 --- a/cli/proc_state.rs +++ b/cli/proc_state.rs @@ -5,7 +5,7 @@ use crate::args::DenoSubcommand; use crate::args::Flags; use crate::args::TypeCheckMode; use crate::cache; -use crate::cache::FastInsecureHash; +use crate::cache::FastInsecureHasher; use crate::cache::TypeCheckCache; use crate::compat; use crate::compat::NodeEsmResolver; @@ -218,7 +218,7 @@ impl ProcState { dir, coverage_dir, options: cli_options, - emit_options_hash: FastInsecureHash::new() + emit_options_hash: FastInsecureHasher::new() // todo(dsherret): use hash of emit options instead as it's more specific .write(&ts_config_result.ts_config.as_bytes()) .finish(),