diff --git a/cli/checksum.rs b/cli/checksum.rs index 41e15db2fb4a8d..a86f527c031bf4 100644 --- a/cli/checksum.rs +++ b/cli/checksum.rs @@ -1,9 +1,12 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. -pub fn gen(v: &[&[u8]]) -> String { - let mut ctx = ring::digest::Context::new(&ring::digest::SHA256); +use ring::digest::Context; +use ring::digest::SHA256; + +pub fn gen(v: &[impl AsRef<[u8]>]) -> String { + let mut ctx = Context::new(&SHA256); for src in v { - ctx.update(src); + ctx.update(src.as_ref()); } let digest = ctx.finish(); let out: Vec = digest @@ -13,3 +16,17 @@ pub fn gen(v: &[&[u8]]) -> String { .collect(); out.join("") } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_gen() { + let actual = gen(&[b"hello world"]); + assert_eq!( + actual, + "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + ); + } +} diff --git a/cli/diagnostics.rs b/cli/diagnostics.rs index b3b5a73c3c7a7a..290007cc720974 100644 --- a/cli/diagnostics.rs +++ b/cli/diagnostics.rs @@ -127,7 +127,7 @@ fn format_message(msg: &str, code: &u64) -> String { } } -#[derive(Clone, Debug, PartialEq)] +#[derive(Clone, Debug, Eq, PartialEq)] pub enum DiagnosticCategory { Warning, Error, @@ -172,7 +172,7 @@ impl From for DiagnosticCategory { } } -#[derive(Debug, Deserialize, Clone)] +#[derive(Debug, Deserialize, Clone, Eq, PartialEq)] #[serde(rename_all = "camelCase")] pub struct DiagnosticMessageChain { message_text: String, @@ -199,26 +199,26 @@ impl DiagnosticMessageChain { } } -#[derive(Debug, Deserialize, Clone)] +#[derive(Debug, Deserialize, Clone, Eq, PartialEq)] #[serde(rename_all = "camelCase")] pub struct Position { pub line: u64, pub character: u64, } -#[derive(Debug, Deserialize, Clone)] +#[derive(Debug, Deserialize, Clone, Eq, PartialEq)] #[serde(rename_all = "camelCase")] pub struct Diagnostic { - category: DiagnosticCategory, - code: u64, - start: Option, - end: Option, - message_text: Option, - message_chain: Option, - source: Option, - source_line: Option, - file_name: Option, - related_information: Option>, + pub category: DiagnosticCategory, + pub code: u64, + pub start: Option, + pub end: Option, + pub message_text: Option, + pub message_chain: Option, + pub source: Option, + pub source_line: Option, + pub file_name: Option, + pub related_information: Option>, } impl Diagnostic { @@ -346,7 +346,7 @@ impl fmt::Display for Diagnostic { } } -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Eq, PartialEq)] pub struct Diagnostics(pub Vec); impl<'de> Deserialize<'de> for Diagnostics { diff --git a/cli/js.rs b/cli/js.rs index d51d7dd92be25c..3d2a17f36d7842 100644 --- a/cli/js.rs +++ b/cli/js.rs @@ -57,7 +57,7 @@ fn compiler_snapshot() { .execute( "", r#" - if (!(bootstrapCompilerRuntime)) { + if (!(startup)) { throw Error("bad"); } console.log(`ts version: ${ts.version}`); diff --git a/cli/main.rs b/cli/main.rs index f6bef9688ef54c..2947dfde2fa96a 100644 --- a/cli/main.rs +++ b/cli/main.rs @@ -51,6 +51,7 @@ mod test_runner; mod text_encoding; mod tokio_util; mod tsc; +pub mod tsc2; mod tsc_config; mod upgrade; mod version; diff --git a/cli/media_type.rs b/cli/media_type.rs index cc3700a66b2917..5d083466be63c2 100644 --- a/cli/media_type.rs +++ b/cli/media_type.rs @@ -10,7 +10,7 @@ use std::path::PathBuf; // Update carefully! #[allow(non_camel_case_types)] #[repr(i32)] -#[derive(Clone, Copy, PartialEq, Debug)] +#[derive(Clone, Copy, Eq, PartialEq, Debug)] pub enum MediaType { JavaScript = 0, JSX = 1, @@ -19,7 +19,9 @@ pub enum MediaType { TSX = 4, Json = 5, Wasm = 6, - Unknown = 8, + TsBuildInfo = 7, + SourceMap = 8, + Unknown = 9, } impl fmt::Display for MediaType { @@ -32,6 +34,8 @@ impl fmt::Display for MediaType { MediaType::TSX => "TSX", MediaType::Json => "Json", MediaType::Wasm => "Wasm", + MediaType::TsBuildInfo => "TsBuildInfo", + MediaType::SourceMap => "SourceMap", MediaType::Unknown => "Unknown", }; write!(f, "{}", value) @@ -56,6 +60,12 @@ impl<'a> From<&'a String> for MediaType { } } +impl Default for MediaType { + fn default() -> Self { + MediaType::Unknown + } +} + impl MediaType { fn from_path(path: &Path) -> Self { match path.extension() { @@ -69,10 +79,42 @@ impl MediaType { Some("cjs") => MediaType::JavaScript, Some("json") => MediaType::Json, Some("wasm") => MediaType::Wasm, + Some("tsbuildinfo") => MediaType::TsBuildInfo, + Some("map") => MediaType::SourceMap, _ => MediaType::Unknown, }, } } + + /// Convert a MediaType to a `ts.Extension`. + /// + /// *NOTE* This is defined in TypeScript as a string based enum. Changes to + /// that enum in TypeScript should be reflected here. + pub fn as_ts_extension(&self) -> String { + let ext = match self { + MediaType::JavaScript => ".js", + MediaType::JSX => ".jsx", + MediaType::TypeScript => ".ts", + MediaType::Dts => ".d.ts", + MediaType::TSX => ".tsx", + MediaType::Json => ".json", + // TypeScript doesn't have an "unknown", so we will treat WASM as JS for + // mapping purposes, though in reality, it is unlikely to ever be passed + // to the compiler. + MediaType::Wasm => ".js", + MediaType::TsBuildInfo => ".tsbuildinfo", + // TypeScript doesn't have an "source map", so we will treat SourceMap as + // JS for mapping purposes, though in reality, it is unlikely to ever be + // passed to the compiler. + MediaType::SourceMap => ".js", + // TypeScript doesn't have an "unknown", so we will treat WASM as JS for + // mapping purposes, though in reality, it is unlikely to ever be passed + // to the compiler. + MediaType::Unknown => ".js", + }; + + ext.into() + } } impl Serialize for MediaType { @@ -88,7 +130,9 @@ impl Serialize for MediaType { MediaType::TSX => 4 as i32, MediaType::Json => 5 as i32, MediaType::Wasm => 6 as i32, - MediaType::Unknown => 8 as i32, + MediaType::TsBuildInfo => 7 as i32, + MediaType::SourceMap => 8 as i32, + MediaType::Unknown => 9 as i32, }; Serialize::serialize(&value, serializer) } @@ -132,6 +176,14 @@ mod tests { MediaType::from(Path::new("foo/bar.cjs")), MediaType::JavaScript ); + assert_eq!( + MediaType::from(Path::new("foo/.tsbuildinfo")), + MediaType::TsBuildInfo + ); + assert_eq!( + MediaType::from(Path::new("foo/bar.js.map")), + MediaType::SourceMap + ); assert_eq!( MediaType::from(Path::new("foo/bar.txt")), MediaType::Unknown @@ -148,7 +200,9 @@ mod tests { assert_eq!(json!(MediaType::TSX), json!(4)); assert_eq!(json!(MediaType::Json), json!(5)); assert_eq!(json!(MediaType::Wasm), json!(6)); - assert_eq!(json!(MediaType::Unknown), json!(8)); + assert_eq!(json!(MediaType::TsBuildInfo), json!(7)); + assert_eq!(json!(MediaType::SourceMap), json!(8)); + assert_eq!(json!(MediaType::Unknown), json!(9)); } #[test] @@ -160,6 +214,8 @@ mod tests { assert_eq!(format!("{}", MediaType::TSX), "TSX"); assert_eq!(format!("{}", MediaType::Json), "Json"); assert_eq!(format!("{}", MediaType::Wasm), "Wasm"); + assert_eq!(format!("{}", MediaType::TsBuildInfo), "BuildInfo"); + assert_eq!(format!("{}", MediaType::SourceMap), "SourceMap"); assert_eq!(format!("{}", MediaType::Unknown), "Unknown"); } } diff --git a/cli/module_graph.rs b/cli/module_graph.rs index a0ecb6d407fcfc..49521c7cce731e 100644 --- a/cli/module_graph.rs +++ b/cli/module_graph.rs @@ -465,7 +465,7 @@ impl ModuleGraphLoader { filename: source_file.filename.to_str().unwrap().to_string(), version_hash: checksum::gen(&[ &source_file.source_code.as_bytes(), - version::DENO.as_bytes(), + &version::DENO.as_bytes(), ]), media_type: source_file.media_type, source_code: "".to_string(), @@ -481,7 +481,7 @@ impl ModuleGraphLoader { let module_specifier = ModuleSpecifier::from(source_file.url.clone()); let version_hash = checksum::gen(&[ &source_file.source_code.as_bytes(), - version::DENO.as_bytes(), + &version::DENO.as_bytes(), ]); let source_code = source_file.source_code.clone(); diff --git a/cli/module_graph2.rs b/cli/module_graph2.rs index 69858641342c84..ddda524c9f1310 100644 --- a/cli/module_graph2.rs +++ b/cli/module_graph2.rs @@ -41,7 +41,7 @@ use std::sync::Mutex; use std::time::Instant; use swc_ecmascript::dep_graph::DependencyKind; -pub type BuildInfoMap = HashMap; +pub type TsBuildInfoMap = HashMap; lazy_static! { /// Matched the `@deno-types` pragma. @@ -379,8 +379,8 @@ impl Module { } } -#[derive(Clone, Debug, PartialEq)] -pub struct Stats(Vec<(String, u128)>); +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Stats(pub Vec<(String, u128)>); impl<'de> Deserialize<'de> for Stats { fn deserialize(deserializer: D) -> result::Result @@ -418,10 +418,10 @@ pub struct TranspileOptions { /// be able to manipulate and handle the graph. #[derive(Debug)] pub struct Graph2 { - build_info: BuildInfoMap, handler: Rc>, modules: HashMap, roots: Vec, + ts_build_info: TsBuildInfoMap, } impl Graph2 { @@ -432,10 +432,10 @@ impl Graph2 { /// pub fn new(handler: Rc>) -> Self { Graph2 { - build_info: HashMap::new(), handler, modules: HashMap::new(), roots: Vec::new(), + ts_build_info: HashMap::new(), } } @@ -568,11 +568,11 @@ impl Graph2 { } } for root_specifier in self.roots.iter() { - if let Some(build_info) = self.build_info.get(&emit_type) { - handler.set_build_info( + if let Some(ts_build_info) = self.ts_build_info.get(&emit_type) { + handler.set_ts_build_info( root_specifier, &emit_type, - build_info.to_owned(), + ts_build_info.to_owned(), )?; } } @@ -580,6 +580,27 @@ impl Graph2 { Ok(()) } + pub fn get_media_type( + &self, + specifier: &ModuleSpecifier, + ) -> Option { + if let Some(module) = self.modules.get(specifier) { + Some(module.media_type) + } else { + None + } + } + + /// Get the source for a given module specifier. If the module is not part + /// of the graph, the result will be `None`. + pub fn get_source(&self, specifier: &ModuleSpecifier) -> Option { + if let Some(module) = self.modules.get(specifier) { + Some(module.source.clone()) + } else { + None + } + } + /// Verify the subresource integrity of the graph based upon the optional /// lockfile, updating the lockfile with any missing resources. This will /// error if any of the resources do not match their lock status. @@ -603,6 +624,56 @@ impl Graph2 { Ok(()) } + /// Given a string specifier and a referring module specifier, provide the + /// resulting module specifier and media type for the module that is part of + /// the graph. + pub fn resolve( + &self, + specifier: &str, + referrer: &ModuleSpecifier, + ) -> Result { + if !self.modules.contains_key(referrer) { + return Err(MissingSpecifier(referrer.to_owned()).into()); + } + let module = self.modules.get(referrer).unwrap(); + if !module.dependencies.contains_key(specifier) { + return Err( + MissingDependency(referrer.to_owned(), specifier.to_owned()).into(), + ); + } + let dependency = module.dependencies.get(specifier).unwrap(); + // If there is a @deno-types pragma that impacts the dependency, then the + // maybe_type property will be set with that specifier, otherwise we use the + // specifier that point to the runtime code. + let resolved_specifier = + if let Some(type_specifier) = dependency.maybe_type.clone() { + type_specifier + } else if let Some(code_specifier) = dependency.maybe_code.clone() { + code_specifier + } else { + return Err( + MissingDependency(referrer.to_owned(), specifier.to_owned()).into(), + ); + }; + if !self.modules.contains_key(&resolved_specifier) { + return Err( + MissingDependency(referrer.to_owned(), resolved_specifier.to_string()) + .into(), + ); + } + let dep_module = self.modules.get(&resolved_specifier).unwrap(); + // In the case that there is a X-TypeScript-Types or a triple-slash types, + // then the `maybe_types` specifier will be populated and we should use that + // instead. + let result = if let Some((_, types)) = dep_module.maybe_types.clone() { + types + } else { + resolved_specifier + }; + + Ok(result) + } + /// Transpile (only transform) the graph, updating any emitted modules /// with the specifier handler. The result contains any performance stats /// from the compiler and optionally any user provided configuration compiler @@ -807,7 +878,7 @@ impl GraphBuilder2 { } #[cfg(test)] -mod tests { +pub mod tests { use super::*; use deno_core::futures::future; @@ -822,8 +893,8 @@ mod tests { #[derive(Debug, Default)] pub struct MockSpecifierHandler { pub fixtures: PathBuf, - pub build_info: HashMap, - pub build_info_calls: Vec<(ModuleSpecifier, EmitType, String)>, + pub ts_build_info: HashMap, + pub ts_build_info_calls: Vec<(ModuleSpecifier, EmitType, String)>, pub cache_calls: Vec<(ModuleSpecifier, EmitType, String, Option)>, pub deps_calls: Vec<(ModuleSpecifier, DependencyMap)>, pub types_calls: Vec<(ModuleSpecifier, String)>, @@ -871,12 +942,12 @@ mod tests { fn fetch(&mut self, specifier: ModuleSpecifier) -> FetchFuture { Box::pin(future::ready(self.get_cache(specifier))) } - fn get_build_info( + fn get_ts_build_info( &self, specifier: &ModuleSpecifier, _cache_type: &EmitType, ) -> Result, AnyError> { - Ok(self.build_info.get(specifier).cloned()) + Ok(self.ts_build_info.get(specifier).cloned()) } fn set_cache( &mut self, @@ -901,19 +972,19 @@ mod tests { self.types_calls.push((specifier.clone(), types)); Ok(()) } - fn set_build_info( + fn set_ts_build_info( &mut self, specifier: &ModuleSpecifier, cache_type: &EmitType, - build_info: String, + ts_build_info: String, ) -> Result<(), AnyError> { self - .build_info - .insert(specifier.clone(), build_info.clone()); - self.build_info_calls.push(( + .ts_build_info + .insert(specifier.clone(), ts_build_info.clone()); + self.ts_build_info_calls.push(( specifier.clone(), cache_type.clone(), - build_info, + ts_build_info, )); Ok(()) } diff --git a/cli/specifier_handler.rs b/cli/specifier_handler.rs index e392a3c3a8c6d5..9e070e106afbeb 100644 --- a/cli/specifier_handler.rs +++ b/cli/specifier_handler.rs @@ -100,7 +100,7 @@ pub trait SpecifierHandler { /// not expected to be cached for each module, but are "lazily" checked when /// a root module is identified. The `emit_type` also indicates what form /// of the module the build info is valid for. - fn get_build_info( + fn get_ts_build_info( &self, specifier: &ModuleSpecifier, emit_type: &EmitType, @@ -125,11 +125,11 @@ pub trait SpecifierHandler { ) -> Result<(), AnyError>; /// Set the build info for a module specifier, also providing the cache type. - fn set_build_info( + fn set_ts_build_info( &mut self, specifier: &ModuleSpecifier, emit_type: &EmitType, - build_info: String, + ts_build_info: String, ) -> Result<(), AnyError>; /// Set the graph dependencies for a given module specifier. @@ -283,7 +283,7 @@ impl SpecifierHandler for FetchHandler { .boxed_local() } - fn get_build_info( + fn get_ts_build_info( &self, specifier: &ModuleSpecifier, emit_type: &EmitType, @@ -294,18 +294,18 @@ impl SpecifierHandler for FetchHandler { let filename = self .disk_cache .get_cache_filename_with_extension(specifier.as_url(), "buildinfo"); - if let Ok(build_info) = self.disk_cache.get(&filename) { - return Ok(Some(String::from_utf8(build_info)?)); + if let Ok(ts_build_info) = self.disk_cache.get(&filename) { + return Ok(Some(String::from_utf8(ts_build_info)?)); } Ok(None) } - fn set_build_info( + fn set_ts_build_info( &mut self, specifier: &ModuleSpecifier, emit_type: &EmitType, - build_info: String, + ts_build_info: String, ) -> Result<(), AnyError> { if emit_type != &EmitType::Cli { return Err(UnsupportedEmitType(emit_type.clone()).into()); @@ -315,7 +315,7 @@ impl SpecifierHandler for FetchHandler { .get_cache_filename_with_extension(specifier.as_url(), "buildinfo"); self .disk_cache - .set(&filename, build_info.as_bytes()) + .set(&filename, ts_build_info.as_bytes()) .map_err(|e| e.into()) } diff --git a/cli/tests/tsc2/file_main.ts b/cli/tests/tsc2/file_main.ts new file mode 100644 index 00000000000000..a45477fde6eb7a --- /dev/null +++ b/cli/tests/tsc2/file_main.ts @@ -0,0 +1 @@ +console.log("hello deno"); diff --git a/cli/tests/tsc2/https_deno.land-x-a.ts b/cli/tests/tsc2/https_deno.land-x-a.ts new file mode 100644 index 00000000000000..72b3a67bc98702 --- /dev/null +++ b/cli/tests/tsc2/https_deno.land-x-a.ts @@ -0,0 +1,3 @@ +import * as b from "./b.ts"; + +console.log(b); diff --git a/cli/tests/tsc2/https_deno.land-x-b.ts b/cli/tests/tsc2/https_deno.land-x-b.ts new file mode 100644 index 00000000000000..59d1689930e55d --- /dev/null +++ b/cli/tests/tsc2/https_deno.land-x-b.ts @@ -0,0 +1 @@ +export const b = "b"; diff --git a/cli/tests/tsc2/https_deno.land-x-mod.ts b/cli/tests/tsc2/https_deno.land-x-mod.ts new file mode 100644 index 00000000000000..a45477fde6eb7a --- /dev/null +++ b/cli/tests/tsc2/https_deno.land-x-mod.ts @@ -0,0 +1 @@ +console.log("hello deno"); diff --git a/cli/tsc.rs b/cli/tsc.rs index aa83da280c0b48..56fc88bb4b55e8 100644 --- a/cli/tsc.rs +++ b/cli/tsc.rs @@ -417,7 +417,7 @@ impl TsCompiler { { let existing_hash = crate::checksum::gen(&[ &source_file.source_code.as_bytes(), - version::DENO.as_bytes(), + &version::DENO.as_bytes(), ]); let expected_hash = file_info["version"].as_str().unwrap().to_string(); @@ -988,7 +988,7 @@ fn execute_in_tsc( } let bootstrap_script = format!( - "globalThis.bootstrapCompilerRuntime({{ debugFlag: {} }})", + "globalThis.startup({{ debugFlag: {}, legacy: true }})", debug_flag ); js_runtime.execute("", &bootstrap_script)?; diff --git a/cli/tsc/99_main_compiler.js b/cli/tsc/99_main_compiler.js index 7bb0c6c9229015..f05876e62d3fc2 100644 --- a/cli/tsc/99_main_compiler.js +++ b/cli/tsc/99_main_compiler.js @@ -4,7 +4,7 @@ // that is created when Deno needs to compile TS/WASM to JS. // // It provides two functions that should be called by Rust: -// - `bootstrapCompilerRuntime` +// - `startup` // This functions must be called when creating isolate // to properly setup runtime. // - `tsCompilerOnMessage` @@ -54,6 +54,9 @@ delete Object.prototype.__proto__; } } + /** @type {Map} */ + const sourceFileCache = new Map(); + /** * @param {import("../dts/typescript").DiagnosticRelatedInformation} diagnostic */ @@ -296,15 +299,15 @@ delete Object.prototype.__proto__; debug(`host.fileExists("${fileName}")`); return false; }, - readFile(fileName) { - debug(`host.readFile("${fileName}")`); + readFile(specifier) { + debug(`host.readFile("${specifier}")`); if (legacy) { - if (fileName == TS_BUILD_INFO) { + if (specifier == TS_BUILD_INFO) { return legacyHostState.buildInfo; } return unreachable(); } else { - return core.jsonOpSync("op_read_file", { fileName }).data; + return core.jsonOpSync("op_load", { specifier }).data; } }, getSourceFile( @@ -338,6 +341,14 @@ delete Object.prototype.__proto__; ); sourceFile.tsSourceFile.version = sourceFile.versionHash; delete sourceFile.sourceCode; + + // This code is to support transition from the "legacy" compiler + // to the new one, by populating the new source file cache. + if ( + !sourceFileCache.has(specifier) && specifier.startsWith(ASSETS) + ) { + sourceFileCache.set(specifier, sourceFile.tsSourceFile); + } } return sourceFile.tsSourceFile; } catch (e) { @@ -349,37 +360,27 @@ delete Object.prototype.__proto__; return undefined; } } else { - const sourceFile = sourceFileCache.get(specifier); + let sourceFile = sourceFileCache.get(specifier); if (sourceFile) { return sourceFile; } - try { - /** @type {{ data: string; hash: string; }} */ - const { data, hash } = core.jsonOpSync( - "op_load_module", - { specifier }, - ); - const sourceFile = ts.createSourceFile( - specifier, - data, - languageVersion, - ); - sourceFile.moduleName = specifier; - sourceFile.version = hash; - sourceFileCache.set(specifier, sourceFile); - return sourceFile; - } catch (err) { - const message = err instanceof Error - ? err.message - : JSON.stringify(err); - debug(` !! error: ${message}`); - if (onError) { - onError(message); - } else { - throw err; - } - } + /** @type {{ data: string; hash: string; }} */ + const { data, hash } = core.jsonOpSync( + "op_load", + { specifier }, + ); + debug({ data, hash }); + assert(data, `"data" is unexpectedly null for "${specifier}".`); + sourceFile = ts.createSourceFile( + specifier, + data, + languageVersion, + ); + sourceFile.moduleName = specifier; + sourceFile.version = hash; + sourceFileCache.set(specifier, sourceFile); + return sourceFile; } }, getDefaultLibFileName() { @@ -392,7 +393,7 @@ delete Object.prototype.__proto__; return `${ASSETS}/lib.deno.worker.d.ts`; } } else { - return `lib.esnext.d.ts`; + return `${ASSETS}/lib.esnext.d.ts`; } }, getDefaultLibLocation() { @@ -403,16 +404,14 @@ delete Object.prototype.__proto__; if (legacy) { legacyHostState.writeFile(fileName, data, sourceFiles); } else { - let maybeModuleName; + let maybeSpecifiers; if (sourceFiles) { - assert(sourceFiles.length === 1, "unexpected number of source files"); - const [sourceFile] = sourceFiles; - maybeModuleName = sourceFile.moduleName; - debug(` moduleName: ${maybeModuleName}`); + maybeSpecifiers = sourceFiles.map((sf) => sf.moduleName); + debug(` specifiers: ${maybeSpecifiers.join(", ")}`); } return core.jsonOpSync( - "op_write_file", - { maybeModuleName, fileName, data }, + "op_emit", + { maybeSpecifiers, fileName, data }, ); } }, @@ -463,7 +462,7 @@ delete Object.prototype.__proto__; return resolved; } else { /** @type {Array<[string, import("../dts/typescript").Extension]>} */ - const resolved = core.jsonOpSync("op_resolve_specifiers", { + const resolved = core.jsonOpSync("op_resolve", { specifiers, base, }); @@ -737,6 +736,7 @@ delete Object.prototype.__proto__; 1208, ]; + /** @type {Array<{ key: string, value: number }>} */ const stats = []; let statsStart = 0; @@ -779,7 +779,6 @@ delete Object.prototype.__proto__; } function performanceEnd() { - // TODO(kitsonk) replace with performance.measure() when landed const duration = new Date() - statsStart; stats.push({ key: "Compile time", value: duration }); return stats; @@ -1328,18 +1327,73 @@ delete Object.prototype.__proto__; } } - let hasBootstrapped = false; + /** + * @typedef {object} Request + * @property {Record} config + * @property {boolean} debug + * @property {string[]} rootNames + */ - function bootstrapCompilerRuntime({ debugFlag }) { - if (hasBootstrapped) { - throw new Error("Worker runtime already bootstrapped"); + /** The API that is called by Rust when executing a request. + * @param {Request} request + */ + function exec({ config, debug: debugFlag, rootNames }) { + setLogDebug(debugFlag, "TS"); + performanceStart(); + debug(">>> exec start", { rootNames }); + debug(config); + + const { options, errors: configFileParsingDiagnostics } = ts + .convertCompilerOptionsFromJson(config, "", "tsconfig.json"); + const program = ts.createIncrementalProgram({ + rootNames, + options, + host, + configFileParsingDiagnostics, + }); + + const { diagnostics: emitDiagnostics } = program.emit(); + + const diagnostics = [ + ...program.getConfigFileParsingDiagnostics(), + ...program.getSyntacticDiagnostics(), + ...program.getOptionsDiagnostics(), + ...program.getGlobalDiagnostics(), + ...program.getSemanticDiagnostics(), + ...emitDiagnostics, + ].filter(({ code }) => + !IGNORED_DIAGNOSTICS.includes(code) && + !IGNORED_COMPILE_DIAGNOSTICS.includes(code) + ); + performanceProgram({ program }); + + // TODO(@kitsonk) when legacy stats are removed, convert to just tuples + let stats = performanceEnd().map(({ key, value }) => [key, value]); + core.jsonOpSync("op_respond", { + diagnostics: fromTypeScriptDiagnostic(diagnostics), + stats, + }); + debug("<<< exec stop"); + } + + let hasStarted = false; + + /** Startup the runtime environment, setting various flags. + * @param {{ debugFlag?: boolean; legacyFlag?: boolean; }} msg + */ + function startup({ debugFlag = false, legacyFlag = true }) { + if (hasStarted) { + throw new Error("The compiler runtime already started."); } - hasBootstrapped = true; - delete globalThis.__bootstrap; + hasStarted = true; core.ops(); + core.registerErrorClass("Error", Error); setLogDebug(!!debugFlag, "TS"); + legacy = legacyFlag; } - globalThis.bootstrapCompilerRuntime = bootstrapCompilerRuntime; + globalThis.startup = startup; + globalThis.exec = exec; + // TODO(@kitsonk) remove when converted from legacy tsc globalThis.tsCompilerOnMessage = tsCompilerOnMessage; })(this); diff --git a/cli/tsc2.rs b/cli/tsc2.rs new file mode 100644 index 00000000000000..79ddab84d5aa4d --- /dev/null +++ b/cli/tsc2.rs @@ -0,0 +1,571 @@ +// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. + +use crate::diagnostics::Diagnostics; +use crate::media_type::MediaType; +use crate::module_graph2::Graph2; +use crate::module_graph2::Stats; +use crate::tsc_config::TsConfig; + +use deno_core::error::anyhow; +use deno_core::error::bail; +use deno_core::error::AnyError; +use deno_core::error::Context; +use deno_core::json_op_sync; +use deno_core::serde_json; +use deno_core::serde_json::json; +use deno_core::serde_json::Value; +use deno_core::JsRuntime; +use deno_core::ModuleSpecifier; +use deno_core::OpFn; +use deno_core::RuntimeOptions; +use deno_core::Snapshot; +use serde::Deserialize; +use serde::Serialize; +use std::cell::RefCell; +use std::rc::Rc; + +#[derive(Debug, Clone, Default, Eq, PartialEq)] +pub struct EmittedFile { + pub data: String, + pub maybe_specifiers: Option>, + pub media_type: MediaType, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Request { + pub config: TsConfig, + pub debug: bool, + pub root_names: Vec, +} + +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct Response { + /// Any diagnostics that have been returned from the checker. + pub diagnostics: Diagnostics, + /// Any files that were emitted during the check. + pub emitted_files: Vec, + /// If there was any build info associated with the exec request. + pub maybe_tsbuildinfo: Option, + /// Statistics from the check. + pub stats: Stats, +} + +struct State { + hash_data: Vec>, + emitted_files: Vec, + graph: Rc>, + maybe_tsbuildinfo: Option, + maybe_response: Option, +} + +impl State { + pub fn new( + graph: Rc>, + hash_data: Vec>, + maybe_tsbuildinfo: Option, + ) -> Self { + State { + hash_data, + emitted_files: Vec::new(), + graph, + maybe_tsbuildinfo, + maybe_response: None, + } + } +} + +fn op(op_fn: F) -> Box +where + F: Fn(&mut State, Value) -> Result + 'static, +{ + json_op_sync(move |s, args, _bufs| { + let state = s.borrow_mut::(); + op_fn(state, args) + }) +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct CreateHashArgs { + /// The string data to be used to generate the hash. This will be mixed with + /// other state data in Deno to derive the final hash. + data: String, +} + +fn create_hash(state: &mut State, args: Value) -> Result { + let v: CreateHashArgs = serde_json::from_value(args) + .context("Invalid request from JavaScript for \"op_create_hash\".")?; + let mut data = vec![v.data.as_bytes().to_owned()]; + data.extend_from_slice(&state.hash_data); + let hash = crate::checksum::gen(&data); + Ok(json!({ "hash": hash })) +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct EmitArgs { + /// The text data/contents of the file. + data: String, + /// The _internal_ filename for the file. This will be used to determine how + /// the file is cached and stored. + file_name: String, + /// A string representation of the specifier that was associated with a + /// module. This should be present on every module that represents a module + /// that was requested to be transformed. + maybe_specifiers: Option>, +} + +fn emit(state: &mut State, args: Value) -> Result { + let v: EmitArgs = serde_json::from_value(args) + .context("Invalid request from JavaScript for \"op_emit\".")?; + match v.file_name.as_ref() { + "deno:///.tsbuildinfo" => state.maybe_tsbuildinfo = Some(v.data), + _ => state.emitted_files.push(EmittedFile { + data: v.data, + maybe_specifiers: if let Some(specifiers) = &v.maybe_specifiers { + let specifiers = specifiers + .iter() + .map(|s| ModuleSpecifier::resolve_url_or_path(s).unwrap()) + .collect(); + Some(specifiers) + } else { + None + }, + media_type: MediaType::from(&v.file_name), + }), + } + + Ok(json!(true)) +} + +#[derive(Debug, Deserialize)] +struct LoadArgs { + /// The fully qualified specifier that should be loaded. + specifier: String, +} + +fn load(state: &mut State, args: Value) -> Result { + let v: LoadArgs = serde_json::from_value(args) + .context("Invalid request from JavaScript for \"op_load\".")?; + let graph = state.graph.borrow(); + let specifier = ModuleSpecifier::resolve_url_or_path(&v.specifier) + .context("Error converting a string module specifier for \"op_load\".")?; + let mut hash: Option = None; + let data = if &v.specifier == "deno:///.tsbuildinfo" { + state.maybe_tsbuildinfo.clone() + } else { + let maybe_source = graph.get_source(&specifier); + if let Some(source) = &maybe_source { + let mut data = vec![source.as_bytes().to_owned()]; + data.extend_from_slice(&state.hash_data); + hash = Some(crate::checksum::gen(&data)); + } + maybe_source + }; + + Ok(json!({ "data": data, "hash": hash })) +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct ResolveArgs { + /// The base specifier that the supplied specifier strings should be resolved + /// relative to. + base: String, + /// A list of specifiers that should be resolved. + specifiers: Vec, +} + +fn resolve(state: &mut State, args: Value) -> Result { + let v: ResolveArgs = serde_json::from_value(args) + .context("Invalid request from JavaScript for \"op_resolve\".")?; + let graph = state.graph.borrow(); + let mut resolved: Vec<(String, String)> = Vec::new(); + let referrer = ModuleSpecifier::resolve_url_or_path(&v.base).context( + "Error converting a string module specifier for \"op_resolve\".", + )?; + for specifier in &v.specifiers { + if specifier.starts_with("asset:///") { + resolved.push(( + specifier.clone(), + MediaType::from(specifier).as_ts_extension().to_string(), + )); + } else { + let resolved_specifier = graph.resolve(specifier, &referrer)?; + let media_type = + if let Some(media_type) = graph.get_media_type(&resolved_specifier) { + media_type + } else { + bail!( + "Unable to resolve media type for specifier: \"{}\"", + resolved_specifier + ) + }; + resolved + .push((resolved_specifier.to_string(), media_type.as_ts_extension())); + } + } + + Ok(json!(resolved)) +} + +#[derive(Debug, Deserialize, Eq, PartialEq)] +pub struct RespondArgs { + pub diagnostics: Diagnostics, + pub stats: Stats, +} + +fn respond(state: &mut State, args: Value) -> Result { + let v: RespondArgs = serde_json::from_value(args) + .context("Error converting the result for \"op_respond\".")?; + state.maybe_response = Some(v); + Ok(json!(true)) +} + +pub fn exec( + request: Request, + snapshot: Snapshot, + graph: Rc>, + hash_data: Vec>, + maybe_tsbuildinfo: Option, +) -> Result { + let mut runtime = JsRuntime::new(RuntimeOptions { + startup_snapshot: Some(snapshot), + ..Default::default() + }); + + { + let op_state = runtime.op_state(); + let mut op_state = op_state.borrow_mut(); + op_state.put(State::new(graph, hash_data, maybe_tsbuildinfo)); + } + + runtime.register_op("op_create_hash", op(create_hash)); + runtime.register_op("op_emit", op(emit)); + runtime.register_op("op_load", op(load)); + runtime.register_op("op_resolve", op(resolve)); + runtime.register_op("op_respond", op(respond)); + + let startup_source = "globalThis.startup({ legacyFlag: false })"; + let request_str = + serde_json::to_string(&request).context("Could not serialize request.")?; + let exec_source = format!("globalThis.exec({})", request_str); + + runtime + .execute("[native code]", startup_source) + .context("Could not properly start the compiler runtime.")?; + runtime + .execute("[native_code]", &exec_source) + .context("Execute request failed.")?; + + let op_state = runtime.op_state(); + let mut op_state = op_state.borrow_mut(); + let state = op_state.take::(); + + if let Some(response) = state.maybe_response { + let diagnostics = response.diagnostics; + let emitted_files = state.emitted_files; + let maybe_tsbuildinfo = state.maybe_tsbuildinfo; + let stats = response.stats; + + Ok(Response { + diagnostics, + emitted_files, + maybe_tsbuildinfo, + stats, + }) + } else { + Err(anyhow!("The response for the exec request was not set.")) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::diagnostics::Diagnostic; + use crate::diagnostics::DiagnosticCategory; + use crate::js; + use crate::module_graph2::tests::MockSpecifierHandler; + use crate::module_graph2::GraphBuilder2; + use crate::tsc_config::TsConfig; + use std::env; + use std::path::PathBuf; + + async fn setup( + maybe_specifier: Option, + maybe_hash_data: Option>>, + maybe_tsbuildinfo: Option, + ) -> State { + let specifier = maybe_specifier.unwrap_or_else(|| { + ModuleSpecifier::resolve_url_or_path("file:///main.ts").unwrap() + }); + let hash_data = maybe_hash_data.unwrap_or_else(|| vec![b"".to_vec()]); + let c = PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").unwrap()); + let fixtures = c.join("tests/tsc2"); + let handler = Rc::new(RefCell::new(MockSpecifierHandler { + fixtures, + ..MockSpecifierHandler::default() + })); + let mut builder = GraphBuilder2::new(handler.clone(), None); + builder + .insert(&specifier) + .await + .expect("module not inserted"); + let graph = Rc::new(RefCell::new( + builder.get_graph(&None).expect("could not get graph"), + )); + State::new(graph, hash_data, maybe_tsbuildinfo) + } + + #[tokio::test] + async fn test_create_hash() { + let mut state = setup(None, Some(vec![b"something".to_vec()]), None).await; + let actual = + create_hash(&mut state, json!({ "data": "some sort of content" })) + .expect("could not invoke op"); + assert_eq!( + actual, + json!({"hash": "ae92df8f104748768838916857a1623b6a3c593110131b0a00f81ad9dac16511"}) + ); + } + + #[tokio::test] + async fn test_emit() { + let mut state = setup(None, None, None).await; + let actual = emit( + &mut state, + json!({ + "data": "some file content", + "fileName": "cache:///some/file.js", + "maybeSpecifiers": ["file:///some/file.ts"] + }), + ) + .expect("should have invoked op"); + assert_eq!(actual, json!(true)); + assert_eq!(state.emitted_files.len(), 1); + assert!(state.maybe_tsbuildinfo.is_none()); + assert_eq!( + state.emitted_files[0], + EmittedFile { + data: "some file content".to_string(), + maybe_specifiers: Some(vec![ModuleSpecifier::resolve_url_or_path( + "file:///some/file.ts" + ) + .unwrap()]), + media_type: MediaType::JavaScript, + } + ); + } + + #[tokio::test] + async fn test_emit_tsbuildinfo() { + let mut state = setup(None, None, None).await; + let actual = emit( + &mut state, + json!({ + "data": "some file content", + "fileName": "deno:///.tsbuildinfo", + }), + ) + .expect("should have invoked op"); + assert_eq!(actual, json!(true)); + assert_eq!(state.emitted_files.len(), 0); + assert_eq!( + state.maybe_tsbuildinfo, + Some("some file content".to_string()) + ); + } + + #[tokio::test] + async fn test_load() { + let mut state = setup( + Some( + ModuleSpecifier::resolve_url_or_path("https://deno.land/x/mod.ts") + .unwrap(), + ), + None, + Some("some content".to_string()), + ) + .await; + let actual = load( + &mut state, + json!({ "specifier": "https://deno.land/x/mod.ts"}), + ) + .expect("should have invoked op"); + assert_eq!( + actual, + json!({ + "data": "console.log(\"hello deno\");\n", + "hash": "149c777056afcc973d5fcbe11421b6d5ddc57b81786765302030d7fc893bf729" + }) + ); + } + + #[tokio::test] + async fn test_load_tsbuildinfo() { + let mut state = setup( + Some( + ModuleSpecifier::resolve_url_or_path("https://deno.land/x/mod.ts") + .unwrap(), + ), + None, + Some("some content".to_string()), + ) + .await; + let actual = + load(&mut state, json!({ "specifier": "deno:///.tsbuildinfo"})) + .expect("should have invoked op"); + assert_eq!( + actual, + json!({ + "data": "some content", + "hash": null + }) + ); + } + + #[tokio::test] + async fn test_load_missing_specifier() { + let mut state = setup(None, None, None).await; + let actual = load( + &mut state, + json!({ "specifier": "https://deno.land/x/mod.ts"}), + ) + .expect("should have invoked op"); + assert_eq!( + actual, + json!({ + "data": null, + "hash": null, + }) + ) + } + + #[tokio::test] + async fn test_resolve() { + let mut state = setup( + Some( + ModuleSpecifier::resolve_url_or_path("https://deno.land/x/a.ts") + .unwrap(), + ), + None, + None, + ) + .await; + let actual = resolve( + &mut state, + json!({ "base": "https://deno.land/x/a.ts", "specifiers": [ "./b.ts" ]}), + ) + .expect("should have invoked op"); + assert_eq!(actual, json!([["https://deno.land/x/b.ts", ".ts"]])); + } + + #[tokio::test] + async fn test_resolve_error() { + let mut state = setup( + Some( + ModuleSpecifier::resolve_url_or_path("https://deno.land/x/a.ts") + .unwrap(), + ), + None, + None, + ) + .await; + resolve( + &mut state, + json!({ "base": "https://deno.land/x/a.ts", "specifiers": [ "./bad.ts" ]}), + ).expect_err("should have errored"); + } + + #[tokio::test] + async fn test_respond() { + let mut state = setup(None, None, None).await; + let actual = respond( + &mut state, + json!({ + "diagnostics": [ + { + "messageText": "Unknown compiler option 'invalid'.", + "category": 1, + "code": 5023 + } + ], + "stats": [["a", 12]] + }), + ) + .expect("should have invoked op"); + assert_eq!(actual, json!(true)); + assert_eq!( + state.maybe_response, + Some(RespondArgs { + diagnostics: Diagnostics(vec![Diagnostic { + category: DiagnosticCategory::Error, + code: 5023, + start: None, + end: None, + message_text: Some( + "Unknown compiler option \'invalid\'.".to_string() + ), + message_chain: None, + source: None, + source_line: None, + file_name: None, + related_information: None, + }]), + stats: Stats(vec![("a".to_string(), 12)]) + }) + ); + } + + #[tokio::test] + async fn test_exec() { + let specifier = + ModuleSpecifier::resolve_url_or_path("https://deno.land/x/a.ts").unwrap(); + let hash_data = vec![b"something".to_vec()]; + let c = PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").unwrap()); + let fixtures = c.join("tests/tsc2"); + let handler = Rc::new(RefCell::new(MockSpecifierHandler { + fixtures, + ..MockSpecifierHandler::default() + })); + let mut builder = GraphBuilder2::new(handler.clone(), None); + builder + .insert(&specifier) + .await + .expect("module not inserted"); + let graph = Rc::new(RefCell::new( + builder.get_graph(&None).expect("could not get graph"), + )); + let config = TsConfig::new(json!({ + "allowJs": true, + "checkJs": false, + "esModuleInterop": true, + "emitDecoratorMetadata": false, + "incremental": true, + "isolatedModules": true, + "jsx": "react", + "jsxFactory": "React.createElement", + "jsxFragmentFactory": "React.Fragment", + "lib": ["deno.window"], + "module": "esnext", + "noEmit": true, + "outDir": "deno:///", + "strict": true, + "target": "esnext", + "tsBuildInfoFile": "deno:///.tsbuildinfo", + })); + let request = Request { + config, + debug: true, + root_names: vec!["https://deno.land/x/a.ts".to_string()], + }; + let actual = + exec(request, js::compiler_isolate_init(), graph, hash_data, None) + .expect("exec should have not errored"); + assert!(actual.diagnostics.0.is_empty()); + assert!(actual.emitted_files.is_empty()); + assert!(actual.maybe_tsbuildinfo.is_some()); + assert_eq!(actual.stats.0.len(), 12); + } +} diff --git a/core/error.rs b/core/error.rs index 8a80e30246543a..7f16af6c125bdf 100644 --- a/core/error.rs +++ b/core/error.rs @@ -1,5 +1,8 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. +pub use anyhow::anyhow; +pub use anyhow::bail; +pub use anyhow::Context; use rusty_v8 as v8; use std::borrow::Cow; use std::convert::TryFrom;