From 1721e1c166ca498fab4123ef83e67790273572d2 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Fri, 12 Jan 2024 14:56:20 -0500 Subject: [PATCH 01/14] feat: store bytes of modules --- Cargo.lock | 10 + Cargo.toml | 1 + js/types.ts | 2 +- lib/lib.rs | 2 +- src/graph.rs | 225 +++++++++++++----- src/lib.rs | 45 ++-- src/source/mod.rs | 102 +++++++- src/symbols/analyzer.rs | 53 ++--- src/symbols/mod.rs | 2 +- src/text_encoding.rs | 67 ++++++ tests/integration_test.rs | 10 +- tests/specs/symbols/Basic.txt | 4 +- .../symbols/ClassPrivateCtorParamProps.txt | 2 +- tests/specs/symbols/Classes01.txt | 2 +- tests/specs/symbols/DeclarationMerging01.txt | 2 +- tests/specs/symbols/ExportAssignment01.txt | 2 +- tests/specs/symbols/ExportDefault01.txt | 10 +- tests/specs/symbols/ExportDefault02.txt | 2 +- tests/specs/symbols/ExportDefault03.txt | 6 +- tests/specs/symbols/ExportDefault04.txt | 2 +- .../specs/symbols/ExportDefaultLiteral01.txt | 2 +- .../specs/symbols/ExportDefaultLiteral02.txt | 2 +- tests/specs/symbols/ExportDestructured.txt | 2 +- tests/specs/symbols/ExportStar01.txt | 10 +- tests/specs/symbols/ExportedDeclare01.txt | 2 +- tests/specs/symbols/ExportedDeclare02.txt | 2 +- tests/specs/symbols/FunctionOverloads01.txt | 2 +- tests/specs/symbols/Functions01.txt | 2 +- tests/specs/symbols/ImportEquals01.txt | 2 +- tests/specs/symbols/ImportEquals02.txt | 4 +- tests/specs/symbols/ImportType01.txt | 4 +- tests/specs/symbols/ImportType02.txt | 4 +- tests/specs/symbols/Interfaces01.txt | 2 +- tests/specs/symbols/Interfaces02.txt | 4 +- tests/specs/symbols/MemberExpr.txt | 2 +- tests/specs/symbols/ModuleExportNameStr01.txt | 8 +- tests/specs/symbols/Namespaces01.txt | 2 +- tests/specs/symbols/Namespaces02.txt | 2 +- tests/specs/symbols/NamespacesAmbient.txt | 4 +- tests/specs/symbols/ReExportJson01.txt | 2 +- tests/specs/symbols/ReexportAll.txt | 8 +- tests/specs/symbols/ReexportDeep.txt | 8 +- tests/specs/symbols/ReexportExport.txt | 2 +- tests/specs/symbols/TsExternalModuleRef01.txt | 4 +- tests/specs/symbols/TsExternalModuleRef02.txt | 4 +- tests/specs/symbols/TsNamespaceExport01.txt | 2 +- tests/specs/symbols/TsQualifiedName01.txt | 2 +- tests/specs/symbols/TypeAlias01.txt | 2 +- tests/specs/symbols/TypeScriptTypesHeader.txt | 12 +- tests/specs/symbols/TypesEntrypoint.txt | 4 +- tests/specs/symbols/UnresolvedParts.txt | 4 +- tests/specs/symbols/UsingDecl.txt | 2 +- 52 files changed, 474 insertions(+), 195 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2be236339..0492f2e39 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -248,6 +248,7 @@ dependencies = [ "data-url", "deno_ast", "deno_semver", + "encoding_rs", "futures", "import_map", "indexmap 2.0.0", @@ -343,6 +344,15 @@ version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" +[[package]] +name = "encoding_rs" +version = "0.8.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" +dependencies = [ + "cfg-if", +] + [[package]] name = "equivalent" version = "1.0.1" diff --git a/Cargo.toml b/Cargo.toml index fc1460da8..63907c3a3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,6 +29,7 @@ async-trait = "0.1.68" data-url = "0.3.0" deno_ast = { version = "1.0.0", features = ["dep_analysis", "module_specifier"] } deno_semver = "0.5.0" +encoding_rs = "0.8.33" futures = "0.3.26" import_map = "0.18.0" indexmap = { version = "2", features = ["serde"] } diff --git a/js/types.ts b/js/types.ts index e54c6a8cd..567f93d5e 100644 --- a/js/types.ts +++ b/js/types.ts @@ -34,7 +34,7 @@ export interface LoadResponseModule { * have been normalized to be lower case values. */ headers?: Record; /** The string value of the loaded resources. */ - content: string; + content: string | Uint8Array; } export interface LoadResponseExternal { diff --git a/lib/lib.rs b/lib/lib.rs index b9c903ed6..b7c92c181 100644 --- a/lib/lib.rs +++ b/lib/lib.rs @@ -274,7 +274,7 @@ pub fn js_parse_module( maybe_headers: JsValue, maybe_default_jsx_import_source: Option, maybe_jsx_import_source_module: Option, - content: String, + content: Vec, maybe_resolve: Option, maybe_resolve_types: Option, ) -> Result { diff --git a/src/graph.rs b/src/graph.rs index d8ad62205..810f48e3d 100644 --- a/src/graph.rs +++ b/src/graph.rs @@ -23,6 +23,7 @@ use crate::packages::JsrPackageInfo; use crate::packages::JsrPackageVersionInfo; use crate::packages::PackageSpecifiers; use crate::source::*; +use crate::text_encoding; use anyhow::anyhow; use deno_ast::dep::DependencyKind; @@ -691,7 +692,7 @@ pub struct WorkspaceMember { #[serde(rename_all = "camelCase")] #[serde(tag = "kind")] pub enum Module { - Esm(EsmModule), + Esm(EsModule), // todo(#239): remove this when updating the --json output for 2.0 #[serde(rename = "asserted")] Json(JsonModule), @@ -719,7 +720,7 @@ impl Module { } } - pub fn esm(&self) -> Option<&EsmModule> { + pub fn esm(&self) -> Option<&EsModule> { if let Module::Esm(module) = &self { Some(module) } else { @@ -786,8 +787,10 @@ pub struct JsonModule { pub specifier: ModuleSpecifier, #[serde(flatten, skip_serializing_if = "Option::is_none")] pub maybe_cache_info: Option, - #[serde(rename = "size", serialize_with = "serialize_source")] - pub source: Arc, + #[serde(skip_serializing)] + pub text: Arc, + #[serde(rename = "size", serialize_with = "serialize_source_bytes")] + pub bytes: Arc<[u8]>, // todo(#240): This will always be MediaType::Json, but it's currently // used in the --json output. It's redundant though. pub media_type: MediaType, @@ -796,7 +799,7 @@ pub struct JsonModule { impl JsonModule { /// Return the size in bytes of the content of the JSON module. pub fn size(&self) -> usize { - self.source.as_bytes().len() + self.text.as_bytes().len() } } @@ -813,9 +816,54 @@ pub struct FastCheckTypeModule { pub source_map: Arc<[u8]>, } +#[derive(Debug, Clone)] +pub struct EsModuleSource { + pub bytes: Arc<[u8]>, + pub text: Option>, +} + +impl EsModuleSource { + pub fn new_with_text(bytes: Arc<[u8]>) -> Result { + let charset = text_encoding::detect_charset(bytes.as_ref()); + let mut text = + match text_encoding::convert_to_utf8(bytes.as_ref(), &charset)? { + Cow::Borrowed(text) => { + if text.starts_with(text_encoding::BOM_CHAR) { + text.to_string() + } else { + return Ok(Self { + bytes: bytes.clone(), + // SAFETY: we know it's avalid utf-8 string at this point + text: unsafe { + let raw_ptr = Arc::into_raw(bytes); + Some(Arc::from_raw(std::mem::transmute::< + *const [u8], + *const str, + >(raw_ptr))) + }, + }); + } + } + Cow::Owned(text) => text, + }; + text_encoding::strip_bom_mut(&mut text); + let text: Arc = Arc::from(text); + Ok(Self::from_text(text)) + } + + /// This is internal because generally we don't want consumers doing + /// their own bytes decoding and instead to use `EsModuleSource::new_with_text`. + pub(crate) fn from_text(text: Arc) -> Self { + Self { + bytes: text.clone().into(), + text: Some(text), + } + } +} + #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "camelCase")] -pub struct EsmModule { +pub struct EsModule { #[serde( skip_serializing_if = "IndexMap::is_empty", serialize_with = "serialize_dependencies" @@ -823,8 +871,8 @@ pub struct EsmModule { pub dependencies: IndexMap, #[serde(flatten, skip_serializing_if = "Option::is_none")] pub maybe_cache_info: Option, - #[serde(rename = "size", serialize_with = "serialize_source")] - pub source: Arc, + #[serde(rename = "size", serialize_with = "serialize_esm_source")] + pub source: EsModuleSource, #[serde(rename = "typesDependency", skip_serializing_if = "Option::is_none")] pub maybe_types_dependency: Option, #[serde(skip_serializing_if = "is_media_type_unknown")] @@ -834,8 +882,8 @@ pub struct EsmModule { pub fast_check: Option, } -impl EsmModule { - fn new(specifier: ModuleSpecifier, source: Arc) -> Self { +impl EsModule { + fn new(specifier: ModuleSpecifier, source: EsModuleSource) -> Self { Self { dependencies: Default::default(), maybe_cache_info: None, @@ -847,11 +895,6 @@ impl EsmModule { } } - /// Return the size in bytes of the content of the module. - pub fn size(&self) -> usize { - self.source.as_bytes().len() - } - pub fn fast_check_diagnostic(&self) -> Option<&FastCheckDiagnostic> { let module_slot = self.fast_check.as_ref()?; match module_slot { @@ -1199,7 +1242,7 @@ impl<'a> ModuleGraphErrorIterator<'a> { fn check_resolution( &self, - module: &EsmModule, + module: &EsModule, mode: ResolutionMode, specifier_text: &str, resolution: &Resolution, @@ -1811,7 +1854,7 @@ pub(crate) fn parse_module( graph_kind: GraphKind, specifier: &ModuleSpecifier, maybe_headers: Option<&HashMap>, - content: Arc, + content: Arc<[u8]>, maybe_attribute_type: Option, maybe_referrer: Option, file_system: &dyn FileSystem, @@ -1834,9 +1877,11 @@ pub(crate) fn parse_module( Some("json") )) { + let source = new_source_with_text(&specifier, content)?; return Ok(Module::Json(JsonModule { maybe_cache_info: None, - source: content, + text: source.text.unwrap(), + bytes: source.bytes, media_type: MediaType::Json, specifier: specifier.clone(), })); @@ -1860,7 +1905,7 @@ pub(crate) fn parse_module( } // Here we check for known ES Modules that we will analyze the dependencies of - match &media_type { + match media_type { MediaType::JavaScript | MediaType::Mjs | MediaType::Cjs @@ -1872,16 +1917,21 @@ pub(crate) fn parse_module( | MediaType::Dts | MediaType::Dmts | MediaType::Dcts => { - match module_analyzer.analyze(specifier, content.clone(), media_type) { + let source = new_source_with_text(&specifier, content)?; + match module_analyzer.analyze( + specifier, + source.text.clone().unwrap(), + media_type, + ) { Ok(module_info) => { // Return the module as a valid module - Ok(Module::Esm(parse_esm_module_from_module_info( + Ok(Module::Esm(parse_es_module_from_module_info( graph_kind, specifier, media_type, maybe_headers, module_info, - content, + source, file_system, maybe_resolver, maybe_npm_resolver, @@ -1893,20 +1943,21 @@ pub(crate) fn parse_module( } } MediaType::Unknown if is_root => { + let source = new_source_with_text(&specifier, content)?; match module_analyzer.analyze( specifier, - content.clone(), + source.text.clone().unwrap(), MediaType::JavaScript, ) { Ok(module_info) => { // Return the module as a valid module - Ok(Module::Esm(parse_esm_module_from_module_info( + Ok(Module::Esm(parse_es_module_from_module_info( graph_kind, specifier, media_type, maybe_headers, module_info, - content, + source, file_system, maybe_resolver, maybe_npm_resolver, @@ -1917,7 +1968,11 @@ pub(crate) fn parse_module( } } } - _ => Err(ModuleError::UnsupportedMediaType( + MediaType::Json + | MediaType::Wasm + | MediaType::TsBuildInfo + | MediaType::SourceMap + | MediaType::Unknown => Err(ModuleError::UnsupportedMediaType( specifier.clone(), media_type, maybe_referrer, @@ -1926,18 +1981,18 @@ pub(crate) fn parse_module( } #[allow(clippy::too_many_arguments)] -pub(crate) fn parse_esm_module_from_module_info( +pub(crate) fn parse_es_module_from_module_info( graph_kind: GraphKind, specifier: &ModuleSpecifier, media_type: MediaType, maybe_headers: Option<&HashMap>, module_info: ModuleInfo, - source: Arc, + source: EsModuleSource, file_system: &dyn FileSystem, maybe_resolver: Option<&dyn Resolver>, maybe_npm_resolver: Option<&dyn NpmResolver>, -) -> EsmModule { - let mut module = EsmModule::new(specifier.clone(), source); +) -> EsModule { + let mut module = EsModule::new(specifier.clone(), source); module.media_type = media_type; // Analyze the TypeScript triple-slash references @@ -2654,7 +2709,7 @@ struct PendingState { #[derive(Clone)] enum ContentOrModuleInfo { - Content(Arc), + Content(Arc<[u8]>), ModuleInfo(ModuleInfo), } @@ -3120,11 +3175,40 @@ impl<'a, 'graph> Builder<'a, 'graph> { match slot { ModuleSlot::Module(module) => { match module { - Module::Esm(module) => { - module.source = content; - } + Module::Esm(module) => match module.media_type { + MediaType::JavaScript + | MediaType::Jsx + | MediaType::Mjs + | MediaType::Cjs + | MediaType::TypeScript + | MediaType::Mts + | MediaType::Cts + | MediaType::Dts + | MediaType::Dmts + | MediaType::Dcts + | MediaType::Tsx + | MediaType::Json => { + match new_source_with_text(&module.specifier, content) + { + Ok(source) => { + module.source = source; + } + Err(err) => *slot = ModuleSlot::Err(err), + } + } + MediaType::Wasm + | MediaType::TsBuildInfo + | MediaType::SourceMap + | MediaType::Unknown => todo!(), + }, Module::Json(module) => { - module.source = content; + match new_source_with_text(&module.specifier, content) { + Ok(source) => { + module.bytes = source.bytes; + module.text = source.text.unwrap(); + } + Err(err) => *slot = ModuleSlot::Err(err), + } } Module::Npm(_) | Module::Node(_) @@ -3587,7 +3671,7 @@ impl<'a, 'graph> Builder<'a, 'graph> { match data { Some(LoadResponse::Module { content, .. }) => { let package_info: JsrPackageInfo = - serde_json::from_str(&content).map_err(|e| Arc::new(e.into()))?; + serde_json::from_slice(&content).map_err(|e| Arc::new(e.into()))?; Ok(Some(Arc::new(package_info))) } _ => Ok(None), @@ -3629,7 +3713,7 @@ impl<'a, 'graph> Builder<'a, 'graph> { match data { Some(LoadResponse::Module { content, .. }) => { let version_info: JsrPackageVersionInfo = - serde_json::from_str(&content).map_err(|e| Arc::new(e.into()))?; + serde_json::from_slice(&content).map_err(|e| Arc::new(e.into()))?; Ok(Arc::new(version_info)) } _ => Err(Arc::new(anyhow!("Not found: {}", specifier))), @@ -3744,7 +3828,7 @@ impl<'a, 'graph> Builder<'a, 'graph> { .boxed_local() }); ( - String::new().into(), + Arc::new([]) as Arc<[u8]>, Some(ProvidedModuleAnalyzer(RefCell::new(Some(info)))), ) } @@ -4180,6 +4264,15 @@ impl<'a> NpmSpecifierResolver<'a> { } } +fn new_source_with_text( + specifier: &ModuleSpecifier, + text: Arc<[u8]>, +) -> Result { + EsModuleSource::new_with_text(text).map_err(|err| { + ModuleError::LoadingErr(specifier.clone(), None, Arc::new(err.into())) + }) +} + impl Serialize for Resolution { fn serialize(&self, serializer: S) -> Result where @@ -4228,8 +4321,18 @@ where seq.end() } -fn serialize_source( - source: &Arc, +fn serialize_esm_source( + source: &EsModuleSource, + serializer: S, +) -> Result +where + S: Serializer, +{ + serializer.serialize_u32(source.bytes.len() as u32) +} + +fn serialize_source_bytes( + source: &Arc<[u8]>, serializer: S, ) -> Result where @@ -4286,13 +4389,14 @@ mod tests { fn test_module_dependency_includes() { let specifier = ModuleSpecifier::parse("file:///a.ts").unwrap(); let module_analyzer = DefaultModuleAnalyzer::default(); - let content = - "import * as b from \"./b.ts\";\nimport * as c from \"./b.ts\""; + let content: Arc<[u8]> = Arc::from( + b"import * as b from \"./b.ts\";\nimport * as c from \"./b.ts\"".to_vec(), + ); let module = parse_module( GraphKind::All, &specifier, None, - content.into(), + content, None, None, &NullFileSystem, @@ -4389,7 +4493,7 @@ mod tests { Ok(Some(LoadResponse::Module { specifier: specifier.clone(), maybe_headers: None, - content: "await import('file:///bar.js')".into(), + content: b"await import('file:///bar.js')".to_vec().into(), })) }) } @@ -4400,7 +4504,7 @@ mod tests { Ok(Some(LoadResponse::Module { specifier: specifier.clone(), maybe_headers: None, - content: "import 'file:///baz.js'".into(), + content: b"import 'file:///baz.js'".to_vec().into(), })) }) } @@ -4411,7 +4515,7 @@ mod tests { Ok(Some(LoadResponse::Module { specifier: specifier.clone(), maybe_headers: None, - content: "console.log('Hello, world!')".into(), + content: b"console.log('Hello, world!')".to_vec().into(), })) }) } @@ -4455,7 +4559,7 @@ mod tests { Ok(Some(LoadResponse::Module { specifier: specifier.clone(), maybe_headers: None, - content: "await import('file:///bar.js')".into(), + content: b"await import('file:///bar.js')".to_vec().into(), })) }), "file:///bar.js" => Box::pin(async move { Ok(None) }), @@ -4542,14 +4646,14 @@ mod tests { Ok(Some(LoadResponse::Module { specifier: Url::parse("file:///foo_actual.js").unwrap(), maybe_headers: None, - content: "import 'file:///bar.js'".into(), + content: b"import 'file:///bar.js'".to_vec().into(), })) }), "file:///bar.js" => Box::pin(async move { Ok(Some(LoadResponse::Module { specifier: Url::parse("file:///bar_actual.js").unwrap(), maybe_headers: None, - content: "(".into(), + content: b"(".to_vec().into(), })) }), _ => unreachable!(), @@ -4611,29 +4715,29 @@ mod tests { specifier: specifier.clone(), maybe_headers: None, content: - "import 'FILE:///baz.js'; import 'file:///bar.js'; import 'http://deno.land/foo.js';" - .into(), + b"import 'FILE:///baz.js'; import 'file:///bar.js'; import 'http://deno.land/foo.js';" + .to_vec().into(), })) }), "http://deno.land/foo.js" => Box::pin(async move { Ok(Some(LoadResponse::Module { specifier: specifier.clone(), maybe_headers: None, - content: "export {}".into(), + content: b"export {}".to_vec().into(), })) }), "file:///bar.js" => Box::pin(async move { Ok(Some(LoadResponse::Module { specifier: specifier.clone(), maybe_headers: None, - content: "console.log('Hello, world!')".into(), + content: b"console.log('Hello, world!')".to_vec().into(), })) }), "file:///baz.js" => Box::pin(async move { Ok(Some(LoadResponse::Module { specifier: specifier.clone(), maybe_headers: None, - content: "console.log('Hello, world 2!')".into(), + content: b"console.log('Hello, world 2!')".to_vec().into(), })) }), _ => unreachable!(), @@ -4736,7 +4840,9 @@ mod tests { specifier: specifier.clone(), maybe_headers: None, content: - "import 'file:///bar.js'; await import('file:///bar.js')".into(), + b"import 'file:///bar.js'; await import('file:///bar.js')" + .to_vec() + .into(), })) }), "file:///bar.js" => { @@ -4746,7 +4852,7 @@ mod tests { Ok(Some(LoadResponse::Module { specifier: specifier.clone(), maybe_headers: None, - content: "console.log('Hello, world!')".into(), + content: b"console.log('Hello, world!')".to_vec().into(), })) }) } @@ -4782,7 +4888,7 @@ mod tests { Ok(Some(LoadResponse::Module { specifier: specifier.clone(), maybe_headers: None, - content: " + content: b" /// /// /* @jsxImportSource file:///bar.ts */ @@ -4793,6 +4899,7 @@ mod tests { import type {} from 'file:///bar.ts'; /** @typedef { import('file:///bar.ts') } bar */ " + .to_vec() .into(), })) }), @@ -4802,7 +4909,7 @@ mod tests { Ok(Some(LoadResponse::Module { specifier: specifier.clone(), maybe_headers: None, - content: "".into(), + content: b"".to_vec().into(), })) }) } @@ -4812,7 +4919,7 @@ mod tests { Ok(Some(LoadResponse::Module { specifier: specifier.clone(), maybe_headers: None, - content: "{}".into(), + content: b"{}".to_vec().into(), })) }) } diff --git a/src/lib.rs b/src/lib.rs index fa6790696..1dbfe7f07 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -18,7 +18,6 @@ use source::NpmResolver; use source::Resolver; use std::collections::HashMap; - use std::sync::Arc; pub use analyzer::analyze_deno_types; @@ -49,7 +48,7 @@ pub use fast_check::FastCheckModule; pub use graph::BuildDiagnostic; pub use graph::BuildOptions; pub use graph::Dependency; -pub use graph::EsmModule; +pub use graph::EsModule; pub use graph::ExternalModule; pub use graph::FastCheckTypeModule; pub use graph::FastCheckTypeModuleSlot; @@ -91,7 +90,7 @@ pub struct ParseModuleOptions<'a> { pub graph_kind: GraphKind, pub specifier: &'a ModuleSpecifier, pub maybe_headers: Option<&'a HashMap>, - pub content: Arc, + pub content: Arc<[u8]>, pub file_system: &'a dyn FileSystem, pub maybe_resolver: Option<&'a dyn Resolver>, pub maybe_module_analyzer: Option<&'a dyn ModuleAnalyzer>, @@ -135,14 +134,14 @@ pub struct ParseModuleFromAstOptions<'a> { } /// Parse an individual module from an AST, returning the module. -pub fn parse_module_from_ast(options: ParseModuleFromAstOptions) -> EsmModule { - graph::parse_esm_module_from_module_info( +pub fn parse_module_from_ast(options: ParseModuleFromAstOptions) -> EsModule { + graph::parse_es_module_from_module_info( options.graph_kind, options.specifier, options.parsed_source.media_type(), options.maybe_headers, DefaultModuleAnalyzer::module_info(options.parsed_source), - options.parsed_source.text_info().text(), + graph::EsModuleSource::from_text(options.parsed_source.text_info().text()), options.file_system, options.maybe_resolver, options.maybe_npm_resolver, @@ -1769,8 +1768,11 @@ export function a(a) { assert_eq!(graph.module_slots.len(), 3); let data_specifier = ModuleSpecifier::parse("data:application/typescript,export%20*%20from%20%22https://example.com/c.ts%22;").unwrap(); let module = graph.get(&data_specifier).unwrap().esm().unwrap(); - let source: &str = &module.source; - assert_eq!(source, r#"export * from "https://example.com/c.ts";"#,); + let source = module.source.text.clone().unwrap(); + assert_eq!( + source.as_ref(), + r#"export * from "https://example.com/c.ts";"#, + ); } #[tokio::test] @@ -3006,7 +3008,7 @@ export function a(a) { #[test] fn test_parse_module() { let specifier = ModuleSpecifier::parse("file:///a/test01.ts").unwrap(); - let code = r#" + let code = br#" /// import { a } from "./a.ts"; import * as b from "./b.ts"; @@ -3019,7 +3021,7 @@ export function a(a) { graph_kind: GraphKind::All, specifier: &specifier, maybe_headers: None, - content: code.into(), + content: code.to_vec().into(), file_system: &NullFileSystem, maybe_resolver: None, maybe_module_analyzer: None, @@ -3036,7 +3038,7 @@ export function a(a) { graph_kind: GraphKind::CodeOnly, specifier: &specifier, maybe_headers: None, - content: code.into(), + content: code.to_vec().into(), file_system: &NullFileSystem, maybe_resolver: None, maybe_module_analyzer: None, @@ -3054,10 +3056,11 @@ export function a(a) { graph_kind: GraphKind::All, specifier: &specifier, maybe_headers: None, - content: r#" + content: br#" import a from "./a.json" assert { type: "json" }; await import("./b.json", { assert: { type: "json" } }); "# + .to_vec() .into(), file_system: &NullFileSystem, maybe_resolver: None, @@ -3120,13 +3123,14 @@ export function a(a) { graph_kind: GraphKind::All, specifier: &specifier, maybe_headers: None, - content: r#" + content: br#" /** @jsxImportSource https://example.com/preact */ export function A() { return
Hello Deno
; } "# + .to_vec() .into(), file_system: &NullFileSystem, maybe_resolver: None, @@ -3164,11 +3168,12 @@ export function a(a) { graph_kind: GraphKind::All, specifier: &specifier, maybe_headers: None, - content: r#" + content: br#" export function A() { return
Hello Deno
; } "# + .to_vec() .into(), file_system: &NullFileSystem, maybe_resolver: Some(&R), @@ -3204,9 +3209,10 @@ export function a(a) { graph_kind: GraphKind::All, specifier: &specifier, maybe_headers, - content: r#"declare interface A { + content: br#"declare interface A { a: string; }"# + .to_vec() .into(), file_system: &NullFileSystem, maybe_resolver: None, @@ -3219,7 +3225,7 @@ export function a(a) { #[test] fn test_parse_module_with_jsdoc_imports() { let specifier = ModuleSpecifier::parse("file:///a/test.js").unwrap(); - let code = r#" + let code = br#" /** * Some js doc * @@ -3234,7 +3240,7 @@ export function a(a) { graph_kind: GraphKind::All, specifier: &specifier, maybe_headers: None, - content: code.into(), + content: code.to_vec().into(), file_system: &NullFileSystem, maybe_resolver: None, maybe_module_analyzer: None, @@ -3290,7 +3296,7 @@ export function a(a) { graph_kind: GraphKind::CodeOnly, specifier: &specifier, maybe_headers: None, - content: code.into(), + content: code.to_vec().into(), file_system: &NullFileSystem, maybe_resolver: None, maybe_module_analyzer: None, @@ -3315,7 +3321,7 @@ export function a(a) { graph_kind: GraphKind::All, specifier: &specifier, maybe_headers: None, - content: r#" + content: br#" /** * Some js doc * @@ -3326,6 +3332,7 @@ export function a(a: A): B { return; } "# + .to_vec() .into(), file_system: &NullFileSystem, maybe_resolver: None, diff --git a/src/source/mod.rs b/src/source/mod.rs index 9027c8d76..361f8ea80 100644 --- a/src/source/mod.rs +++ b/src/source/mod.rs @@ -5,6 +5,7 @@ use crate::module_specifier::resolve_import; use crate::packages::JsrPackageInfo; use crate::packages::JsrPackageVersionInfo; use crate::text_encoding::strip_bom_mut; +use crate::text_encoding::BOM_CHAR; use crate::ModuleInfo; use crate::SpecifierError; use deno_semver::package::PackageNv; @@ -19,6 +20,7 @@ use futures::future::LocalBoxFuture; use once_cell::sync::Lazy; use serde::Deserialize; use serde::Serialize; +use std::borrow::Cow; use std::collections::HashMap; use std::fmt; use std::path::PathBuf; @@ -31,6 +33,96 @@ pub use file_system::*; pub const DEFAULT_JSX_IMPORT_SOURCE_MODULE: &str = "jsx-runtime"; +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub enum StringOrBytes { + String(Arc), + Bytes(Arc<[u8]>), +} + +impl StringOrBytes { + pub fn from_vec(bytes: Vec) -> StringOrBytes { + StringOrBytes::Bytes(Arc::from(bytes.into_boxed_slice())) + } + + pub fn into_arc_string(self) -> Arc { + match self { + Self::String(s) => s, + // note, this should never happen in practice + Self::Bytes(bytes) => { + // Parse some bytes into a UTF-8 string as the WHATWG encoding spec specifies + // in https://encoding.spec.whatwg.org/#utf-8-decode. + // + // This is used by the HTML spec to parse the bytes representing ES modules + // into strings. + let mut string = match String::from_utf8_lossy(&bytes) { + Cow::Borrowed(text) => { + if text.starts_with(BOM_CHAR) { + text.to_string() + } else { + // SAFETY: we know it's avalid utf-8 string at this point + unsafe { + let raw_ptr = Arc::into_raw(bytes); + return Arc::from_raw(std::mem::transmute::< + *const [u8], + *const str, + >(raw_ptr)); + } + } + } + Cow::Owned(string) => string, + }; + strip_bom_mut(&mut string); + string.into() + } + } + } + + pub fn into_arc_bytes(self) -> Arc<[u8]> { + match self { + Self::String(s) => s.into(), + Self::Bytes(b) => b, + } + } + + pub fn as_bytes(&self) -> &[u8] { + match self { + Self::String(s) => s.as_bytes(), + Self::Bytes(b) => b, + } + } + + pub fn len(&self) -> usize { + match self { + Self::String(s) => s.len(), + Self::Bytes(b) => b.len(), + } + } +} + +impl From for StringOrBytes { + fn from(s: String) -> Self { + Self::String(s.into()) + } +} + +impl From> for StringOrBytes { + fn from(s: Arc) -> Self { + Self::String(s) + } +} + +impl From<&'static str> for StringOrBytes { + fn from(s: &'static str) -> Self { + Self::String(s.into()) + } +} + +impl From> for StringOrBytes { + fn from(s: Arc<[u8]>) -> Self { + Self::Bytes(s) + } +} + /// Information that comes from an external source which can be optionally /// included in the module graph. #[derive(Debug, Default, Clone, Deserialize, Serialize)] @@ -61,7 +153,7 @@ pub enum LoadResponse { /// A loaded module. Module { /// The content of the remote module. - content: Arc, + content: Arc<[u8]>, /// The final specifier of the module. specifier: ModuleSpecifier, /// If the module is a remote module, the headers should be returned as a @@ -154,7 +246,7 @@ pub trait Loader { fn cache_module_info( &mut self, _specifier: &ModuleSpecifier, - _source: &str, + _source: &Arc<[u8]>, _module_info: &ModuleInfo, ) { } @@ -293,12 +385,10 @@ pub fn load_data_url( .map_err(|_| anyhow!("Unable to decode data url."))?; let mut headers: HashMap = HashMap::with_capacity(1); headers.insert("content-type".to_string(), url.mime_type().to_string()); - let mut content = String::from_utf8(bytes)?; - strip_bom_mut(&mut content); Ok(Some(LoadResponse::Module { specifier: specifier.clone(), maybe_headers: Some(headers), - content: content.into(), + content: Arc::from(bytes), })) } @@ -334,7 +424,7 @@ impl> Source { .map(|(k, v)| (k.as_ref().to_string(), v.as_ref().to_string())) .collect() }), - content: content.as_ref().into(), + content: Arc::from(content.as_ref().to_string().into_bytes()), }), Source::External(specifier) => Ok(LoadResponse::External { specifier: ModuleSpecifier::parse(specifier.as_ref()).unwrap(), diff --git a/src/symbols/analyzer.rs b/src/symbols/analyzer.rs index 704d768a6..c38c84cd4 100644 --- a/src/symbols/analyzer.rs +++ b/src/symbols/analyzer.rs @@ -17,7 +17,7 @@ use deno_ast::SourceTextInfo; use indexmap::IndexMap; use indexmap::IndexSet; -use crate::EsmModule; +use crate::EsModule; use crate::JsonModule; use crate::ModuleGraph; use crate::ModuleParser; @@ -83,7 +83,7 @@ impl<'a> RootSymbol<'a> { }; match graph_module { - crate::Module::Esm(esm_module) => self.analyze_esm_module(esm_module), + crate::Module::Esm(es_module) => self.analyze_es_module(es_module), crate::Module::Json(json_module) => { Some(self.analyze_json_module(json_module)) } @@ -151,14 +151,11 @@ impl<'a> RootSymbol<'a> { ) } - fn analyze_esm_module( - &self, - esm_module: &EsmModule, - ) -> Option { - let Ok(source) = self.parsed_source(esm_module) else { + fn analyze_es_module(&self, es_module: &EsModule) -> Option { + let Some(source) = self.parsed_source(es_module) else { return None; }; - let specifier = &esm_module.specifier; + let specifier = &es_module.specifier; let module = source.module(); let module_id = ModuleId(self.ids_to_modules.len() as u32); @@ -168,7 +165,7 @@ impl<'a> RootSymbol<'a> { builder: &builder, }; filler.fill(module); - let module_symbol = EsmModuleInfo { + let module_symbol = EsModuleInfo { specifier: specifier.clone(), module_id, source: source.clone(), @@ -188,7 +185,7 @@ impl<'a> RootSymbol<'a> { let specifier = &json_module.specifier; // it's not ideal having to use SourceTextInfo here, but it makes // it easier to interop with ParsedSource - let source_text_info = SourceTextInfo::new(json_module.source.clone()); + let source_text_info = SourceTextInfo::new(json_module.text.clone()); let range = source_text_info.range(); let module_id = ModuleId(self.ids_to_modules.len() as u32); let decls = { @@ -241,16 +238,16 @@ impl<'a> RootSymbol<'a> { self.ids_to_modules.get(&module_id).unwrap().as_ref() } - fn parsed_source( - &self, - graph_module: &EsmModule, - ) -> Result { - self.parser.parse_module(ParseOptions { - specifier: &graph_module.specifier, - source: graph_module.source.clone(), - media_type: graph_module.media_type, - scope_analysis: true, - }) + fn parsed_source(&self, graph_module: &EsModule) -> Option { + self + .parser + .parse_module(ParseOptions { + specifier: &graph_module.specifier, + source: graph_module.source.text.clone()?, + media_type: graph_module.media_type, + scope_analysis: true, + }) + .ok() } } @@ -1207,7 +1204,7 @@ impl UniqueSymbolId { #[derive(Debug, Clone, Copy)] pub enum ModuleInfoRef<'a> { Json(&'a JsonModuleInfo), - Esm(&'a EsmModuleInfo), + Esm(&'a EsModuleInfo), } impl<'a> ModuleInfoRef<'a> { @@ -1218,7 +1215,7 @@ impl<'a> ModuleInfoRef<'a> { } } - pub fn esm(&self) -> Option<&'a EsmModuleInfo> { + pub fn esm(&self) -> Option<&'a EsModuleInfo> { match self { Self::Json(_) => None, Self::Esm(esm) => Some(esm), @@ -1345,7 +1342,7 @@ impl<'a> ModuleInfoRef<'a> { #[derive(Debug, Clone)] pub enum ModuleInfo { Json(Box), - Esm(EsmModuleInfo), + Esm(EsModuleInfo), } impl ModuleInfo { @@ -1356,7 +1353,7 @@ impl ModuleInfo { } } - pub fn esm(&self) -> Option<&EsmModuleInfo> { + pub fn esm(&self) -> Option<&EsModuleInfo> { match self { Self::Json(_) => None, Self::Esm(esm) => Some(esm), @@ -1440,7 +1437,7 @@ impl JsonModuleInfo { } #[derive(Clone)] -pub struct EsmModuleInfo { +pub struct EsModuleInfo { module_id: ModuleId, specifier: ModuleSpecifier, source: ParsedSource, @@ -1451,9 +1448,9 @@ pub struct EsmModuleInfo { symbols: IndexMap, } -impl std::fmt::Debug for EsmModuleInfo { +impl std::fmt::Debug for EsModuleInfo { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("EsmModuleInfo") + f.debug_struct("EsModuleInfo") .field("module_id", &self.module_id) .field("specifier", &self.specifier.as_str()) .field( @@ -1470,7 +1467,7 @@ impl std::fmt::Debug for EsmModuleInfo { } } -impl EsmModuleInfo { +impl EsModuleInfo { pub fn as_ref(&self) -> ModuleInfoRef { ModuleInfoRef::Esm(self) } diff --git a/src/symbols/mod.rs b/src/symbols/mod.rs index 11b6d0332..fdd442e3a 100644 --- a/src/symbols/mod.rs +++ b/src/symbols/mod.rs @@ -1,6 +1,6 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -pub use self::analyzer::EsmModuleInfo; +pub use self::analyzer::EsModuleInfo; pub use self::analyzer::ExportDeclRef; pub use self::analyzer::FileDep; pub use self::analyzer::FileDepName; diff --git a/src/text_encoding.rs b/src/text_encoding.rs index 221a3d17a..b1433c6a6 100644 --- a/src/text_encoding.rs +++ b/src/text_encoding.rs @@ -1,5 +1,44 @@ +use std::borrow::Cow; + pub const BOM_CHAR: char = '\u{FEFF}'; +/// Attempts to detect the character encoding of the provided bytes. +/// +/// Supports UTF-8, UTF-16 Little Endian and UTF-16 Big Endian. +pub fn detect_charset(bytes: &'_ [u8]) -> &'static str { + const UTF16_LE_BOM: &[u8] = b"\xFF\xFE"; + const UTF16_BE_BOM: &[u8] = b"\xFE\xFF"; + + if bytes.starts_with(UTF16_LE_BOM) { + "utf-16le" + } else if bytes.starts_with(UTF16_BE_BOM) { + "utf-16be" + } else { + // Assume everything else is utf-8 + "utf-8" + } +} + +/// Attempts to convert the provided bytes to a UTF-8 string. +/// +/// Supports all encodings supported by the encoding_rs crate, which includes +/// all encodings specified in the WHATWG Encoding Standard, and only those +/// encodings (see: ). +pub fn convert_to_utf8<'a>( + bytes: &'a [u8], + charset: &'_ str, +) -> Result, std::io::Error> { + match encoding_rs::Encoding::for_label(charset.as_bytes()) { + Some(encoding) => encoding + .decode_without_bom_handling_and_without_replacement(bytes) + .ok_or_else(|| std::io::ErrorKind::InvalidData.into()), + None => Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!("Unsupported charset: {charset}"), + )), + } +} + /// Strips the byte order mark if it exists from the provided text. pub fn strip_bom_mut(text: &mut String) { if text.starts_with(BOM_CHAR) { @@ -11,6 +50,34 @@ pub fn strip_bom_mut(text: &mut String) { mod test { use super::*; + fn test_detection(test_data: &[u8], expected_charset: &str) { + let detected_charset = detect_charset(test_data); + assert_eq!( + expected_charset.to_lowercase(), + detected_charset.to_lowercase() + ); + } + + #[test] + fn test_detection_utf8_no_bom() { + let test_data = "Hello UTF-8 it is \u{23F0} for Deno!" + .to_owned() + .into_bytes(); + test_detection(&test_data, "utf-8"); + } + + #[test] + fn test_detection_utf16_little_endian() { + let test_data = b"\xFF\xFEHello UTF-16LE".to_owned().to_vec(); + test_detection(&test_data, "utf-16le"); + } + + #[test] + fn test_detection_utf16_big_endian() { + let test_data = b"\xFE\xFFHello UTF-16BE".to_owned().to_vec(); + test_detection(&test_data, "utf-16be"); + } + #[test] fn strip_bom_mut_with_bom() { let mut text = format!("{BOM_CHAR}text"); diff --git a/tests/integration_test.rs b/tests/integration_test.rs index 9804a9399..2c9ea1272 100644 --- a/tests/integration_test.rs +++ b/tests/integration_test.rs @@ -428,7 +428,7 @@ async fn test_jsr_version_not_found_then_found() { Ok(Some(LoadResponse::Module { specifier: specifier.clone(), maybe_headers: None, - content: "import 'jsr:@scope/a@1.2".into(), + content: b"import 'jsr:@scope/a@1.2".to_vec().into(), })) }), "https://jsr.io/@scope/a/meta.json" => { @@ -439,11 +439,11 @@ async fn test_jsr_version_not_found_then_found() { content: match cache_setting { CacheSetting::Only | CacheSetting::Use => { // first time it won't have the version - r#"{ "versions": { "1.0.0": {} } }"#.into() + br#"{ "versions": { "1.0.0": {} } }"#.to_vec().into() } CacheSetting::Reload => { // then on reload it will - r#"{ "versions": { "1.0.0": {}, "1.2.0": {} } }"#.into() + br#"{ "versions": { "1.0.0": {}, "1.2.0": {} } }"#.to_vec().into() } }, })) @@ -453,14 +453,14 @@ async fn test_jsr_version_not_found_then_found() { Ok(Some(LoadResponse::Module { specifier: specifier.clone(), maybe_headers: None, - content: r#"{ "exports": { ".": "./mod.ts" } }"#.into(), + content: br#"{ "exports": { ".": "./mod.ts" } }"#.to_vec().into(), })) }), "https://jsr.io/@scope/a/1.2.0/mod.ts" => Box::pin(async move { Ok(Some(LoadResponse::Module { specifier: specifier.clone(), maybe_headers: None, - content: "console.log('Hello, world!')".into(), + content: b"console.log('Hello, world!')".to_vec().into(), })) }), _ => unreachable!(), diff --git a/tests/specs/symbols/Basic.txt b/tests/specs/symbols/Basic.txt index 696cf3891..a5ceb096a 100644 --- a/tests/specs/symbols/Basic.txt +++ b/tests/specs/symbols/Basic.txt @@ -27,7 +27,7 @@ export default class C { export type D = typeof C; # output -file:///a.ts: EsmModuleInfo { +file:///a.ts: EsModuleInfo { module_id: ModuleId( 1, ), @@ -386,7 +386,7 @@ file:///a.ts: EsmModuleInfo { 6:102..115 [Id(("AInner", #2))] 9:173..198 [Id(("C", #2))] -file:///mod.ts: EsmModuleInfo { +file:///mod.ts: EsModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/ClassPrivateCtorParamProps.txt b/tests/specs/symbols/ClassPrivateCtorParamProps.txt index 018f51abf..482502cb8 100644 --- a/tests/specs/symbols/ClassPrivateCtorParamProps.txt +++ b/tests/specs/symbols/ClassPrivateCtorParamProps.txt @@ -12,7 +12,7 @@ class PublicClass {} class PrivateClass {} # output -file:///mod.ts: EsmModuleInfo { +file:///mod.ts: EsModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/Classes01.txt b/tests/specs/symbols/Classes01.txt index 736eeb22e..770d7275e 100644 --- a/tests/specs/symbols/Classes01.txt +++ b/tests/specs/symbols/Classes01.txt @@ -60,7 +60,7 @@ class ClassWithIndexSignatures { } # output -file:///mod.ts: EsmModuleInfo { +file:///mod.ts: EsModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/DeclarationMerging01.txt b/tests/specs/symbols/DeclarationMerging01.txt index 767dcdfbb..24283e0d1 100644 --- a/tests/specs/symbols/DeclarationMerging01.txt +++ b/tests/specs/symbols/DeclarationMerging01.txt @@ -25,7 +25,7 @@ namespace Test { export { Album, Color, Test }; # output -file:///mod.ts: EsmModuleInfo { +file:///mod.ts: EsModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/ExportAssignment01.txt b/tests/specs/symbols/ExportAssignment01.txt index 1c474b119..e22221214 100644 --- a/tests/specs/symbols/ExportAssignment01.txt +++ b/tests/specs/symbols/ExportAssignment01.txt @@ -6,7 +6,7 @@ function test() { export = test; # output -file:///mod.ts: EsmModuleInfo { +file:///mod.ts: EsModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/ExportDefault01.txt b/tests/specs/symbols/ExportDefault01.txt index 59d4e2c55..9271aa6bf 100644 --- a/tests/specs/symbols/ExportDefault01.txt +++ b/tests/specs/symbols/ExportDefault01.txt @@ -41,7 +41,7 @@ export default function Test() { } # output -file:///a.ts: EsmModuleInfo { +file:///a.ts: EsModuleInfo { module_id: ModuleId( 1, ), @@ -460,7 +460,7 @@ file:///a.ts: EsmModuleInfo { 5:172..180 [Id(("A", #2))] 8:213..218 [Id(("B", #2))] -file:///class.ts: EsmModuleInfo { +file:///class.ts: EsModuleInfo { module_id: ModuleId( 2, ), @@ -537,7 +537,7 @@ file:///class.ts: EsmModuleInfo { }, }, } -file:///function.ts: EsmModuleInfo { +file:///function.ts: EsModuleInfo { module_id: ModuleId( 3, ), @@ -614,7 +614,7 @@ file:///function.ts: EsmModuleInfo { }, }, } -file:///interface.ts: EsmModuleInfo { +file:///interface.ts: EsModuleInfo { module_id: ModuleId( 4, ), @@ -691,7 +691,7 @@ file:///interface.ts: EsmModuleInfo { }, }, } -file:///mod.ts: EsmModuleInfo { +file:///mod.ts: EsModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/ExportDefault02.txt b/tests/specs/symbols/ExportDefault02.txt index 62859384e..1b9b7bf95 100644 --- a/tests/specs/symbols/ExportDefault02.txt +++ b/tests/specs/symbols/ExportDefault02.txt @@ -2,7 +2,7 @@ export default Test; class Test {} # output -file:///mod.ts: EsmModuleInfo { +file:///mod.ts: EsModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/ExportDefault03.txt b/tests/specs/symbols/ExportDefault03.txt index 4c230e7d3..bb296a1e4 100644 --- a/tests/specs/symbols/ExportDefault03.txt +++ b/tests/specs/symbols/ExportDefault03.txt @@ -13,7 +13,7 @@ export class B {} export class B1 {} # output -file:///a.ts: EsmModuleInfo { +file:///a.ts: EsModuleInfo { module_id: ModuleId( 1, ), @@ -122,7 +122,7 @@ file:///a.ts: EsmModuleInfo { }, }, } -file:///b.ts: EsmModuleInfo { +file:///b.ts: EsModuleInfo { module_id: ModuleId( 2, ), @@ -235,7 +235,7 @@ file:///b.ts: EsmModuleInfo { }, }, } -file:///mod.ts: EsmModuleInfo { +file:///mod.ts: EsModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/ExportDefault04.txt b/tests/specs/symbols/ExportDefault04.txt index 839e8513a..e5e311db2 100644 --- a/tests/specs/symbols/ExportDefault04.txt +++ b/tests/specs/symbols/ExportDefault04.txt @@ -2,7 +2,7 @@ export default 1 + 1; # output -file:///mod.ts: EsmModuleInfo { +file:///mod.ts: EsModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/ExportDefaultLiteral01.txt b/tests/specs/symbols/ExportDefaultLiteral01.txt index be92a193f..d47f4b311 100644 --- a/tests/specs/symbols/ExportDefaultLiteral01.txt +++ b/tests/specs/symbols/ExportDefaultLiteral01.txt @@ -2,7 +2,7 @@ export default 1; # output -file:///mod.ts: EsmModuleInfo { +file:///mod.ts: EsModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/ExportDefaultLiteral02.txt b/tests/specs/symbols/ExportDefaultLiteral02.txt index 4ac747710..d5882993c 100644 --- a/tests/specs/symbols/ExportDefaultLiteral02.txt +++ b/tests/specs/symbols/ExportDefaultLiteral02.txt @@ -2,7 +2,7 @@ export default "test"; # output -file:///mod.ts: EsmModuleInfo { +file:///mod.ts: EsModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/ExportDestructured.txt b/tests/specs/symbols/ExportDestructured.txt index deb578357..ee6901dc8 100644 --- a/tests/specs/symbols/ExportDestructured.txt +++ b/tests/specs/symbols/ExportDestructured.txt @@ -15,7 +15,7 @@ export const { } = c; # output -file:///mod.ts: EsmModuleInfo { +file:///mod.ts: EsModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/ExportStar01.txt b/tests/specs/symbols/ExportStar01.txt index 4ae4af501..633a7e444 100644 --- a/tests/specs/symbols/ExportStar01.txt +++ b/tests/specs/symbols/ExportStar01.txt @@ -36,7 +36,7 @@ export default function Test() { } # output -file:///a.ts: EsmModuleInfo { +file:///a.ts: EsModuleInfo { module_id: ModuleId( 1, ), @@ -455,7 +455,7 @@ file:///a.ts: EsmModuleInfo { 5:172..180 [Id(("A", #2))] 8:213..218 [Id(("B", #2))] -file:///class.ts: EsmModuleInfo { +file:///class.ts: EsModuleInfo { module_id: ModuleId( 2, ), @@ -532,7 +532,7 @@ file:///class.ts: EsmModuleInfo { }, }, } -file:///function.ts: EsmModuleInfo { +file:///function.ts: EsModuleInfo { module_id: ModuleId( 3, ), @@ -609,7 +609,7 @@ file:///function.ts: EsmModuleInfo { }, }, } -file:///interface.ts: EsmModuleInfo { +file:///interface.ts: EsModuleInfo { module_id: ModuleId( 4, ), @@ -686,7 +686,7 @@ file:///interface.ts: EsmModuleInfo { }, }, } -file:///mod.ts: EsmModuleInfo { +file:///mod.ts: EsModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/ExportedDeclare01.txt b/tests/specs/symbols/ExportedDeclare01.txt index 5db2734f6..2c297f501 100644 --- a/tests/specs/symbols/ExportedDeclare01.txt +++ b/tests/specs/symbols/ExportedDeclare01.txt @@ -13,7 +13,7 @@ import inner = Test.test; export { foo as bar, inner as baz, MyClass }; # output -file:///mod.ts: EsmModuleInfo { +file:///mod.ts: EsModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/ExportedDeclare02.txt b/tests/specs/symbols/ExportedDeclare02.txt index c8318fe11..4c1ac5b27 100644 --- a/tests/specs/symbols/ExportedDeclare02.txt +++ b/tests/specs/symbols/ExportedDeclare02.txt @@ -3,7 +3,7 @@ declare function foo(): number; export function bar() {}; # output -file:///mod.ts: EsmModuleInfo { +file:///mod.ts: EsModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/FunctionOverloads01.txt b/tests/specs/symbols/FunctionOverloads01.txt index 4a12f94b5..0b616fddd 100644 --- a/tests/specs/symbols/FunctionOverloads01.txt +++ b/tests/specs/symbols/FunctionOverloads01.txt @@ -15,7 +15,7 @@ import InnerInner = Inner.inner; export { InnerInner }; # output -file:///mod.ts: EsmModuleInfo { +file:///mod.ts: EsModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/Functions01.txt b/tests/specs/symbols/Functions01.txt index cd6737f00..5815bf0bf 100644 --- a/tests/specs/symbols/Functions01.txt +++ b/tests/specs/symbols/Functions01.txt @@ -22,7 +22,7 @@ class PrivateParam {} class PrivateReturn {} # output -file:///mod.ts: EsmModuleInfo { +file:///mod.ts: EsModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/ImportEquals01.txt b/tests/specs/symbols/ImportEquals01.txt index 81cfe1809..683f422b0 100644 --- a/tests/specs/symbols/ImportEquals01.txt +++ b/tests/specs/symbols/ImportEquals01.txt @@ -7,7 +7,7 @@ namespace Test { } # output -file:///mod.ts: EsmModuleInfo { +file:///mod.ts: EsModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/ImportEquals02.txt b/tests/specs/symbols/ImportEquals02.txt index 4daeacf72..e4be505b4 100644 --- a/tests/specs/symbols/ImportEquals02.txt +++ b/tests/specs/symbols/ImportEquals02.txt @@ -25,7 +25,7 @@ namespace A.B { export { A }; # output -file:///a.ts: EsmModuleInfo { +file:///a.ts: EsModuleInfo { module_id: ModuleId( 1, ), @@ -342,7 +342,7 @@ file:///a.ts: EsmModuleInfo { }, }, } -file:///mod.ts: EsmModuleInfo { +file:///mod.ts: EsModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/ImportType01.txt b/tests/specs/symbols/ImportType01.txt index 996429605..813ca53b7 100644 --- a/tests/specs/symbols/ImportType01.txt +++ b/tests/specs/symbols/ImportType01.txt @@ -11,7 +11,7 @@ export namespace A { } # output -file:///a.ts: EsmModuleInfo { +file:///a.ts: EsModuleInfo { module_id: ModuleId( 1, ), @@ -158,7 +158,7 @@ file:///a.ts: EsmModuleInfo { }, }, } -file:///mod.ts: EsmModuleInfo { +file:///mod.ts: EsModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/ImportType02.txt b/tests/specs/symbols/ImportType02.txt index 3d0604c8c..38159effe 100644 --- a/tests/specs/symbols/ImportType02.txt +++ b/tests/specs/symbols/ImportType02.txt @@ -11,7 +11,7 @@ export default function test() { } # output -file:///a.ts: EsmModuleInfo { +file:///a.ts: EsModuleInfo { module_id: ModuleId( 1, ), @@ -156,7 +156,7 @@ file:///a.ts: EsmModuleInfo { }, }, } -file:///mod.ts: EsmModuleInfo { +file:///mod.ts: EsModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/Interfaces01.txt b/tests/specs/symbols/Interfaces01.txt index d74029846..c14e1f207 100644 --- a/tests/specs/symbols/Interfaces01.txt +++ b/tests/specs/symbols/Interfaces01.txt @@ -20,7 +20,7 @@ interface B { } # output -file:///mod.ts: EsmModuleInfo { +file:///mod.ts: EsModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/Interfaces02.txt b/tests/specs/symbols/Interfaces02.txt index 62e5c1e2a..8cea229a6 100644 --- a/tests/specs/symbols/Interfaces02.txt +++ b/tests/specs/symbols/Interfaces02.txt @@ -27,7 +27,7 @@ interface Prop2 {} export { A1 }; # output -file:///a.ts: EsmModuleInfo { +file:///a.ts: EsModuleInfo { module_id: ModuleId( 1, ), @@ -255,7 +255,7 @@ file:///a.ts: EsmModuleInfo { == symbol deps (types only) == 3:51..64 [Id(("Prop2", #2))] -file:///mod.ts: EsmModuleInfo { +file:///mod.ts: EsModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/MemberExpr.txt b/tests/specs/symbols/MemberExpr.txt index d9869ed3a..68bb63539 100644 --- a/tests/specs/symbols/MemberExpr.txt +++ b/tests/specs/symbols/MemberExpr.txt @@ -10,7 +10,7 @@ export class Test { } # output -file:///mod.ts: EsmModuleInfo { +file:///mod.ts: EsModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/ModuleExportNameStr01.txt b/tests/specs/symbols/ModuleExportNameStr01.txt index d86415307..cb49069f6 100644 --- a/tests/specs/symbols/ModuleExportNameStr01.txt +++ b/tests/specs/symbols/ModuleExportNameStr01.txt @@ -13,7 +13,7 @@ export { MyClass as "some-name" } from "./c.ts"; export class MyClass {} # output -file:///a.ts: EsmModuleInfo { +file:///a.ts: EsModuleInfo { module_id: ModuleId( 1, ), @@ -122,7 +122,7 @@ file:///a.ts: EsmModuleInfo { }, }, } -file:///b.ts: EsmModuleInfo { +file:///b.ts: EsModuleInfo { module_id: ModuleId( 2, ), @@ -195,7 +195,7 @@ file:///b.ts: EsmModuleInfo { }, }, } -file:///c.ts: EsmModuleInfo { +file:///c.ts: EsModuleInfo { module_id: ModuleId( 3, ), @@ -272,7 +272,7 @@ file:///c.ts: EsmModuleInfo { }, }, } -file:///mod.ts: EsmModuleInfo { +file:///mod.ts: EsModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/Namespaces01.txt b/tests/specs/symbols/Namespaces01.txt index 53c634754..af4b6ddcd 100644 --- a/tests/specs/symbols/Namespaces01.txt +++ b/tests/specs/symbols/Namespaces01.txt @@ -14,7 +14,7 @@ import test2 = Namespace1.Inner.test2; export { test, test2 }; # output -file:///mod.ts: EsmModuleInfo { +file:///mod.ts: EsModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/Namespaces02.txt b/tests/specs/symbols/Namespaces02.txt index bf7c73ee1..648fa061d 100644 --- a/tests/specs/symbols/Namespaces02.txt +++ b/tests/specs/symbols/Namespaces02.txt @@ -10,7 +10,7 @@ export namespace RootNs { } # output -file:///mod.ts: EsmModuleInfo { +file:///mod.ts: EsModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/NamespacesAmbient.txt b/tests/specs/symbols/NamespacesAmbient.txt index 24775738d..302dbdd3c 100644 --- a/tests/specs/symbols/NamespacesAmbient.txt +++ b/tests/specs/symbols/NamespacesAmbient.txt @@ -47,7 +47,7 @@ namespace Inner { } # output -file:///mod.ts: EsmModuleInfo { +file:///mod.ts: EsModuleInfo { module_id: ModuleId( 0, ), @@ -805,7 +805,7 @@ file:///mod.ts: EsmModuleInfo { }, }, } -file:///other.d.ts: EsmModuleInfo { +file:///other.d.ts: EsModuleInfo { module_id: ModuleId( 1, ), diff --git a/tests/specs/symbols/ReExportJson01.txt b/tests/specs/symbols/ReExportJson01.txt index 813a6f369..7f32cde05 100644 --- a/tests/specs/symbols/ReExportJson01.txt +++ b/tests/specs/symbols/ReExportJson01.txt @@ -51,7 +51,7 @@ file:///bar.json: JsonModuleInfo { members: {}, }, } -file:///mod.ts: EsmModuleInfo { +file:///mod.ts: EsModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/ReexportAll.txt b/tests/specs/symbols/ReexportAll.txt index 32f821368..fa082b3cc 100644 --- a/tests/specs/symbols/ReexportAll.txt +++ b/tests/specs/symbols/ReexportAll.txt @@ -18,7 +18,7 @@ export { Test } from "./b.ts"; export { Test, test1 } from "./c.ts"; # output -file:///a.ts: EsmModuleInfo { +file:///a.ts: EsModuleInfo { module_id: ModuleId( 1, ), @@ -394,7 +394,7 @@ file:///a.ts: EsmModuleInfo { }, }, } -file:///b.ts: EsmModuleInfo { +file:///b.ts: EsModuleInfo { module_id: ModuleId( 2, ), @@ -471,7 +471,7 @@ file:///b.ts: EsmModuleInfo { }, }, } -file:///c.ts: EsmModuleInfo { +file:///c.ts: EsModuleInfo { module_id: ModuleId( 3, ), @@ -546,7 +546,7 @@ file:///c.ts: EsmModuleInfo { }, }, } -file:///mod.ts: EsmModuleInfo { +file:///mod.ts: EsModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/ReexportDeep.txt b/tests/specs/symbols/ReexportDeep.txt index 16801bbe6..57633db07 100644 --- a/tests/specs/symbols/ReexportDeep.txt +++ b/tests/specs/symbols/ReexportDeep.txt @@ -16,7 +16,7 @@ export declare namespace MessageEntity { } # output -file:///a.ts: EsmModuleInfo { +file:///a.ts: EsModuleInfo { module_id: ModuleId( 1, ), @@ -56,7 +56,7 @@ file:///a.ts: EsmModuleInfo { }, }, } -file:///b.ts: EsmModuleInfo { +file:///b.ts: EsModuleInfo { module_id: ModuleId( 2, ), @@ -96,7 +96,7 @@ file:///b.ts: EsmModuleInfo { }, }, } -file:///c.ts: EsmModuleInfo { +file:///c.ts: EsModuleInfo { module_id: ModuleId( 3, ), @@ -243,7 +243,7 @@ file:///c.ts: EsmModuleInfo { }, }, } -file:///mod.ts: EsmModuleInfo { +file:///mod.ts: EsModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/ReexportExport.txt b/tests/specs/symbols/ReexportExport.txt index 9c8d1e6ce..9b7d0aebc 100644 --- a/tests/specs/symbols/ReexportExport.txt +++ b/tests/specs/symbols/ReexportExport.txt @@ -3,7 +3,7 @@ export function foo(): void {} export { foo as bar }; # output -file:///mod.ts: EsmModuleInfo { +file:///mod.ts: EsModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/TsExternalModuleRef01.txt b/tests/specs/symbols/TsExternalModuleRef01.txt index 97709f362..69f1fd377 100644 --- a/tests/specs/symbols/TsExternalModuleRef01.txt +++ b/tests/specs/symbols/TsExternalModuleRef01.txt @@ -11,7 +11,7 @@ namespace MyNamespace { export = MyNamespace; # output -file:///a.ts: EsmModuleInfo { +file:///a.ts: EsModuleInfo { module_id: ModuleId( 1, ), @@ -157,7 +157,7 @@ file:///a.ts: EsmModuleInfo { }, }, } -file:///mod.ts: EsmModuleInfo { +file:///mod.ts: EsModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/TsExternalModuleRef02.txt b/tests/specs/symbols/TsExternalModuleRef02.txt index a21984b34..f8a14e8d9 100644 --- a/tests/specs/symbols/TsExternalModuleRef02.txt +++ b/tests/specs/symbols/TsExternalModuleRef02.txt @@ -11,7 +11,7 @@ namespace MyNamespace { export = MyNamespace; # output -file:///a.ts: EsmModuleInfo { +file:///a.ts: EsModuleInfo { module_id: ModuleId( 1, ), @@ -157,7 +157,7 @@ file:///a.ts: EsmModuleInfo { }, }, } -file:///mod.ts: EsmModuleInfo { +file:///mod.ts: EsModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/TsNamespaceExport01.txt b/tests/specs/symbols/TsNamespaceExport01.txt index b25924770..c5109ed77 100644 --- a/tests/specs/symbols/TsNamespaceExport01.txt +++ b/tests/specs/symbols/TsNamespaceExport01.txt @@ -8,7 +8,7 @@ export function isPrime(x: number): boolean { export as namespace mathLib; # output -file:///mod.ts: EsmModuleInfo { +file:///mod.ts: EsModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/TsQualifiedName01.txt b/tests/specs/symbols/TsQualifiedName01.txt index d5b41714d..9a1cdee58 100644 --- a/tests/specs/symbols/TsQualifiedName01.txt +++ b/tests/specs/symbols/TsQualifiedName01.txt @@ -8,7 +8,7 @@ namespace Other { } # output -file:///mod.ts: EsmModuleInfo { +file:///mod.ts: EsModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/TypeAlias01.txt b/tests/specs/symbols/TypeAlias01.txt index 55c1034f4..e48f78d5c 100644 --- a/tests/specs/symbols/TypeAlias01.txt +++ b/tests/specs/symbols/TypeAlias01.txt @@ -8,7 +8,7 @@ interface Other2 {} export type Tuple = [lat: number, long: number]; # output -file:///mod.ts: EsmModuleInfo { +file:///mod.ts: EsModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/TypeScriptTypesHeader.txt b/tests/specs/symbols/TypeScriptTypesHeader.txt index 153ce138f..307b139fe 100644 --- a/tests/specs/symbols/TypeScriptTypesHeader.txt +++ b/tests/specs/symbols/TypeScriptTypesHeader.txt @@ -33,7 +33,7 @@ export interface MyInterface3 { } # output -file:///mod.ts: EsmModuleInfo { +file:///mod.ts: EsModuleInfo { module_id: ModuleId( 0, ), @@ -227,7 +227,7 @@ file:///mod.ts: EsmModuleInfo { == symbol deps (types only) == 4:148..223 [Id(("MyInterface", #2)), Id(("MyInterface2", #2)), Id(("MyInterface3", #2))] -file:///other.d.ts: EsmModuleInfo { +file:///other.d.ts: EsModuleInfo { module_id: ModuleId( 1, ), @@ -336,7 +336,7 @@ file:///other.d.ts: EsmModuleInfo { }, }, } -file:///other.js: EsmModuleInfo { +file:///other.js: EsModuleInfo { module_id: ModuleId( 2, ), @@ -374,7 +374,7 @@ file:///other.js: EsmModuleInfo { }, }, } -file:///typescript.ts: EsmModuleInfo { +file:///typescript.ts: EsModuleInfo { module_id: ModuleId( 3, ), @@ -483,7 +483,7 @@ file:///typescript.ts: EsmModuleInfo { }, }, } -https://localhost/mod.d.ts: EsmModuleInfo { +https://localhost/mod.d.ts: EsModuleInfo { module_id: ModuleId( 4, ), @@ -660,7 +660,7 @@ https://localhost/mod.d.ts: EsmModuleInfo { }, }, } -https://localhost/mod.js: EsmModuleInfo { +https://localhost/mod.js: EsModuleInfo { module_id: ModuleId( 5, ), diff --git a/tests/specs/symbols/TypesEntrypoint.txt b/tests/specs/symbols/TypesEntrypoint.txt index 8b50452ec..a18cd20ca 100644 --- a/tests/specs/symbols/TypesEntrypoint.txt +++ b/tests/specs/symbols/TypesEntrypoint.txt @@ -11,7 +11,7 @@ export interface MyInterface { } # output -file:///mod.d.ts: EsmModuleInfo { +file:///mod.d.ts: EsModuleInfo { module_id: ModuleId( 0, ), @@ -120,7 +120,7 @@ file:///mod.d.ts: EsmModuleInfo { }, }, } -file:///mod.js: EsmModuleInfo { +file:///mod.js: EsModuleInfo { module_id: ModuleId( 1, ), diff --git a/tests/specs/symbols/UnresolvedParts.txt b/tests/specs/symbols/UnresolvedParts.txt index 3f5807c3b..a5af6d98d 100644 --- a/tests/specs/symbols/UnresolvedParts.txt +++ b/tests/specs/symbols/UnresolvedParts.txt @@ -16,7 +16,7 @@ export import NonExistent2 = other.NotFound; export class Other {} # output -file:///mod.ts: EsmModuleInfo { +file:///mod.ts: EsModuleInfo { module_id: ModuleId( 0, ), @@ -316,7 +316,7 @@ file:///mod.ts: EsmModuleInfo { }, }, } -file:///other.ts: EsmModuleInfo { +file:///other.ts: EsModuleInfo { module_id: ModuleId( 1, ), diff --git a/tests/specs/symbols/UsingDecl.txt b/tests/specs/symbols/UsingDecl.txt index cb62f9084..3540e9441 100644 --- a/tests/specs/symbols/UsingDecl.txt +++ b/tests/specs/symbols/UsingDecl.txt @@ -5,7 +5,7 @@ using test = 5; export { test }; # output -file:///mod.ts: EsmModuleInfo { +file:///mod.ts: EsModuleInfo { module_id: ModuleId( 0, ), From 8f40c0f16052961c67743e50f48d2af9efe4187b Mon Sep 17 00:00:00 2001 From: David Sherret Date: Fri, 12 Jan 2024 14:57:16 -0500 Subject: [PATCH 02/14] Remove old StringOrBytes idea --- src/source/mod.rs | 93 ----------------------------------------- src/symbols/analyzer.rs | 1 - 2 files changed, 94 deletions(-) diff --git a/src/source/mod.rs b/src/source/mod.rs index 361f8ea80..4456b8694 100644 --- a/src/source/mod.rs +++ b/src/source/mod.rs @@ -4,8 +4,6 @@ use crate::graph::Range; use crate::module_specifier::resolve_import; use crate::packages::JsrPackageInfo; use crate::packages::JsrPackageVersionInfo; -use crate::text_encoding::strip_bom_mut; -use crate::text_encoding::BOM_CHAR; use crate::ModuleInfo; use crate::SpecifierError; use deno_semver::package::PackageNv; @@ -20,7 +18,6 @@ use futures::future::LocalBoxFuture; use once_cell::sync::Lazy; use serde::Deserialize; use serde::Serialize; -use std::borrow::Cow; use std::collections::HashMap; use std::fmt; use std::path::PathBuf; @@ -33,96 +30,6 @@ pub use file_system::*; pub const DEFAULT_JSX_IMPORT_SOURCE_MODULE: &str = "jsx-runtime"; -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub enum StringOrBytes { - String(Arc), - Bytes(Arc<[u8]>), -} - -impl StringOrBytes { - pub fn from_vec(bytes: Vec) -> StringOrBytes { - StringOrBytes::Bytes(Arc::from(bytes.into_boxed_slice())) - } - - pub fn into_arc_string(self) -> Arc { - match self { - Self::String(s) => s, - // note, this should never happen in practice - Self::Bytes(bytes) => { - // Parse some bytes into a UTF-8 string as the WHATWG encoding spec specifies - // in https://encoding.spec.whatwg.org/#utf-8-decode. - // - // This is used by the HTML spec to parse the bytes representing ES modules - // into strings. - let mut string = match String::from_utf8_lossy(&bytes) { - Cow::Borrowed(text) => { - if text.starts_with(BOM_CHAR) { - text.to_string() - } else { - // SAFETY: we know it's avalid utf-8 string at this point - unsafe { - let raw_ptr = Arc::into_raw(bytes); - return Arc::from_raw(std::mem::transmute::< - *const [u8], - *const str, - >(raw_ptr)); - } - } - } - Cow::Owned(string) => string, - }; - strip_bom_mut(&mut string); - string.into() - } - } - } - - pub fn into_arc_bytes(self) -> Arc<[u8]> { - match self { - Self::String(s) => s.into(), - Self::Bytes(b) => b, - } - } - - pub fn as_bytes(&self) -> &[u8] { - match self { - Self::String(s) => s.as_bytes(), - Self::Bytes(b) => b, - } - } - - pub fn len(&self) -> usize { - match self { - Self::String(s) => s.len(), - Self::Bytes(b) => b.len(), - } - } -} - -impl From for StringOrBytes { - fn from(s: String) -> Self { - Self::String(s.into()) - } -} - -impl From> for StringOrBytes { - fn from(s: Arc) -> Self { - Self::String(s) - } -} - -impl From<&'static str> for StringOrBytes { - fn from(s: &'static str) -> Self { - Self::String(s.into()) - } -} - -impl From> for StringOrBytes { - fn from(s: Arc<[u8]>) -> Self { - Self::Bytes(s) - } -} - /// Information that comes from an external source which can be optionally /// included in the module graph. #[derive(Debug, Default, Clone, Deserialize, Serialize)] diff --git a/src/symbols/analyzer.rs b/src/symbols/analyzer.rs index c38c84cd4..3b034b394 100644 --- a/src/symbols/analyzer.rs +++ b/src/symbols/analyzer.rs @@ -6,7 +6,6 @@ use std::cell::Ref; use std::cell::RefCell; use std::hash::Hash; -use anyhow::Result; use deno_ast::swc::ast::*; use deno_ast::swc::utils::find_pat_ids; use deno_ast::ModuleSpecifier; From e514cb43d7f1965610b4131f6799a16f859e54ed Mon Sep 17 00:00:00 2001 From: David Sherret Date: Fri, 12 Jan 2024 15:07:34 -0500 Subject: [PATCH 03/14] Update --- src/symbols/analyzer.rs | 3 ++- tests/specs/symbols/Basic.txt | 4 ++-- tests/specs/symbols/ClassPrivateCtorParamProps.txt | 2 +- tests/specs/symbols/Classes01.txt | 2 +- tests/specs/symbols/DeclarationMerging01.txt | 2 +- tests/specs/symbols/ExportAssignment01.txt | 2 +- tests/specs/symbols/ExportDefault01.txt | 10 +++++----- tests/specs/symbols/ExportDefault02.txt | 2 +- tests/specs/symbols/ExportDefault03.txt | 6 +++--- tests/specs/symbols/ExportDefault04.txt | 2 +- tests/specs/symbols/ExportDefaultLiteral01.txt | 2 +- tests/specs/symbols/ExportDefaultLiteral02.txt | 2 +- tests/specs/symbols/ExportDestructured.txt | 2 +- tests/specs/symbols/ExportStar01.txt | 10 +++++----- tests/specs/symbols/ExportedDeclare01.txt | 2 +- tests/specs/symbols/ExportedDeclare02.txt | 2 +- tests/specs/symbols/FunctionOverloads01.txt | 2 +- tests/specs/symbols/Functions01.txt | 2 +- tests/specs/symbols/ImportEquals01.txt | 2 +- tests/specs/symbols/ImportEquals02.txt | 4 ++-- tests/specs/symbols/ImportType01.txt | 4 ++-- tests/specs/symbols/ImportType02.txt | 4 ++-- tests/specs/symbols/Interfaces01.txt | 2 +- tests/specs/symbols/Interfaces02.txt | 4 ++-- tests/specs/symbols/MemberExpr.txt | 2 +- tests/specs/symbols/ModuleExportNameStr01.txt | 8 ++++---- tests/specs/symbols/Namespaces01.txt | 2 +- tests/specs/symbols/Namespaces02.txt | 2 +- tests/specs/symbols/NamespacesAmbient.txt | 4 ++-- tests/specs/symbols/ReExportJson01.txt | 2 +- tests/specs/symbols/ReexportAll.txt | 8 ++++---- tests/specs/symbols/ReexportDeep.txt | 8 ++++---- tests/specs/symbols/ReexportExport.txt | 2 +- tests/specs/symbols/TsExternalModuleRef01.txt | 4 ++-- tests/specs/symbols/TsExternalModuleRef02.txt | 4 ++-- tests/specs/symbols/TsNamespaceExport01.txt | 2 +- tests/specs/symbols/TsQualifiedName01.txt | 2 +- tests/specs/symbols/TypeAlias01.txt | 2 +- tests/specs/symbols/TypeScriptTypesHeader.txt | 12 ++++++------ tests/specs/symbols/TypesEntrypoint.txt | 4 ++-- tests/specs/symbols/UnresolvedParts.txt | 4 ++-- tests/specs/symbols/UsingDecl.txt | 2 +- 42 files changed, 77 insertions(+), 76 deletions(-) diff --git a/src/symbols/analyzer.rs b/src/symbols/analyzer.rs index 3b034b394..1c4a0c872 100644 --- a/src/symbols/analyzer.rs +++ b/src/symbols/analyzer.rs @@ -1449,7 +1449,8 @@ pub struct EsModuleInfo { impl std::fmt::Debug for EsModuleInfo { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("EsModuleInfo") + // todo(dsherret): rename in a separate PR in order to reduce noise + f.debug_struct("EsmModuleInfo") .field("module_id", &self.module_id) .field("specifier", &self.specifier.as_str()) .field( diff --git a/tests/specs/symbols/Basic.txt b/tests/specs/symbols/Basic.txt index a5ceb096a..696cf3891 100644 --- a/tests/specs/symbols/Basic.txt +++ b/tests/specs/symbols/Basic.txt @@ -27,7 +27,7 @@ export default class C { export type D = typeof C; # output -file:///a.ts: EsModuleInfo { +file:///a.ts: EsmModuleInfo { module_id: ModuleId( 1, ), @@ -386,7 +386,7 @@ file:///a.ts: EsModuleInfo { 6:102..115 [Id(("AInner", #2))] 9:173..198 [Id(("C", #2))] -file:///mod.ts: EsModuleInfo { +file:///mod.ts: EsmModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/ClassPrivateCtorParamProps.txt b/tests/specs/symbols/ClassPrivateCtorParamProps.txt index 482502cb8..018f51abf 100644 --- a/tests/specs/symbols/ClassPrivateCtorParamProps.txt +++ b/tests/specs/symbols/ClassPrivateCtorParamProps.txt @@ -12,7 +12,7 @@ class PublicClass {} class PrivateClass {} # output -file:///mod.ts: EsModuleInfo { +file:///mod.ts: EsmModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/Classes01.txt b/tests/specs/symbols/Classes01.txt index 770d7275e..736eeb22e 100644 --- a/tests/specs/symbols/Classes01.txt +++ b/tests/specs/symbols/Classes01.txt @@ -60,7 +60,7 @@ class ClassWithIndexSignatures { } # output -file:///mod.ts: EsModuleInfo { +file:///mod.ts: EsmModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/DeclarationMerging01.txt b/tests/specs/symbols/DeclarationMerging01.txt index 24283e0d1..767dcdfbb 100644 --- a/tests/specs/symbols/DeclarationMerging01.txt +++ b/tests/specs/symbols/DeclarationMerging01.txt @@ -25,7 +25,7 @@ namespace Test { export { Album, Color, Test }; # output -file:///mod.ts: EsModuleInfo { +file:///mod.ts: EsmModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/ExportAssignment01.txt b/tests/specs/symbols/ExportAssignment01.txt index e22221214..1c474b119 100644 --- a/tests/specs/symbols/ExportAssignment01.txt +++ b/tests/specs/symbols/ExportAssignment01.txt @@ -6,7 +6,7 @@ function test() { export = test; # output -file:///mod.ts: EsModuleInfo { +file:///mod.ts: EsmModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/ExportDefault01.txt b/tests/specs/symbols/ExportDefault01.txt index 9271aa6bf..59d4e2c55 100644 --- a/tests/specs/symbols/ExportDefault01.txt +++ b/tests/specs/symbols/ExportDefault01.txt @@ -41,7 +41,7 @@ export default function Test() { } # output -file:///a.ts: EsModuleInfo { +file:///a.ts: EsmModuleInfo { module_id: ModuleId( 1, ), @@ -460,7 +460,7 @@ file:///a.ts: EsModuleInfo { 5:172..180 [Id(("A", #2))] 8:213..218 [Id(("B", #2))] -file:///class.ts: EsModuleInfo { +file:///class.ts: EsmModuleInfo { module_id: ModuleId( 2, ), @@ -537,7 +537,7 @@ file:///class.ts: EsModuleInfo { }, }, } -file:///function.ts: EsModuleInfo { +file:///function.ts: EsmModuleInfo { module_id: ModuleId( 3, ), @@ -614,7 +614,7 @@ file:///function.ts: EsModuleInfo { }, }, } -file:///interface.ts: EsModuleInfo { +file:///interface.ts: EsmModuleInfo { module_id: ModuleId( 4, ), @@ -691,7 +691,7 @@ file:///interface.ts: EsModuleInfo { }, }, } -file:///mod.ts: EsModuleInfo { +file:///mod.ts: EsmModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/ExportDefault02.txt b/tests/specs/symbols/ExportDefault02.txt index 1b9b7bf95..62859384e 100644 --- a/tests/specs/symbols/ExportDefault02.txt +++ b/tests/specs/symbols/ExportDefault02.txt @@ -2,7 +2,7 @@ export default Test; class Test {} # output -file:///mod.ts: EsModuleInfo { +file:///mod.ts: EsmModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/ExportDefault03.txt b/tests/specs/symbols/ExportDefault03.txt index bb296a1e4..4c230e7d3 100644 --- a/tests/specs/symbols/ExportDefault03.txt +++ b/tests/specs/symbols/ExportDefault03.txt @@ -13,7 +13,7 @@ export class B {} export class B1 {} # output -file:///a.ts: EsModuleInfo { +file:///a.ts: EsmModuleInfo { module_id: ModuleId( 1, ), @@ -122,7 +122,7 @@ file:///a.ts: EsModuleInfo { }, }, } -file:///b.ts: EsModuleInfo { +file:///b.ts: EsmModuleInfo { module_id: ModuleId( 2, ), @@ -235,7 +235,7 @@ file:///b.ts: EsModuleInfo { }, }, } -file:///mod.ts: EsModuleInfo { +file:///mod.ts: EsmModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/ExportDefault04.txt b/tests/specs/symbols/ExportDefault04.txt index e5e311db2..839e8513a 100644 --- a/tests/specs/symbols/ExportDefault04.txt +++ b/tests/specs/symbols/ExportDefault04.txt @@ -2,7 +2,7 @@ export default 1 + 1; # output -file:///mod.ts: EsModuleInfo { +file:///mod.ts: EsmModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/ExportDefaultLiteral01.txt b/tests/specs/symbols/ExportDefaultLiteral01.txt index d47f4b311..be92a193f 100644 --- a/tests/specs/symbols/ExportDefaultLiteral01.txt +++ b/tests/specs/symbols/ExportDefaultLiteral01.txt @@ -2,7 +2,7 @@ export default 1; # output -file:///mod.ts: EsModuleInfo { +file:///mod.ts: EsmModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/ExportDefaultLiteral02.txt b/tests/specs/symbols/ExportDefaultLiteral02.txt index d5882993c..4ac747710 100644 --- a/tests/specs/symbols/ExportDefaultLiteral02.txt +++ b/tests/specs/symbols/ExportDefaultLiteral02.txt @@ -2,7 +2,7 @@ export default "test"; # output -file:///mod.ts: EsModuleInfo { +file:///mod.ts: EsmModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/ExportDestructured.txt b/tests/specs/symbols/ExportDestructured.txt index ee6901dc8..deb578357 100644 --- a/tests/specs/symbols/ExportDestructured.txt +++ b/tests/specs/symbols/ExportDestructured.txt @@ -15,7 +15,7 @@ export const { } = c; # output -file:///mod.ts: EsModuleInfo { +file:///mod.ts: EsmModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/ExportStar01.txt b/tests/specs/symbols/ExportStar01.txt index 633a7e444..4ae4af501 100644 --- a/tests/specs/symbols/ExportStar01.txt +++ b/tests/specs/symbols/ExportStar01.txt @@ -36,7 +36,7 @@ export default function Test() { } # output -file:///a.ts: EsModuleInfo { +file:///a.ts: EsmModuleInfo { module_id: ModuleId( 1, ), @@ -455,7 +455,7 @@ file:///a.ts: EsModuleInfo { 5:172..180 [Id(("A", #2))] 8:213..218 [Id(("B", #2))] -file:///class.ts: EsModuleInfo { +file:///class.ts: EsmModuleInfo { module_id: ModuleId( 2, ), @@ -532,7 +532,7 @@ file:///class.ts: EsModuleInfo { }, }, } -file:///function.ts: EsModuleInfo { +file:///function.ts: EsmModuleInfo { module_id: ModuleId( 3, ), @@ -609,7 +609,7 @@ file:///function.ts: EsModuleInfo { }, }, } -file:///interface.ts: EsModuleInfo { +file:///interface.ts: EsmModuleInfo { module_id: ModuleId( 4, ), @@ -686,7 +686,7 @@ file:///interface.ts: EsModuleInfo { }, }, } -file:///mod.ts: EsModuleInfo { +file:///mod.ts: EsmModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/ExportedDeclare01.txt b/tests/specs/symbols/ExportedDeclare01.txt index 2c297f501..5db2734f6 100644 --- a/tests/specs/symbols/ExportedDeclare01.txt +++ b/tests/specs/symbols/ExportedDeclare01.txt @@ -13,7 +13,7 @@ import inner = Test.test; export { foo as bar, inner as baz, MyClass }; # output -file:///mod.ts: EsModuleInfo { +file:///mod.ts: EsmModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/ExportedDeclare02.txt b/tests/specs/symbols/ExportedDeclare02.txt index 4c1ac5b27..c8318fe11 100644 --- a/tests/specs/symbols/ExportedDeclare02.txt +++ b/tests/specs/symbols/ExportedDeclare02.txt @@ -3,7 +3,7 @@ declare function foo(): number; export function bar() {}; # output -file:///mod.ts: EsModuleInfo { +file:///mod.ts: EsmModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/FunctionOverloads01.txt b/tests/specs/symbols/FunctionOverloads01.txt index 0b616fddd..4a12f94b5 100644 --- a/tests/specs/symbols/FunctionOverloads01.txt +++ b/tests/specs/symbols/FunctionOverloads01.txt @@ -15,7 +15,7 @@ import InnerInner = Inner.inner; export { InnerInner }; # output -file:///mod.ts: EsModuleInfo { +file:///mod.ts: EsmModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/Functions01.txt b/tests/specs/symbols/Functions01.txt index 5815bf0bf..cd6737f00 100644 --- a/tests/specs/symbols/Functions01.txt +++ b/tests/specs/symbols/Functions01.txt @@ -22,7 +22,7 @@ class PrivateParam {} class PrivateReturn {} # output -file:///mod.ts: EsModuleInfo { +file:///mod.ts: EsmModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/ImportEquals01.txt b/tests/specs/symbols/ImportEquals01.txt index 683f422b0..81cfe1809 100644 --- a/tests/specs/symbols/ImportEquals01.txt +++ b/tests/specs/symbols/ImportEquals01.txt @@ -7,7 +7,7 @@ namespace Test { } # output -file:///mod.ts: EsModuleInfo { +file:///mod.ts: EsmModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/ImportEquals02.txt b/tests/specs/symbols/ImportEquals02.txt index e4be505b4..4daeacf72 100644 --- a/tests/specs/symbols/ImportEquals02.txt +++ b/tests/specs/symbols/ImportEquals02.txt @@ -25,7 +25,7 @@ namespace A.B { export { A }; # output -file:///a.ts: EsModuleInfo { +file:///a.ts: EsmModuleInfo { module_id: ModuleId( 1, ), @@ -342,7 +342,7 @@ file:///a.ts: EsModuleInfo { }, }, } -file:///mod.ts: EsModuleInfo { +file:///mod.ts: EsmModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/ImportType01.txt b/tests/specs/symbols/ImportType01.txt index 813ca53b7..996429605 100644 --- a/tests/specs/symbols/ImportType01.txt +++ b/tests/specs/symbols/ImportType01.txt @@ -11,7 +11,7 @@ export namespace A { } # output -file:///a.ts: EsModuleInfo { +file:///a.ts: EsmModuleInfo { module_id: ModuleId( 1, ), @@ -158,7 +158,7 @@ file:///a.ts: EsModuleInfo { }, }, } -file:///mod.ts: EsModuleInfo { +file:///mod.ts: EsmModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/ImportType02.txt b/tests/specs/symbols/ImportType02.txt index 38159effe..3d0604c8c 100644 --- a/tests/specs/symbols/ImportType02.txt +++ b/tests/specs/symbols/ImportType02.txt @@ -11,7 +11,7 @@ export default function test() { } # output -file:///a.ts: EsModuleInfo { +file:///a.ts: EsmModuleInfo { module_id: ModuleId( 1, ), @@ -156,7 +156,7 @@ file:///a.ts: EsModuleInfo { }, }, } -file:///mod.ts: EsModuleInfo { +file:///mod.ts: EsmModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/Interfaces01.txt b/tests/specs/symbols/Interfaces01.txt index c14e1f207..d74029846 100644 --- a/tests/specs/symbols/Interfaces01.txt +++ b/tests/specs/symbols/Interfaces01.txt @@ -20,7 +20,7 @@ interface B { } # output -file:///mod.ts: EsModuleInfo { +file:///mod.ts: EsmModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/Interfaces02.txt b/tests/specs/symbols/Interfaces02.txt index 8cea229a6..62e5c1e2a 100644 --- a/tests/specs/symbols/Interfaces02.txt +++ b/tests/specs/symbols/Interfaces02.txt @@ -27,7 +27,7 @@ interface Prop2 {} export { A1 }; # output -file:///a.ts: EsModuleInfo { +file:///a.ts: EsmModuleInfo { module_id: ModuleId( 1, ), @@ -255,7 +255,7 @@ file:///a.ts: EsModuleInfo { == symbol deps (types only) == 3:51..64 [Id(("Prop2", #2))] -file:///mod.ts: EsModuleInfo { +file:///mod.ts: EsmModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/MemberExpr.txt b/tests/specs/symbols/MemberExpr.txt index 68bb63539..d9869ed3a 100644 --- a/tests/specs/symbols/MemberExpr.txt +++ b/tests/specs/symbols/MemberExpr.txt @@ -10,7 +10,7 @@ export class Test { } # output -file:///mod.ts: EsModuleInfo { +file:///mod.ts: EsmModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/ModuleExportNameStr01.txt b/tests/specs/symbols/ModuleExportNameStr01.txt index cb49069f6..d86415307 100644 --- a/tests/specs/symbols/ModuleExportNameStr01.txt +++ b/tests/specs/symbols/ModuleExportNameStr01.txt @@ -13,7 +13,7 @@ export { MyClass as "some-name" } from "./c.ts"; export class MyClass {} # output -file:///a.ts: EsModuleInfo { +file:///a.ts: EsmModuleInfo { module_id: ModuleId( 1, ), @@ -122,7 +122,7 @@ file:///a.ts: EsModuleInfo { }, }, } -file:///b.ts: EsModuleInfo { +file:///b.ts: EsmModuleInfo { module_id: ModuleId( 2, ), @@ -195,7 +195,7 @@ file:///b.ts: EsModuleInfo { }, }, } -file:///c.ts: EsModuleInfo { +file:///c.ts: EsmModuleInfo { module_id: ModuleId( 3, ), @@ -272,7 +272,7 @@ file:///c.ts: EsModuleInfo { }, }, } -file:///mod.ts: EsModuleInfo { +file:///mod.ts: EsmModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/Namespaces01.txt b/tests/specs/symbols/Namespaces01.txt index af4b6ddcd..53c634754 100644 --- a/tests/specs/symbols/Namespaces01.txt +++ b/tests/specs/symbols/Namespaces01.txt @@ -14,7 +14,7 @@ import test2 = Namespace1.Inner.test2; export { test, test2 }; # output -file:///mod.ts: EsModuleInfo { +file:///mod.ts: EsmModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/Namespaces02.txt b/tests/specs/symbols/Namespaces02.txt index 648fa061d..bf7c73ee1 100644 --- a/tests/specs/symbols/Namespaces02.txt +++ b/tests/specs/symbols/Namespaces02.txt @@ -10,7 +10,7 @@ export namespace RootNs { } # output -file:///mod.ts: EsModuleInfo { +file:///mod.ts: EsmModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/NamespacesAmbient.txt b/tests/specs/symbols/NamespacesAmbient.txt index 302dbdd3c..24775738d 100644 --- a/tests/specs/symbols/NamespacesAmbient.txt +++ b/tests/specs/symbols/NamespacesAmbient.txt @@ -47,7 +47,7 @@ namespace Inner { } # output -file:///mod.ts: EsModuleInfo { +file:///mod.ts: EsmModuleInfo { module_id: ModuleId( 0, ), @@ -805,7 +805,7 @@ file:///mod.ts: EsModuleInfo { }, }, } -file:///other.d.ts: EsModuleInfo { +file:///other.d.ts: EsmModuleInfo { module_id: ModuleId( 1, ), diff --git a/tests/specs/symbols/ReExportJson01.txt b/tests/specs/symbols/ReExportJson01.txt index 7f32cde05..813a6f369 100644 --- a/tests/specs/symbols/ReExportJson01.txt +++ b/tests/specs/symbols/ReExportJson01.txt @@ -51,7 +51,7 @@ file:///bar.json: JsonModuleInfo { members: {}, }, } -file:///mod.ts: EsModuleInfo { +file:///mod.ts: EsmModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/ReexportAll.txt b/tests/specs/symbols/ReexportAll.txt index fa082b3cc..32f821368 100644 --- a/tests/specs/symbols/ReexportAll.txt +++ b/tests/specs/symbols/ReexportAll.txt @@ -18,7 +18,7 @@ export { Test } from "./b.ts"; export { Test, test1 } from "./c.ts"; # output -file:///a.ts: EsModuleInfo { +file:///a.ts: EsmModuleInfo { module_id: ModuleId( 1, ), @@ -394,7 +394,7 @@ file:///a.ts: EsModuleInfo { }, }, } -file:///b.ts: EsModuleInfo { +file:///b.ts: EsmModuleInfo { module_id: ModuleId( 2, ), @@ -471,7 +471,7 @@ file:///b.ts: EsModuleInfo { }, }, } -file:///c.ts: EsModuleInfo { +file:///c.ts: EsmModuleInfo { module_id: ModuleId( 3, ), @@ -546,7 +546,7 @@ file:///c.ts: EsModuleInfo { }, }, } -file:///mod.ts: EsModuleInfo { +file:///mod.ts: EsmModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/ReexportDeep.txt b/tests/specs/symbols/ReexportDeep.txt index 57633db07..16801bbe6 100644 --- a/tests/specs/symbols/ReexportDeep.txt +++ b/tests/specs/symbols/ReexportDeep.txt @@ -16,7 +16,7 @@ export declare namespace MessageEntity { } # output -file:///a.ts: EsModuleInfo { +file:///a.ts: EsmModuleInfo { module_id: ModuleId( 1, ), @@ -56,7 +56,7 @@ file:///a.ts: EsModuleInfo { }, }, } -file:///b.ts: EsModuleInfo { +file:///b.ts: EsmModuleInfo { module_id: ModuleId( 2, ), @@ -96,7 +96,7 @@ file:///b.ts: EsModuleInfo { }, }, } -file:///c.ts: EsModuleInfo { +file:///c.ts: EsmModuleInfo { module_id: ModuleId( 3, ), @@ -243,7 +243,7 @@ file:///c.ts: EsModuleInfo { }, }, } -file:///mod.ts: EsModuleInfo { +file:///mod.ts: EsmModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/ReexportExport.txt b/tests/specs/symbols/ReexportExport.txt index 9b7d0aebc..9c8d1e6ce 100644 --- a/tests/specs/symbols/ReexportExport.txt +++ b/tests/specs/symbols/ReexportExport.txt @@ -3,7 +3,7 @@ export function foo(): void {} export { foo as bar }; # output -file:///mod.ts: EsModuleInfo { +file:///mod.ts: EsmModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/TsExternalModuleRef01.txt b/tests/specs/symbols/TsExternalModuleRef01.txt index 69f1fd377..97709f362 100644 --- a/tests/specs/symbols/TsExternalModuleRef01.txt +++ b/tests/specs/symbols/TsExternalModuleRef01.txt @@ -11,7 +11,7 @@ namespace MyNamespace { export = MyNamespace; # output -file:///a.ts: EsModuleInfo { +file:///a.ts: EsmModuleInfo { module_id: ModuleId( 1, ), @@ -157,7 +157,7 @@ file:///a.ts: EsModuleInfo { }, }, } -file:///mod.ts: EsModuleInfo { +file:///mod.ts: EsmModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/TsExternalModuleRef02.txt b/tests/specs/symbols/TsExternalModuleRef02.txt index f8a14e8d9..a21984b34 100644 --- a/tests/specs/symbols/TsExternalModuleRef02.txt +++ b/tests/specs/symbols/TsExternalModuleRef02.txt @@ -11,7 +11,7 @@ namespace MyNamespace { export = MyNamespace; # output -file:///a.ts: EsModuleInfo { +file:///a.ts: EsmModuleInfo { module_id: ModuleId( 1, ), @@ -157,7 +157,7 @@ file:///a.ts: EsModuleInfo { }, }, } -file:///mod.ts: EsModuleInfo { +file:///mod.ts: EsmModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/TsNamespaceExport01.txt b/tests/specs/symbols/TsNamespaceExport01.txt index c5109ed77..b25924770 100644 --- a/tests/specs/symbols/TsNamespaceExport01.txt +++ b/tests/specs/symbols/TsNamespaceExport01.txt @@ -8,7 +8,7 @@ export function isPrime(x: number): boolean { export as namespace mathLib; # output -file:///mod.ts: EsModuleInfo { +file:///mod.ts: EsmModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/TsQualifiedName01.txt b/tests/specs/symbols/TsQualifiedName01.txt index 9a1cdee58..d5b41714d 100644 --- a/tests/specs/symbols/TsQualifiedName01.txt +++ b/tests/specs/symbols/TsQualifiedName01.txt @@ -8,7 +8,7 @@ namespace Other { } # output -file:///mod.ts: EsModuleInfo { +file:///mod.ts: EsmModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/TypeAlias01.txt b/tests/specs/symbols/TypeAlias01.txt index e48f78d5c..55c1034f4 100644 --- a/tests/specs/symbols/TypeAlias01.txt +++ b/tests/specs/symbols/TypeAlias01.txt @@ -8,7 +8,7 @@ interface Other2 {} export type Tuple = [lat: number, long: number]; # output -file:///mod.ts: EsModuleInfo { +file:///mod.ts: EsmModuleInfo { module_id: ModuleId( 0, ), diff --git a/tests/specs/symbols/TypeScriptTypesHeader.txt b/tests/specs/symbols/TypeScriptTypesHeader.txt index 307b139fe..153ce138f 100644 --- a/tests/specs/symbols/TypeScriptTypesHeader.txt +++ b/tests/specs/symbols/TypeScriptTypesHeader.txt @@ -33,7 +33,7 @@ export interface MyInterface3 { } # output -file:///mod.ts: EsModuleInfo { +file:///mod.ts: EsmModuleInfo { module_id: ModuleId( 0, ), @@ -227,7 +227,7 @@ file:///mod.ts: EsModuleInfo { == symbol deps (types only) == 4:148..223 [Id(("MyInterface", #2)), Id(("MyInterface2", #2)), Id(("MyInterface3", #2))] -file:///other.d.ts: EsModuleInfo { +file:///other.d.ts: EsmModuleInfo { module_id: ModuleId( 1, ), @@ -336,7 +336,7 @@ file:///other.d.ts: EsModuleInfo { }, }, } -file:///other.js: EsModuleInfo { +file:///other.js: EsmModuleInfo { module_id: ModuleId( 2, ), @@ -374,7 +374,7 @@ file:///other.js: EsModuleInfo { }, }, } -file:///typescript.ts: EsModuleInfo { +file:///typescript.ts: EsmModuleInfo { module_id: ModuleId( 3, ), @@ -483,7 +483,7 @@ file:///typescript.ts: EsModuleInfo { }, }, } -https://localhost/mod.d.ts: EsModuleInfo { +https://localhost/mod.d.ts: EsmModuleInfo { module_id: ModuleId( 4, ), @@ -660,7 +660,7 @@ https://localhost/mod.d.ts: EsModuleInfo { }, }, } -https://localhost/mod.js: EsModuleInfo { +https://localhost/mod.js: EsmModuleInfo { module_id: ModuleId( 5, ), diff --git a/tests/specs/symbols/TypesEntrypoint.txt b/tests/specs/symbols/TypesEntrypoint.txt index a18cd20ca..8b50452ec 100644 --- a/tests/specs/symbols/TypesEntrypoint.txt +++ b/tests/specs/symbols/TypesEntrypoint.txt @@ -11,7 +11,7 @@ export interface MyInterface { } # output -file:///mod.d.ts: EsModuleInfo { +file:///mod.d.ts: EsmModuleInfo { module_id: ModuleId( 0, ), @@ -120,7 +120,7 @@ file:///mod.d.ts: EsModuleInfo { }, }, } -file:///mod.js: EsModuleInfo { +file:///mod.js: EsmModuleInfo { module_id: ModuleId( 1, ), diff --git a/tests/specs/symbols/UnresolvedParts.txt b/tests/specs/symbols/UnresolvedParts.txt index a5af6d98d..3f5807c3b 100644 --- a/tests/specs/symbols/UnresolvedParts.txt +++ b/tests/specs/symbols/UnresolvedParts.txt @@ -16,7 +16,7 @@ export import NonExistent2 = other.NotFound; export class Other {} # output -file:///mod.ts: EsModuleInfo { +file:///mod.ts: EsmModuleInfo { module_id: ModuleId( 0, ), @@ -316,7 +316,7 @@ file:///mod.ts: EsModuleInfo { }, }, } -file:///other.ts: EsModuleInfo { +file:///other.ts: EsmModuleInfo { module_id: ModuleId( 1, ), diff --git a/tests/specs/symbols/UsingDecl.txt b/tests/specs/symbols/UsingDecl.txt index 3540e9441..cb62f9084 100644 --- a/tests/specs/symbols/UsingDecl.txt +++ b/tests/specs/symbols/UsingDecl.txt @@ -5,7 +5,7 @@ using test = 5; export { test }; # output -file:///mod.ts: EsModuleInfo { +file:///mod.ts: EsmModuleInfo { module_id: ModuleId( 0, ), From b97c4916fb0ece2d15121a3371cf7b706a677d9d Mon Sep 17 00:00:00 2001 From: David Sherret Date: Fri, 12 Jan 2024 15:50:43 -0500 Subject: [PATCH 04/14] Fix --- src/graph.rs | 9 +++++---- src/symbols/analyzer.rs | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/graph.rs b/src/graph.rs index 810f48e3d..b6816e7df 100644 --- a/src/graph.rs +++ b/src/graph.rs @@ -826,7 +826,7 @@ impl EsModuleSource { pub fn new_with_text(bytes: Arc<[u8]>) -> Result { let charset = text_encoding::detect_charset(bytes.as_ref()); let mut text = - match text_encoding::convert_to_utf8(bytes.as_ref(), &charset)? { + match text_encoding::convert_to_utf8(bytes.as_ref(), charset)? { Cow::Borrowed(text) => { if text.starts_with(text_encoding::BOM_CHAR) { text.to_string() @@ -1877,7 +1877,7 @@ pub(crate) fn parse_module( Some("json") )) { - let source = new_source_with_text(&specifier, content)?; + let source = new_source_with_text(specifier, content)?; return Ok(Module::Json(JsonModule { maybe_cache_info: None, text: source.text.unwrap(), @@ -1917,7 +1917,7 @@ pub(crate) fn parse_module( | MediaType::Dts | MediaType::Dmts | MediaType::Dcts => { - let source = new_source_with_text(&specifier, content)?; + let source = new_source_with_text(specifier, content)?; match module_analyzer.analyze( specifier, source.text.clone().unwrap(), @@ -1943,7 +1943,7 @@ pub(crate) fn parse_module( } } MediaType::Unknown if is_root => { - let source = new_source_with_text(&specifier, content)?; + let source = new_source_with_text(specifier, content)?; match module_analyzer.analyze( specifier, source.text.clone().unwrap(), @@ -4264,6 +4264,7 @@ impl<'a> NpmSpecifierResolver<'a> { } } +#[allow(clippy::result_large_err)] fn new_source_with_text( specifier: &ModuleSpecifier, text: Arc<[u8]>, diff --git a/src/symbols/analyzer.rs b/src/symbols/analyzer.rs index 1c4a0c872..ca6d19e51 100644 --- a/src/symbols/analyzer.rs +++ b/src/symbols/analyzer.rs @@ -1449,7 +1449,7 @@ pub struct EsModuleInfo { impl std::fmt::Debug for EsModuleInfo { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - // todo(dsherret): rename in a separate PR in order to reduce noise + // todo(dsherret): rename to EsModuleInfo in a separate PR in order to reduce noise f.debug_struct("EsmModuleInfo") .field("module_id", &self.module_id) .field("specifier", &self.specifier.as_str()) From d736a34dbaa9aaa3289b7a53ff1a9fc2ddb0920d Mon Sep 17 00:00:00 2001 From: David Sherret Date: Tue, 30 Jan 2024 17:01:32 -0500 Subject: [PATCH 05/14] Switch to an enum now for simplicity. We'll do this properly in the future --- src/graph.rs | 91 +++++++++++++++-------------------------- src/lib.rs | 4 +- src/symbols/analyzer.rs | 2 +- src/text_encoding.rs | 29 +++++++++++++ 4 files changed, 66 insertions(+), 60 deletions(-) diff --git a/src/graph.rs b/src/graph.rs index c75bd3ce9..c20f39704 100644 --- a/src/graph.rs +++ b/src/graph.rs @@ -789,10 +789,8 @@ pub struct JsonModule { pub specifier: ModuleSpecifier, #[serde(flatten, skip_serializing_if = "Option::is_none")] pub maybe_cache_info: Option, - #[serde(skip_serializing)] + #[serde(rename = "size", serialize_with = "serialize_text")] pub text: Arc, - #[serde(rename = "size", serialize_with = "serialize_source_bytes")] - pub bytes: Arc<[u8]>, // todo(#240): This will always be MediaType::Json, but it's currently // used in the --json output. It's redundant though. pub media_type: MediaType, @@ -819,46 +817,29 @@ pub struct FastCheckTypeModule { } #[derive(Debug, Clone)] -pub struct EsModuleSource { - pub bytes: Arc<[u8]>, - pub text: Option>, +pub enum EsModuleSource { + Bytes(Arc<[u8]>), + Text(Arc), } impl EsModuleSource { - pub fn new_with_text(bytes: Arc<[u8]>) -> Result { - let charset = text_encoding::detect_charset(bytes.as_ref()); - let mut text = - match text_encoding::convert_to_utf8(bytes.as_ref(), charset)? { - Cow::Borrowed(text) => { - if text.starts_with(text_encoding::BOM_CHAR) { - text.to_string() - } else { - return Ok(Self { - bytes: bytes.clone(), - // SAFETY: we know it's avalid utf-8 string at this point - text: unsafe { - let raw_ptr = Arc::into_raw(bytes); - Some(Arc::from_raw(std::mem::transmute::< - *const [u8], - *const str, - >(raw_ptr))) - }, - }); - } - } - Cow::Owned(text) => text, - }; - text_encoding::strip_bom_mut(&mut text); - let text: Arc = Arc::from(text); - Ok(Self::from_text(text)) + pub fn new_as_text_from_bytes( + bytes: Arc<[u8]>, + ) -> Result { + text_encoding::arc_bytes_to_text(bytes).map(EsModuleSource::Text) } - /// This is internal because generally we don't want consumers doing - /// their own bytes decoding and instead to use `EsModuleSource::new_with_text`. - pub(crate) fn from_text(text: Arc) -> Self { - Self { - bytes: text.clone().into(), - text: Some(text), + pub fn text(&self) -> Option<&Arc> { + match self { + Self::Bytes(_) => None, + Self::Text(text) => Some(text), + } + } + + pub fn len(&self) -> usize { + match self { + Self::Bytes(bytes) => bytes.len(), + Self::Text(text) => text.len(), } } } @@ -1879,11 +1860,12 @@ pub(crate) fn parse_module( Some("json") )) { - let source = new_source_with_text(specifier, content)?; + let text = text_encoding::arc_bytes_to_text(content).map_err(|err| { + ModuleError::LoadingErr(specifier.clone(), None, Arc::new(err.into())) + })?; return Ok(Module::Json(JsonModule { maybe_cache_info: None, - text: source.text.unwrap(), - bytes: source.bytes, + text, media_type: MediaType::Json, specifier: specifier.clone(), })); @@ -1920,11 +1902,7 @@ pub(crate) fn parse_module( | MediaType::Dmts | MediaType::Dcts => { let source = new_source_with_text(specifier, content)?; - match module_analyzer.analyze( - specifier, - source.text.clone().unwrap(), - media_type, - ) { + match module_analyzer.analyze(specifier, source.clone(), media_type) { Ok(module_info) => { // Return the module as a valid module Ok(Module::Esm(parse_es_module_from_module_info( @@ -1933,7 +1911,7 @@ pub(crate) fn parse_module( media_type, maybe_headers, module_info, - source, + EsModuleSource::Text(source), file_system, maybe_resolver, maybe_npm_resolver, @@ -1948,7 +1926,7 @@ pub(crate) fn parse_module( let source = new_source_with_text(specifier, content)?; match module_analyzer.analyze( specifier, - source.text.clone().unwrap(), + source.clone(), MediaType::JavaScript, ) { Ok(module_info) => { @@ -1959,7 +1937,7 @@ pub(crate) fn parse_module( media_type, maybe_headers, module_info, - source, + EsModuleSource::Text(source), file_system, maybe_resolver, maybe_npm_resolver, @@ -3205,7 +3183,7 @@ impl<'a, 'graph> Builder<'a, 'graph> { match new_source_with_text(&module.specifier, content) { Ok(source) => { - module.source = source; + module.source = EsModuleSource::Text(source); } Err(err) => *slot = ModuleSlot::Err(err), } @@ -3218,8 +3196,7 @@ impl<'a, 'graph> Builder<'a, 'graph> { Module::Json(module) => { match new_source_with_text(&module.specifier, content) { Ok(source) => { - module.bytes = source.bytes; - module.text = source.text.unwrap(); + module.text = source; } Err(err) => *slot = ModuleSlot::Err(err), } @@ -4303,8 +4280,8 @@ impl<'a> NpmSpecifierResolver<'a> { fn new_source_with_text( specifier: &ModuleSpecifier, text: Arc<[u8]>, -) -> Result { - EsModuleSource::new_with_text(text).map_err(|err| { +) -> Result, ModuleError> { + text_encoding::arc_bytes_to_text(text).map_err(|err| { ModuleError::LoadingErr(specifier.clone(), None, Arc::new(err.into())) }) } @@ -4364,11 +4341,11 @@ fn serialize_esm_source( where S: Serializer, { - serializer.serialize_u32(source.bytes.len() as u32) + serializer.serialize_u32(source.len() as u32) } -fn serialize_source_bytes( - source: &Arc<[u8]>, +fn serialize_text( + source: &Arc, serializer: S, ) -> Result where diff --git a/src/lib.rs b/src/lib.rs index 53ab40014..f7d79b0fc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -142,7 +142,7 @@ pub fn parse_module_from_ast(options: ParseModuleFromAstOptions) -> EsModule { options.parsed_source.media_type(), options.maybe_headers, DefaultModuleAnalyzer::module_info(options.parsed_source), - graph::EsModuleSource::from_text(options.parsed_source.text_info().text()), + graph::EsModuleSource::Text(options.parsed_source.text_info().text()), options.file_system, options.maybe_resolver, options.maybe_npm_resolver, @@ -1769,7 +1769,7 @@ export function a(a) { assert_eq!(graph.module_slots.len(), 3); let data_specifier = ModuleSpecifier::parse("data:application/typescript,export%20*%20from%20%22https://example.com/c.ts%22;").unwrap(); let module = graph.get(&data_specifier).unwrap().esm().unwrap(); - let source = module.source.text.clone().unwrap(); + let source = module.source.text().clone().unwrap(); assert_eq!( source.as_ref(), r#"export * from "https://example.com/c.ts";"#, diff --git a/src/symbols/analyzer.rs b/src/symbols/analyzer.rs index 3b034b394..209b188c9 100644 --- a/src/symbols/analyzer.rs +++ b/src/symbols/analyzer.rs @@ -242,7 +242,7 @@ impl<'a> RootSymbol<'a> { .parser .parse_module(ParseOptions { specifier: &graph_module.specifier, - source: graph_module.source.text.clone()?, + source: graph_module.source.text()?.clone(), media_type: graph_module.media_type, scope_analysis: true, }) diff --git a/src/text_encoding.rs b/src/text_encoding.rs index b1433c6a6..ce4205103 100644 --- a/src/text_encoding.rs +++ b/src/text_encoding.rs @@ -1,4 +1,5 @@ use std::borrow::Cow; +use std::sync::Arc; pub const BOM_CHAR: char = '\u{FEFF}'; @@ -46,6 +47,34 @@ pub fn strip_bom_mut(text: &mut String) { } } +/// Converts an `Arc<[u8]>` to an `Arc`. +pub fn arc_bytes_to_text(bytes: Arc<[u8]>) -> Result, std::io::Error> { + let charset = detect_charset(bytes.as_ref()); + let text = match convert_to_utf8(bytes.as_ref(), charset)? { + Cow::Borrowed(text) => { + if text.starts_with(BOM_CHAR) { + text[..BOM_CHAR.len_utf8()].to_string() + } else { + return Ok( + // SAFETY: we know it's avalid utf-8 string at this point + unsafe { + let raw_ptr = Arc::into_raw(bytes); + Arc::from_raw(std::mem::transmute::<*const [u8], *const str>( + raw_ptr, + )) + }, + ); + } + } + Cow::Owned(mut text) => { + strip_bom_mut(&mut text); + text + } + }; + let text: Arc = Arc::from(text); + Ok(text) +} + #[cfg(test)] mod test { use super::*; From 87a0f97fb9360abe373658557156d9ea70be0823 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Wed, 31 Jan 2024 09:36:03 -0500 Subject: [PATCH 06/14] Update, but probably not going to do this. --- src/graph.rs | 28 +++++++++++++++++++++------- src/lib.rs | 2 +- src/symbols/analyzer.rs | 4 ++-- 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/src/graph.rs b/src/graph.rs index c20f39704..5aaa68d5e 100644 --- a/src/graph.rs +++ b/src/graph.rs @@ -789,8 +789,8 @@ pub struct JsonModule { pub specifier: ModuleSpecifier, #[serde(flatten, skip_serializing_if = "Option::is_none")] pub maybe_cache_info: Option, - #[serde(rename = "size", serialize_with = "serialize_text")] - pub text: Arc, + #[serde(rename = "size", serialize_with = "serialize_source")] + pub source: Arc, // todo(#240): This will always be MediaType::Json, but it's currently // used in the --json output. It's redundant though. pub media_type: MediaType, @@ -799,7 +799,7 @@ pub struct JsonModule { impl JsonModule { /// Return the size in bytes of the content of the JSON module. pub fn size(&self) -> usize { - self.text.as_bytes().len() + self.source.as_bytes().len() } } @@ -829,7 +829,21 @@ impl EsModuleSource { text_encoding::arc_bytes_to_text(bytes).map(EsModuleSource::Text) } - pub fn text(&self) -> Option<&Arc> { + pub fn as_bytes(&self) -> &[u8] { + match self { + Self::Bytes(bytes) => bytes, + Self::Text(text) => text.as_bytes(), + } + } + + pub fn bytes(&self) -> Arc<[u8]> { + match self { + Self::Bytes(bytes) => bytes.clone(), + Self::Text(text) => Arc::from(text.clone()), + } + } + + pub fn maybe_text(&self) -> Option<&Arc> { match self { Self::Bytes(_) => None, Self::Text(text) => Some(text), @@ -1865,7 +1879,7 @@ pub(crate) fn parse_module( })?; return Ok(Module::Json(JsonModule { maybe_cache_info: None, - text, + source: text, media_type: MediaType::Json, specifier: specifier.clone(), })); @@ -3196,7 +3210,7 @@ impl<'a, 'graph> Builder<'a, 'graph> { Module::Json(module) => { match new_source_with_text(&module.specifier, content) { Ok(source) => { - module.text = source; + module.source = source; } Err(err) => *slot = ModuleSlot::Err(err), } @@ -4344,7 +4358,7 @@ where serializer.serialize_u32(source.len() as u32) } -fn serialize_text( +fn serialize_source( source: &Arc, serializer: S, ) -> Result diff --git a/src/lib.rs b/src/lib.rs index f7d79b0fc..8fc77b6c5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1769,7 +1769,7 @@ export function a(a) { assert_eq!(graph.module_slots.len(), 3); let data_specifier = ModuleSpecifier::parse("data:application/typescript,export%20*%20from%20%22https://example.com/c.ts%22;").unwrap(); let module = graph.get(&data_specifier).unwrap().esm().unwrap(); - let source = module.source.text().clone().unwrap(); + let source = module.source.maybe_text().unwrap(); assert_eq!( source.as_ref(), r#"export * from "https://example.com/c.ts";"#, diff --git a/src/symbols/analyzer.rs b/src/symbols/analyzer.rs index 209b188c9..8033bb528 100644 --- a/src/symbols/analyzer.rs +++ b/src/symbols/analyzer.rs @@ -184,7 +184,7 @@ impl<'a> RootSymbol<'a> { let specifier = &json_module.specifier; // it's not ideal having to use SourceTextInfo here, but it makes // it easier to interop with ParsedSource - let source_text_info = SourceTextInfo::new(json_module.text.clone()); + let source_text_info = SourceTextInfo::new(json_module.source.clone()); let range = source_text_info.range(); let module_id = ModuleId(self.ids_to_modules.len() as u32); let decls = { @@ -242,7 +242,7 @@ impl<'a> RootSymbol<'a> { .parser .parse_module(ParseOptions { specifier: &graph_module.specifier, - source: graph_module.source.text()?.clone(), + source: graph_module.source.maybe_text()?.clone(), media_type: graph_module.media_type, scope_analysis: true, }) From 22da8ea7d6aa0fd20ce441ae718c3da9bf9f7e26 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Wed, 31 Jan 2024 09:46:54 -0500 Subject: [PATCH 07/14] Revert. --- src/fast_check/range_finder.rs | 2 +- src/graph.rs | 107 +++++++++------------------------ src/lib.rs | 11 ++-- src/symbols/analyzer.rs | 17 +++--- 4 files changed, 46 insertions(+), 91 deletions(-) diff --git a/src/fast_check/range_finder.rs b/src/fast_check/range_finder.rs index 9435dcba5..4514627a7 100644 --- a/src/fast_check/range_finder.rs +++ b/src/fast_check/range_finder.rs @@ -1025,7 +1025,7 @@ impl<'a> PublicRangeFinder<'a> { return true; // just analyze it }; match module { - crate::Module::Esm(m) => is_typed_media_type(m.media_type), + crate::Module::Script(m) => is_typed_media_type(m.media_type), crate::Module::Json(_) => true, crate::Module::Npm(_) | crate::Module::Node(_) diff --git a/src/graph.rs b/src/graph.rs index 5aaa68d5e..0ea6d9678 100644 --- a/src/graph.rs +++ b/src/graph.rs @@ -694,7 +694,9 @@ pub struct WorkspaceMember { #[serde(rename_all = "camelCase")] #[serde(tag = "kind")] pub enum Module { - Esm(EsModule), + // todo(#239): remove this when updating the --json output for 2.0 + #[serde(rename = "esm")] + Script(ScriptModule), // todo(#239): remove this when updating the --json output for 2.0 #[serde(rename = "asserted")] Json(JsonModule), @@ -706,7 +708,7 @@ pub enum Module { impl Module { pub fn specifier(&self) -> &ModuleSpecifier { match self { - Module::Esm(module) => &module.specifier, + Module::Script(module) => &module.specifier, Module::Json(module) => &module.specifier, Module::Npm(module) => &module.specifier, Module::Node(module) => &module.specifier, @@ -722,8 +724,8 @@ impl Module { } } - pub fn esm(&self) -> Option<&EsModule> { - if let Module::Esm(module) = &self { + pub fn esm(&self) -> Option<&ScriptModule> { + if let Module::Script(module) = &self { Some(module) } else { None @@ -816,51 +818,9 @@ pub struct FastCheckTypeModule { pub source_map: Arc<[u8]>, } -#[derive(Debug, Clone)] -pub enum EsModuleSource { - Bytes(Arc<[u8]>), - Text(Arc), -} - -impl EsModuleSource { - pub fn new_as_text_from_bytes( - bytes: Arc<[u8]>, - ) -> Result { - text_encoding::arc_bytes_to_text(bytes).map(EsModuleSource::Text) - } - - pub fn as_bytes(&self) -> &[u8] { - match self { - Self::Bytes(bytes) => bytes, - Self::Text(text) => text.as_bytes(), - } - } - - pub fn bytes(&self) -> Arc<[u8]> { - match self { - Self::Bytes(bytes) => bytes.clone(), - Self::Text(text) => Arc::from(text.clone()), - } - } - - pub fn maybe_text(&self) -> Option<&Arc> { - match self { - Self::Bytes(_) => None, - Self::Text(text) => Some(text), - } - } - - pub fn len(&self) -> usize { - match self { - Self::Bytes(bytes) => bytes.len(), - Self::Text(text) => text.len(), - } - } -} - #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "camelCase")] -pub struct EsModule { +pub struct ScriptModule { #[serde( skip_serializing_if = "IndexMap::is_empty", serialize_with = "serialize_dependencies" @@ -868,8 +828,8 @@ pub struct EsModule { pub dependencies: IndexMap, #[serde(flatten, skip_serializing_if = "Option::is_none")] pub maybe_cache_info: Option, - #[serde(rename = "size", serialize_with = "serialize_esm_source")] - pub source: EsModuleSource, + #[serde(rename = "size", serialize_with = "serialize_source")] + pub source: Arc, #[serde(rename = "typesDependency", skip_serializing_if = "Option::is_none")] pub maybe_types_dependency: Option, #[serde(skip_serializing_if = "is_media_type_unknown")] @@ -879,8 +839,8 @@ pub struct EsModule { pub fast_check: Option, } -impl EsModule { - fn new(specifier: ModuleSpecifier, source: EsModuleSource) -> Self { +impl ScriptModule { + fn new(specifier: ModuleSpecifier, source: Arc) -> Self { Self { dependencies: Default::default(), maybe_cache_info: None, @@ -1133,7 +1093,7 @@ impl<'a> Iterator for ModuleEntryIterator<'a> { fn next(&mut self) -> Option { match self.previous_module.take() { Some(ModuleEntryRef::Module(module)) => match module { - Module::Esm(module) => { + Module::Script(module) => { let check_types = (self.check_js || !matches!( module.media_type, @@ -1239,7 +1199,7 @@ impl<'a> ModuleGraphErrorIterator<'a> { fn check_resolution( &self, - module: &EsModule, + module: &ScriptModule, mode: ResolutionMode, specifier_text: &str, resolution: &Resolution, @@ -1316,7 +1276,7 @@ impl<'a> Iterator for ModuleGraphErrorIterator<'a> { if let Some((_, module_entry)) = self.iterator.next() { match module_entry { - ModuleEntryRef::Module(Module::Esm(module)) => { + ModuleEntryRef::Module(Module::Script(module)) => { let check_types = (check_js || !matches!( module.media_type, @@ -1621,7 +1581,7 @@ impl ModuleGraph { prefer_types: bool, ) -> Option { match referring_module { - Module::Esm(referring_module) => { + Module::Script(referring_module) => { let dependency = referring_module.dependencies.get(specifier)?; self.resolve_dependency_from_dep(dependency, prefer_types) } @@ -1649,7 +1609,7 @@ impl ModuleGraph { // Even if we resolved the specifier, it doesn't mean the module is actually // there, so check in the module slots match self.module_slots.get(&resolved_specifier) { - Some(ModuleSlot::Module(Module::Esm(module))) if prefer_types => { + Some(ModuleSlot::Module(Module::Script(module))) if prefer_types => { // check for if this module has a types dependency if let Some(Resolution::Ok(resolved)) = module .maybe_types_dependency @@ -1919,13 +1879,13 @@ pub(crate) fn parse_module( match module_analyzer.analyze(specifier, source.clone(), media_type) { Ok(module_info) => { // Return the module as a valid module - Ok(Module::Esm(parse_es_module_from_module_info( + Ok(Module::Script(parse_es_module_from_module_info( graph_kind, specifier, media_type, maybe_headers, module_info, - EsModuleSource::Text(source), + source, file_system, maybe_resolver, maybe_npm_resolver, @@ -1945,13 +1905,13 @@ pub(crate) fn parse_module( ) { Ok(module_info) => { // Return the module as a valid module - Ok(Module::Esm(parse_es_module_from_module_info( + Ok(Module::Script(parse_es_module_from_module_info( graph_kind, specifier, media_type, maybe_headers, module_info, - EsModuleSource::Text(source), + source, file_system, maybe_resolver, maybe_npm_resolver, @@ -1981,12 +1941,12 @@ pub(crate) fn parse_es_module_from_module_info( media_type: MediaType, maybe_headers: Option<&HashMap>, module_info: ModuleInfo, - source: EsModuleSource, + source: Arc, file_system: &dyn FileSystem, maybe_resolver: Option<&dyn Resolver>, maybe_npm_resolver: Option<&dyn NpmResolver>, -) -> EsModule { - let mut module = EsModule::new(specifier.clone(), source); +) -> ScriptModule { + let mut module = ScriptModule::new(specifier.clone(), source); module.media_type = media_type; // Analyze the TypeScript triple-slash references @@ -3181,7 +3141,7 @@ impl<'a, 'graph> Builder<'a, 'graph> { match slot { ModuleSlot::Module(module) => { match module { - Module::Esm(module) => match module.media_type { + Module::Script(module) => match module.media_type { MediaType::JavaScript | MediaType::Jsx | MediaType::Mjs @@ -3197,7 +3157,7 @@ impl<'a, 'graph> Builder<'a, 'graph> { match new_source_with_text(&module.specifier, content) { Ok(source) => { - module.source = EsModuleSource::Text(source); + module.source = source; } Err(err) => *slot = ModuleSlot::Err(err), } @@ -3276,7 +3236,7 @@ impl<'a, 'graph> Builder<'a, 'graph> { module.maybe_cache_info = self.loader.get_cache_info(&module.specifier); } - Module::Esm(module) => { + Module::Script(module) => { module.maybe_cache_info = self.loader.get_cache_info(&module.specifier); } @@ -3881,7 +3841,8 @@ impl<'a, 'graph> Builder<'a, 'graph> { Err(err) => ModuleSlot::Err(err), }; - if let ModuleSlot::Module(Module::Esm(module)) = module_slot.borrow_mut() { + if let ModuleSlot::Module(Module::Script(module)) = module_slot.borrow_mut() + { if matches!(self.graph.graph_kind, GraphKind::All | GraphKind::CodeOnly) || module.maybe_types_dependency.is_none() { @@ -4012,7 +3973,7 @@ impl<'a, 'graph> Builder<'a, 'graph> { let module_slot = self.graph.module_slots.get_mut(&specifier).unwrap(); let module = match module_slot { ModuleSlot::Module(m) => match m { - Module::Esm(m) => m, + Module::Script(m) => m, _ => continue, }, ModuleSlot::Err(_) | ModuleSlot::Pending => continue, @@ -4348,16 +4309,6 @@ where seq.end() } -fn serialize_esm_source( - source: &EsModuleSource, - serializer: S, -) -> Result -where - S: Serializer, -{ - serializer.serialize_u32(source.len() as u32) -} - fn serialize_source( source: &Arc, serializer: S, diff --git a/src/lib.rs b/src/lib.rs index 8fc77b6c5..5b9a51f72 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -49,7 +49,6 @@ pub use graph::BuildDiagnostic; pub use graph::BuildOptions; pub use graph::Dependency; pub use graph::DiagnosticRange; -pub use graph::EsModule; pub use graph::ExternalModule; pub use graph::FastCheckTypeModule; pub use graph::FastCheckTypeModuleSlot; @@ -67,6 +66,7 @@ pub use graph::Range; pub use graph::Resolution; pub use graph::ResolutionError; pub use graph::ResolutionResolved; +pub use graph::ScriptModule; pub use graph::TypesDependency; pub use graph::WalkOptions; pub use graph::WorkspaceMember; @@ -135,14 +135,16 @@ pub struct ParseModuleFromAstOptions<'a> { } /// Parse an individual module from an AST, returning the module. -pub fn parse_module_from_ast(options: ParseModuleFromAstOptions) -> EsModule { +pub fn parse_module_from_ast( + options: ParseModuleFromAstOptions, +) -> ScriptModule { graph::parse_es_module_from_module_info( options.graph_kind, options.specifier, options.parsed_source.media_type(), options.maybe_headers, DefaultModuleAnalyzer::module_info(options.parsed_source), - graph::EsModuleSource::Text(options.parsed_source.text_info().text()), + options.parsed_source.text_info().text(), options.file_system, options.maybe_resolver, options.maybe_npm_resolver, @@ -1769,9 +1771,8 @@ export function a(a) { assert_eq!(graph.module_slots.len(), 3); let data_specifier = ModuleSpecifier::parse("data:application/typescript,export%20*%20from%20%22https://example.com/c.ts%22;").unwrap(); let module = graph.get(&data_specifier).unwrap().esm().unwrap(); - let source = module.source.maybe_text().unwrap(); assert_eq!( - source.as_ref(), + module.source.as_ref(), r#"export * from "https://example.com/c.ts";"#, ); } diff --git a/src/symbols/analyzer.rs b/src/symbols/analyzer.rs index 8033bb528..0745f1099 100644 --- a/src/symbols/analyzer.rs +++ b/src/symbols/analyzer.rs @@ -16,11 +16,11 @@ use deno_ast::SourceTextInfo; use indexmap::IndexMap; use indexmap::IndexSet; -use crate::EsModule; use crate::JsonModule; use crate::ModuleGraph; use crate::ModuleParser; use crate::ParseOptions; +use crate::ScriptModule; use super::collections::AdditiveOnlyIndexMap; use super::collections::AdditiveOnlyIndexMapForCopyValues; @@ -82,7 +82,7 @@ impl<'a> RootSymbol<'a> { }; match graph_module { - crate::Module::Esm(es_module) => self.analyze_es_module(es_module), + crate::Module::Script(es_module) => self.analyze_script_module(es_module), crate::Module::Json(json_module) => { Some(self.analyze_json_module(json_module)) } @@ -150,11 +150,14 @@ impl<'a> RootSymbol<'a> { ) } - fn analyze_es_module(&self, es_module: &EsModule) -> Option { - let Some(source) = self.parsed_source(es_module) else { + fn analyze_script_module( + &self, + script_module: &ScriptModule, + ) -> Option { + let Some(source) = self.parsed_source(script_module) else { return None; }; - let specifier = &es_module.specifier; + let specifier = &script_module.specifier; let module = source.module(); let module_id = ModuleId(self.ids_to_modules.len() as u32); @@ -237,12 +240,12 @@ impl<'a> RootSymbol<'a> { self.ids_to_modules.get(&module_id).unwrap().as_ref() } - fn parsed_source(&self, graph_module: &EsModule) -> Option { + fn parsed_source(&self, graph_module: &ScriptModule) -> Option { self .parser .parse_module(ParseOptions { specifier: &graph_module.specifier, - source: graph_module.source.maybe_text()?.clone(), + source: graph_module.source.clone(), media_type: graph_module.media_type, scope_analysis: true, }) From 95264e53952e896640484cc136732935e7404eaa Mon Sep 17 00:00:00 2001 From: David Sherret Date: Wed, 31 Jan 2024 09:59:15 -0500 Subject: [PATCH 08/14] Switch to just storing Arc --- src/fast_check/range_finder.rs | 2 +- src/graph.rs | 75 +++++++++++++--------------------- src/lib.rs | 26 ++++++------ src/symbols/analyzer.rs | 30 +++++++------- tests/integration_test.rs | 2 +- 5 files changed, 57 insertions(+), 78 deletions(-) diff --git a/src/fast_check/range_finder.rs b/src/fast_check/range_finder.rs index 4514627a7..3c439351c 100644 --- a/src/fast_check/range_finder.rs +++ b/src/fast_check/range_finder.rs @@ -1025,7 +1025,7 @@ impl<'a> PublicRangeFinder<'a> { return true; // just analyze it }; match module { - crate::Module::Script(m) => is_typed_media_type(m.media_type), + crate::Module::Js(m) => is_typed_media_type(m.media_type), crate::Module::Json(_) => true, crate::Module::Npm(_) | crate::Module::Node(_) diff --git a/src/graph.rs b/src/graph.rs index 0ea6d9678..db7471a6c 100644 --- a/src/graph.rs +++ b/src/graph.rs @@ -696,7 +696,7 @@ pub struct WorkspaceMember { pub enum Module { // todo(#239): remove this when updating the --json output for 2.0 #[serde(rename = "esm")] - Script(ScriptModule), + Js(JsModule), // todo(#239): remove this when updating the --json output for 2.0 #[serde(rename = "asserted")] Json(JsonModule), @@ -708,7 +708,7 @@ pub enum Module { impl Module { pub fn specifier(&self) -> &ModuleSpecifier { match self { - Module::Script(module) => &module.specifier, + Module::Js(module) => &module.specifier, Module::Json(module) => &module.specifier, Module::Npm(module) => &module.specifier, Module::Node(module) => &module.specifier, @@ -724,8 +724,8 @@ impl Module { } } - pub fn esm(&self) -> Option<&ScriptModule> { - if let Module::Script(module) = &self { + pub fn js(&self) -> Option<&JsModule> { + if let Module::Js(module) = &self { Some(module) } else { None @@ -820,7 +820,7 @@ pub struct FastCheckTypeModule { #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "camelCase")] -pub struct ScriptModule { +pub struct JsModule { #[serde( skip_serializing_if = "IndexMap::is_empty", serialize_with = "serialize_dependencies" @@ -839,7 +839,7 @@ pub struct ScriptModule { pub fast_check: Option, } -impl ScriptModule { +impl JsModule { fn new(specifier: ModuleSpecifier, source: Arc) -> Self { Self { dependencies: Default::default(), @@ -1093,7 +1093,7 @@ impl<'a> Iterator for ModuleEntryIterator<'a> { fn next(&mut self) -> Option { match self.previous_module.take() { Some(ModuleEntryRef::Module(module)) => match module { - Module::Script(module) => { + Module::Js(module) => { let check_types = (self.check_js || !matches!( module.media_type, @@ -1199,7 +1199,7 @@ impl<'a> ModuleGraphErrorIterator<'a> { fn check_resolution( &self, - module: &ScriptModule, + module: &JsModule, mode: ResolutionMode, specifier_text: &str, resolution: &Resolution, @@ -1276,7 +1276,7 @@ impl<'a> Iterator for ModuleGraphErrorIterator<'a> { if let Some((_, module_entry)) = self.iterator.next() { match module_entry { - ModuleEntryRef::Module(Module::Script(module)) => { + ModuleEntryRef::Module(Module::Js(module)) => { let check_types = (check_js || !matches!( module.media_type, @@ -1581,7 +1581,7 @@ impl ModuleGraph { prefer_types: bool, ) -> Option { match referring_module { - Module::Script(referring_module) => { + Module::Js(referring_module) => { let dependency = referring_module.dependencies.get(specifier)?; self.resolve_dependency_from_dep(dependency, prefer_types) } @@ -1609,7 +1609,7 @@ impl ModuleGraph { // Even if we resolved the specifier, it doesn't mean the module is actually // there, so check in the module slots match self.module_slots.get(&resolved_specifier) { - Some(ModuleSlot::Module(Module::Script(module))) if prefer_types => { + Some(ModuleSlot::Module(Module::Js(module))) if prefer_types => { // check for if this module has a types dependency if let Some(Resolution::Ok(resolved)) = module .maybe_types_dependency @@ -1674,7 +1674,7 @@ impl ModuleGraph { return Ok(None); }; - if let Some(specifier) = module.esm().and_then(|m| { + if let Some(specifier) = module.js().and_then(|m| { m.maybe_types_dependency .as_ref() .and_then(|d| d.dependency.ok()) @@ -1879,7 +1879,7 @@ pub(crate) fn parse_module( match module_analyzer.analyze(specifier, source.clone(), media_type) { Ok(module_info) => { // Return the module as a valid module - Ok(Module::Script(parse_es_module_from_module_info( + Ok(Module::Js(parse_js_module_from_module_info( graph_kind, specifier, media_type, @@ -1905,7 +1905,7 @@ pub(crate) fn parse_module( ) { Ok(module_info) => { // Return the module as a valid module - Ok(Module::Script(parse_es_module_from_module_info( + Ok(Module::Js(parse_js_module_from_module_info( graph_kind, specifier, media_type, @@ -1935,7 +1935,7 @@ pub(crate) fn parse_module( } #[allow(clippy::too_many_arguments)] -pub(crate) fn parse_es_module_from_module_info( +pub(crate) fn parse_js_module_from_module_info( graph_kind: GraphKind, specifier: &ModuleSpecifier, media_type: MediaType, @@ -1945,8 +1945,8 @@ pub(crate) fn parse_es_module_from_module_info( file_system: &dyn FileSystem, maybe_resolver: Option<&dyn Resolver>, maybe_npm_resolver: Option<&dyn NpmResolver>, -) -> ScriptModule { - let mut module = ScriptModule::new(specifier.clone(), source); +) -> JsModule { + let mut module = JsModule::new(specifier.clone(), source); module.media_type = media_type; // Analyze the TypeScript triple-slash references @@ -3141,32 +3141,14 @@ impl<'a, 'graph> Builder<'a, 'graph> { match slot { ModuleSlot::Module(module) => { match module { - Module::Script(module) => match module.media_type { - MediaType::JavaScript - | MediaType::Jsx - | MediaType::Mjs - | MediaType::Cjs - | MediaType::TypeScript - | MediaType::Mts - | MediaType::Cts - | MediaType::Dts - | MediaType::Dmts - | MediaType::Dcts - | MediaType::Tsx - | MediaType::Json => { - match new_source_with_text(&module.specifier, content) - { - Ok(source) => { - module.source = source; - } - Err(err) => *slot = ModuleSlot::Err(err), + Module::Js(module) => { + match new_source_with_text(&module.specifier, content) { + Ok(source) => { + module.source = source; } + Err(err) => *slot = ModuleSlot::Err(err), } - MediaType::Wasm - | MediaType::TsBuildInfo - | MediaType::SourceMap - | MediaType::Unknown => todo!(), - }, + } Module::Json(module) => { match new_source_with_text(&module.specifier, content) { Ok(source) => { @@ -3236,7 +3218,7 @@ impl<'a, 'graph> Builder<'a, 'graph> { module.maybe_cache_info = self.loader.get_cache_info(&module.specifier); } - Module::Script(module) => { + Module::Js(module) => { module.maybe_cache_info = self.loader.get_cache_info(&module.specifier); } @@ -3841,8 +3823,7 @@ impl<'a, 'graph> Builder<'a, 'graph> { Err(err) => ModuleSlot::Err(err), }; - if let ModuleSlot::Module(Module::Script(module)) = module_slot.borrow_mut() - { + if let ModuleSlot::Module(Module::Js(module)) = module_slot.borrow_mut() { if matches!(self.graph.graph_kind, GraphKind::All | GraphKind::CodeOnly) || module.maybe_types_dependency.is_none() { @@ -3973,7 +3954,7 @@ impl<'a, 'graph> Builder<'a, 'graph> { let module_slot = self.graph.module_slots.get_mut(&specifier).unwrap(); let module = match module_slot { ModuleSlot::Module(m) => match m { - Module::Script(m) => m, + Module::Js(m) => m, _ => continue, }, ModuleSlot::Err(_) | ModuleSlot::Pending => continue, @@ -4385,7 +4366,7 @@ mod tests { None, ) .unwrap(); - let module = module.esm().unwrap(); + let module = module.js().unwrap(); assert_eq!(module.dependencies.len(), 1); let dependency = module.dependencies.first().unwrap().1; assert_eq!( @@ -4915,7 +4896,7 @@ mod tests { .await; graph.valid().unwrap(); let module = graph.get(&Url::parse("file:///foo.ts").unwrap()).unwrap(); - let module = module.esm().unwrap(); + let module = module.js().unwrap(); let dependency_a = module.dependencies.get("file:///bar.ts").unwrap(); let dependency_b = module.dependencies.get("file:///baz.json").unwrap(); assert_eq!( diff --git a/src/lib.rs b/src/lib.rs index 5b9a51f72..21eb2d4a8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -54,6 +54,7 @@ pub use graph::FastCheckTypeModule; pub use graph::FastCheckTypeModuleSlot; pub use graph::GraphImport; pub use graph::GraphKind; +pub use graph::JsModule; pub use graph::JsonModule; pub use graph::Module; pub use graph::ModuleEntryRef; @@ -66,7 +67,6 @@ pub use graph::Range; pub use graph::Resolution; pub use graph::ResolutionError; pub use graph::ResolutionResolved; -pub use graph::ScriptModule; pub use graph::TypesDependency; pub use graph::WalkOptions; pub use graph::WorkspaceMember; @@ -135,10 +135,8 @@ pub struct ParseModuleFromAstOptions<'a> { } /// Parse an individual module from an AST, returning the module. -pub fn parse_module_from_ast( - options: ParseModuleFromAstOptions, -) -> ScriptModule { - graph::parse_es_module_from_module_info( +pub fn parse_module_from_ast(options: ParseModuleFromAstOptions) -> JsModule { + graph::parse_js_module_from_module_info( options.graph_kind, options.specifier, options.parsed_source.media_type(), @@ -221,7 +219,7 @@ mod tests { .unwrap() .module() .unwrap() - .esm() + .js() .unwrap(); assert_eq!(module.dependencies.len(), 1); let maybe_dependency = module.dependencies.get("./test02.ts"); @@ -1008,7 +1006,7 @@ console.log(a); .unwrap() .module() .unwrap() - .esm() + .js() .unwrap(); assert_eq!(module.media_type, MediaType::TypeScript); } @@ -1770,7 +1768,7 @@ export function a(a) { .await; assert_eq!(graph.module_slots.len(), 3); let data_specifier = ModuleSpecifier::parse("data:application/typescript,export%20*%20from%20%22https://example.com/c.ts%22;").unwrap(); - let module = graph.get(&data_specifier).unwrap().esm().unwrap(); + let module = graph.get(&data_specifier).unwrap().js().unwrap(); assert_eq!( module.source.as_ref(), r#"export * from "https://example.com/c.ts";"#, @@ -1817,7 +1815,7 @@ export function a(a) { }, ) .await; - let module = graph.get(&graph.roots[0]).unwrap().esm().unwrap(); + let module = graph.get(&graph.roots[0]).unwrap().js().unwrap(); let maybe_dep = module.dependencies.get("b"); assert!(maybe_dep.is_some()); let dep = maybe_dep.unwrap(); @@ -1877,7 +1875,7 @@ export function a(a) { }, ) .await; - let module = graph.get(&graph.roots[0]).unwrap().esm().unwrap(); + let module = graph.get(&graph.roots[0]).unwrap().js().unwrap(); let types_dep = module.maybe_types_dependency.as_ref().unwrap(); assert_eq!(types_dep.specifier, "file:///a.js"); assert_eq!( @@ -3030,7 +3028,7 @@ export function a(a) { maybe_npm_resolver: None, }) .unwrap(); - let actual = actual.esm().unwrap(); + let actual = actual.js().unwrap(); assert_eq!(actual.dependencies.len(), 7); assert_eq!(actual.specifier, specifier); assert_eq!(actual.media_type, MediaType::TypeScript); @@ -3047,7 +3045,7 @@ export function a(a) { maybe_npm_resolver: None, }) .unwrap(); - let actual = actual.esm().unwrap(); + let actual = actual.js().unwrap(); assert_eq!(actual.dependencies.len(), 4); } @@ -3140,7 +3138,7 @@ export function a(a) { maybe_npm_resolver: None, }) .unwrap(); - let actual = actual.esm().unwrap(); + let actual = actual.js().unwrap(); assert_eq!(actual.dependencies.len(), 1); let dep = actual .dependencies @@ -3183,7 +3181,7 @@ export function a(a) { maybe_npm_resolver: None, }) .unwrap(); - let actual = actual.esm().unwrap(); + let actual = actual.js().unwrap(); assert_eq!(actual.dependencies.len(), 1); let dep = actual .dependencies diff --git a/src/symbols/analyzer.rs b/src/symbols/analyzer.rs index 0745f1099..43edf616d 100644 --- a/src/symbols/analyzer.rs +++ b/src/symbols/analyzer.rs @@ -16,11 +16,11 @@ use deno_ast::SourceTextInfo; use indexmap::IndexMap; use indexmap::IndexSet; +use crate::JsModule; use crate::JsonModule; use crate::ModuleGraph; use crate::ModuleParser; use crate::ParseOptions; -use crate::ScriptModule; use super::collections::AdditiveOnlyIndexMap; use super::collections::AdditiveOnlyIndexMapForCopyValues; @@ -82,7 +82,7 @@ impl<'a> RootSymbol<'a> { }; match graph_module { - crate::Module::Script(es_module) => self.analyze_script_module(es_module), + crate::Module::Js(js_module) => self.analyze_js_module(js_module), crate::Module::Json(json_module) => { Some(self.analyze_json_module(json_module)) } @@ -150,11 +150,11 @@ impl<'a> RootSymbol<'a> { ) } - fn analyze_script_module( + fn analyze_js_module( &self, - script_module: &ScriptModule, + script_module: &JsModule, ) -> Option { - let Some(source) = self.parsed_source(script_module) else { + let Ok(source) = self.parsed_source(script_module) else { return None; }; let specifier = &script_module.specifier; @@ -240,16 +240,16 @@ impl<'a> RootSymbol<'a> { self.ids_to_modules.get(&module_id).unwrap().as_ref() } - fn parsed_source(&self, graph_module: &ScriptModule) -> Option { - self - .parser - .parse_module(ParseOptions { - specifier: &graph_module.specifier, - source: graph_module.source.clone(), - media_type: graph_module.media_type, - scope_analysis: true, - }) - .ok() + fn parsed_source( + &self, + graph_module: &JsModule, + ) -> Result { + self.parser.parse_module(ParseOptions { + specifier: &graph_module.specifier, + source: graph_module.source.clone(), + media_type: graph_module.media_type, + scope_analysis: true, + }) } } diff --git a/tests/integration_test.rs b/tests/integration_test.rs index 50e51854b..93b35bb13 100644 --- a/tests/integration_test.rs +++ b/tests/integration_test.rs @@ -89,7 +89,7 @@ async fn test_graph_specs() { } // now the fast check modules let fast_check_modules = result.graph.modules().filter_map(|module| { - let module = module.esm()?; + let module = module.js()?; let fast_check = module.fast_check.as_ref()?; Some((module, fast_check)) }); From dd748e8a8fb5e19ed48d65687cc7d8f5edca66c9 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Wed, 31 Jan 2024 10:09:35 -0500 Subject: [PATCH 09/14] Maybe fix --- deno.json | 2 +- js/mod.ts | 2 +- js/test.ts | 28 ++++++++++++++-------------- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/deno.json b/deno.json index dfe8a4bc4..9e652829b 100644 --- a/deno.json +++ b/deno.json @@ -1,6 +1,6 @@ { "tasks": { - "build": "cp LICENSE js/LICENSE && deno run --unstable -A --no-check https://deno.land/x/wasmbuild@0.15.1/main.ts --no-default-features --project deno_graph_wasm --out js", + "build": "cp LICENSE js/LICENSE && deno run -A --no-check https://deno.land/x/wasmbuild@0.15.1/main.ts --no-default-features --project deno_graph_wasm --out js", "build:npm": "deno run -A _build_npm.ts", "test": "deno test --allow-read --allow-net" }, diff --git a/js/mod.ts b/js/mod.ts index 4435c166a..38266446d 100644 --- a/js/mod.ts +++ b/js/mod.ts @@ -192,7 +192,7 @@ export async function init(opts?: wasm.InstantiateOptions) { */ export function parseModule( specifier: string, - content: string, + content: Uint8Array, options: ParseModuleOptions = {}, ): ModuleJson { const { diff --git a/js/test.ts b/js/test.ts index 32b015277..fea01259c 100644 --- a/js/test.ts +++ b/js/test.ts @@ -538,13 +538,13 @@ Deno.test({ await init(); const module = parseModule( "file:///a/test01.js", - ` + new TextEncoder().encode(` /// import { a } from "./a.ts"; import * as b from "./b.ts"; export { c } from "./c.ts"; const d = await import("./d.ts"); - `, + `), ); assertEquals(module, { "specifier": "file:///a/test01.js", @@ -609,9 +609,9 @@ Deno.test({ await init(); const module = parseModule( `https://example.com/a`, - `declare interface A { + new TextEncoder().encode(`declare interface A { a: string; - }`, + }`), { headers: { "content-type": "application/typescript; charset=utf-8", @@ -628,10 +628,10 @@ Deno.test({ await init(); const module = parseModule( `file:///a/test01.tsx`, - `/* @jsxImportSource http://example.com/preact */ + new TextEncoder().encode(`/* @jsxImportSource http://example.com/preact */ export function A() {
Hello Deno
- }`, + }`), { jsxImportSourceModule: "jsx-dev-runtime", }, @@ -651,10 +651,10 @@ Deno.test({ await init(); const module = parseModule( `file:///a/test01.tsx`, - ` + new TextEncoder().encode(` export function A() {
Hello Deno
- }`, + }`), { defaultJsxImportSource: "http://example.com/preact", }, @@ -673,7 +673,7 @@ Deno.test({ await init(); assertThrows( () => { - parseModule("./bad.ts", `console.log("hello");`); + parseModule("./bad.ts", new TextEncoder().encode(`console.log("hello");`)); }, Error, "relative URL without a base", @@ -687,7 +687,7 @@ Deno.test({ await init(); assertThrows( () => { - parseModule("file:///a/test.md", `# Some Markdown\n\n**bold**`); + parseModule("file:///a/test.md", new TextEncoder().encode(`# Some Markdown\n\n**bold**`)); }, Error, "The module's source code could not be parsed", @@ -701,10 +701,10 @@ Deno.test({ await init(); const module = parseModule( "file:///a/test01.js", - ` + new TextEncoder().encode(` import a from "./a.json" with { type: "json" }; await import("./b.json", { with: { type: "json" } }); - `, + `), ); assertEquals(module, { "dependencies": [ @@ -758,10 +758,10 @@ Deno.test({ await init(); const module = parseModule( "file:///a/foo.ts", - ` + new TextEncoder().encode(` /// /// - `, + `), ); assertEquals(module, { "dependencies": [ From 13eff1d2d76ea9110eb4f966dbbaec6140e3749b Mon Sep 17 00:00:00 2001 From: David Sherret Date: Wed, 31 Jan 2024 10:10:30 -0500 Subject: [PATCH 10/14] Format --- js/test.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/js/test.ts b/js/test.ts index fea01259c..acffdffae 100644 --- a/js/test.ts +++ b/js/test.ts @@ -673,7 +673,10 @@ Deno.test({ await init(); assertThrows( () => { - parseModule("./bad.ts", new TextEncoder().encode(`console.log("hello");`)); + parseModule( + "./bad.ts", + new TextEncoder().encode(`console.log("hello");`), + ); }, Error, "relative URL without a base", @@ -687,7 +690,10 @@ Deno.test({ await init(); assertThrows( () => { - parseModule("file:///a/test.md", new TextEncoder().encode(`# Some Markdown\n\n**bold**`)); + parseModule( + "file:///a/test.md", + new TextEncoder().encode(`# Some Markdown\n\n**bold**`), + ); }, Error, "The module's source code could not be parsed", From e561d09843bf1e78ae2802664a405abf52c579b4 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Wed, 31 Jan 2024 11:37:03 -0500 Subject: [PATCH 11/14] Fix --- Cargo.lock | 11 +++++++++++ js/mod.ts | 19 ++++++++++++++++++- lib/Cargo.toml | 1 + lib/lib.rs | 2 ++ 4 files changed, 32 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 47a7d6290..e7714625d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -143,6 +143,16 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "console_error_panic_hook" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06aeb73f470f66dcdbf7223caeebb85984942f22f1adb2a088cf9668146bbbc" +dependencies = [ + "cfg-if", + "wasm-bindgen", +] + [[package]] name = "cpufeatures" version = "0.2.11" @@ -265,6 +275,7 @@ name = "deno_graph_wasm" version = "0.0.0" dependencies = [ "anyhow", + "console_error_panic_hook", "deno_graph", "futures", "js-sys", diff --git a/js/mod.ts b/js/mod.ts index 38266446d..573dcd291 100644 --- a/js/mod.ts +++ b/js/mod.ts @@ -40,6 +40,8 @@ export type { TypesDependency, } from "./types.ts"; +const encoder = new TextEncoder(); + // note: keep this in line with deno_cache export type CacheSetting = "only" | "use" | "reload"; @@ -139,7 +141,22 @@ export async function createGraph( const { createGraph } = await wasm.instantiate(); return await createGraph( rootSpecifiers, - load, + async ( + specifier: string, + isDynamic: boolean, + cacheSetting: CacheSetting, + ) => { + const result = await load(specifier, isDynamic, cacheSetting); + if (result?.kind === "module") { + if (typeof result.content === "string") { + result.content = encoder.encode(result.content); + } + // need to convert to an array for serde_wasm_bindgen to work + // deno-lint-ignore no-explicit-any + (result as any).content = Array.from(result.content); + } + return result; + }, defaultJsxImportSource, jsxImportSourceModule, cacheInfo, diff --git a/lib/Cargo.toml b/lib/Cargo.toml index e7ab26da8..59ec66097 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -15,6 +15,7 @@ crate-type = ["cdylib"] [dependencies] anyhow = "1.0.43" +console_error_panic_hook = "0.1.7" deno_graph = { path = "../" } futures = "0.3.17" js-sys = "0.3.63" diff --git a/lib/lib.rs b/lib/lib.rs index b7c92c181..3742f285a 100644 --- a/lib/lib.rs +++ b/lib/lib.rs @@ -197,6 +197,7 @@ pub async fn js_create_graph( maybe_graph_kind: Option, maybe_imports: JsValue, ) -> Result { + console_error_panic_hook::set_once(); let roots_vec: Vec = serde_wasm_bindgen::from_value(roots) .map_err(|err| JsValue::from(js_sys::Error::new(&err.to_string())))?; let maybe_imports_map: Option>> = @@ -278,6 +279,7 @@ pub fn js_parse_module( maybe_resolve: Option, maybe_resolve_types: Option, ) -> Result { + console_error_panic_hook::set_once(); let maybe_headers: Option> = serde_wasm_bindgen::from_value(maybe_headers) .map_err(|err| js_sys::Error::new(&err.to_string()))?; From c89ac33ac7d56850e669fe235d6f09b9561e89ef Mon Sep 17 00:00:00 2001 From: David Sherret Date: Wed, 31 Jan 2024 11:46:17 -0500 Subject: [PATCH 12/14] Box --- src/graph.rs | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/graph.rs b/src/graph.rs index db7471a6c..600b670bf 100644 --- a/src/graph.rs +++ b/src/graph.rs @@ -1875,7 +1875,8 @@ pub(crate) fn parse_module( | MediaType::Dts | MediaType::Dmts | MediaType::Dcts => { - let source = new_source_with_text(specifier, content)?; + let source = + new_source_with_text(specifier, content).map_err(|err| *err)?; match module_analyzer.analyze(specifier, source.clone(), media_type) { Ok(module_info) => { // Return the module as a valid module @@ -1897,7 +1898,8 @@ pub(crate) fn parse_module( } } MediaType::Unknown if is_root => { - let source = new_source_with_text(specifier, content)?; + let source = + new_source_with_text(specifier, content).map_err(|err| *err)?; match module_analyzer.analyze( specifier, source.clone(), @@ -3146,7 +3148,7 @@ impl<'a, 'graph> Builder<'a, 'graph> { Ok(source) => { module.source = source; } - Err(err) => *slot = ModuleSlot::Err(err), + Err(err) => *slot = ModuleSlot::Err(*err), } } Module::Json(module) => { @@ -3154,7 +3156,7 @@ impl<'a, 'graph> Builder<'a, 'graph> { Ok(source) => { module.source = source; } - Err(err) => *slot = ModuleSlot::Err(err), + Err(err) => *slot = ModuleSlot::Err(*err), } } Module::Npm(_) @@ -4232,13 +4234,16 @@ impl<'a> NpmSpecifierResolver<'a> { } } -#[allow(clippy::result_large_err)] fn new_source_with_text( specifier: &ModuleSpecifier, text: Arc<[u8]>, -) -> Result, ModuleError> { +) -> Result, Box> { text_encoding::arc_bytes_to_text(text).map_err(|err| { - ModuleError::LoadingErr(specifier.clone(), None, Arc::new(err.into())) + Box::new(ModuleError::LoadingErr( + specifier.clone(), + None, + Arc::new(err.into()), + )) }) } From 928c84a421592b1771726e4739318ac80cd694bf Mon Sep 17 00:00:00 2001 From: David Sherret Date: Wed, 31 Jan 2024 13:45:27 -0500 Subject: [PATCH 13/14] Implement this properly --- src/graph.rs | 38 ++++++++++++++++----- src/source/mod.rs | 80 ++++++++++++++++++++++++++++++++++++++++++++ src/text_encoding.rs | 29 ---------------- 3 files changed, 109 insertions(+), 38 deletions(-) diff --git a/src/graph.rs b/src/graph.rs index 600b670bf..cad5ef75b 100644 --- a/src/graph.rs +++ b/src/graph.rs @@ -23,7 +23,6 @@ use crate::packages::JsrPackageInfo; use crate::packages::JsrPackageVersionInfo; use crate::packages::PackageSpecifiers; use crate::source::*; -use crate::text_encoding; use anyhow::anyhow; use deno_ast::dep::DependencyKind; @@ -852,6 +851,11 @@ impl JsModule { } } + /// Return the size in bytes of the content of the module. + pub fn size(&self) -> usize { + self.source.as_bytes().len() + } + pub fn fast_check_diagnostic(&self) -> Option<&FastCheckDiagnostic> { let module_slot = self.fast_check.as_ref()?; match module_slot { @@ -1821,8 +1825,8 @@ pub(crate) fn parse_module( is_dynamic_branch: bool, maybe_npm_resolver: Option<&dyn NpmResolver>, ) -> Result { - let media_type = - MediaType::from_specifier_and_headers(specifier, maybe_headers); + let (media_type, maybe_charset) = + resolve_media_type_and_charset_from_headers(specifier, maybe_headers); // here we check any media types that should have assertions made against them // if they aren't the root and add them to the graph, otherwise we continue @@ -1834,7 +1838,12 @@ pub(crate) fn parse_module( Some("json") )) { - let text = text_encoding::arc_bytes_to_text(content).map_err(|err| { + let text = crate::source::decode_source( + specifier, + content, + maybe_charset.as_deref(), + ) + .map_err(|err| { ModuleError::LoadingErr(specifier.clone(), None, Arc::new(err.into())) })?; return Ok(Module::Json(JsonModule { @@ -1876,7 +1885,8 @@ pub(crate) fn parse_module( | MediaType::Dmts | MediaType::Dcts => { let source = - new_source_with_text(specifier, content).map_err(|err| *err)?; + new_source_with_text(specifier, content, maybe_charset.as_deref()) + .map_err(|err| *err)?; match module_analyzer.analyze(specifier, source.clone(), media_type) { Ok(module_info) => { // Return the module as a valid module @@ -1899,7 +1909,8 @@ pub(crate) fn parse_module( } MediaType::Unknown if is_root => { let source = - new_source_with_text(specifier, content).map_err(|err| *err)?; + new_source_with_text(specifier, content, maybe_charset.as_deref()) + .map_err(|err| *err)?; match module_analyzer.analyze( specifier, source.clone(), @@ -3144,7 +3155,11 @@ impl<'a, 'graph> Builder<'a, 'graph> { ModuleSlot::Module(module) => { match module { Module::Js(module) => { - match new_source_with_text(&module.specifier, content) { + match new_source_with_text( + &module.specifier, + content, + None, // no charset for JSR + ) { Ok(source) => { module.source = source; } @@ -3152,7 +3167,11 @@ impl<'a, 'graph> Builder<'a, 'graph> { } } Module::Json(module) => { - match new_source_with_text(&module.specifier, content) { + match new_source_with_text( + &module.specifier, + content, + None, // no charset for JSR + ) { Ok(source) => { module.source = source; } @@ -4237,8 +4256,9 @@ impl<'a> NpmSpecifierResolver<'a> { fn new_source_with_text( specifier: &ModuleSpecifier, text: Arc<[u8]>, + maybe_charset: Option<&str>, ) -> Result, Box> { - text_encoding::arc_bytes_to_text(text).map_err(|err| { + crate::source::decode_source(specifier, text, maybe_charset).map_err(|err| { Box::new(ModuleError::LoadingErr( specifier.clone(), None, diff --git a/src/source/mod.rs b/src/source/mod.rs index 8bc8b9d7d..058320f44 100644 --- a/src/source/mod.rs +++ b/src/source/mod.rs @@ -4,8 +4,10 @@ use crate::graph::Range; use crate::module_specifier::resolve_import; use crate::packages::JsrPackageInfo; use crate::packages::JsrPackageVersionInfo; +use crate::text_encoding; use crate::ModuleInfo; use crate::SpecifierError; +use deno_ast::MediaType; use deno_semver::package::PackageNv; use deno_semver::package::PackageReq; @@ -18,6 +20,7 @@ use futures::future::LocalBoxFuture; use once_cell::sync::Lazy; use serde::Deserialize; use serde::Serialize; +use std::borrow::Cow; use std::collections::HashMap; use std::fmt; use std::path::PathBuf; @@ -477,6 +480,83 @@ pub trait Reporter: fmt::Debug { ); } +/// Resolve a media type and optionally the charset from a module specifier and +/// the value of a content type header. +pub fn resolve_media_type_and_charset_from_headers( + specifier: &ModuleSpecifier, + maybe_headers: Option<&HashMap>, +) -> (MediaType, Option) { + resolve_media_type_and_charset_from_content_type( + specifier, + maybe_headers.and_then(|h| h.get("content-type")), + ) +} + +/// Resolve a media type and optionally the charset from a module specifier and +/// the value of a content type header. +pub fn resolve_media_type_and_charset_from_content_type( + specifier: &ModuleSpecifier, + maybe_content_type: Option<&String>, +) -> (MediaType, Option) { + if let Some(content_type) = maybe_content_type { + let mut content_types = content_type.split(';'); + let content_type = content_types.next().unwrap(); + let media_type = MediaType::from_content_type(specifier, content_type); + let charset = content_types + .map(str::trim) + .find_map(|s| s.strip_prefix("charset=")) + .map(String::from); + + (media_type, charset) + } else { + (MediaType::from_specifier(specifier), None) + } +} + +/// Decodes the source bytes into a string handling any encoding rules +/// for local vs remote files and dealing with the charset. +pub fn decode_source( + specifier: &ModuleSpecifier, + bytes: Arc<[u8]>, + maybe_charset: Option<&str>, +) -> Result, std::io::Error> { + let charset = if specifier.scheme() == "file" { + text_encoding::detect_charset(bytes.as_ref()) + } else { + maybe_charset.unwrap_or("utf-8") + }; + decode_with_charset(bytes, charset) +} + +fn decode_with_charset( + bytes: Arc<[u8]>, + charset: &str, +) -> Result, std::io::Error> { + let text = match text_encoding::convert_to_utf8(bytes.as_ref(), charset)? { + Cow::Borrowed(text) => { + if text.starts_with(text_encoding::BOM_CHAR) { + text[..text_encoding::BOM_CHAR.len_utf8()].to_string() + } else { + return Ok( + // SAFETY: we know it's a valid utf-8 string at this point + unsafe { + let raw_ptr = Arc::into_raw(bytes); + Arc::from_raw(std::mem::transmute::<*const [u8], *const str>( + raw_ptr, + )) + }, + ); + } + } + Cow::Owned(mut text) => { + text_encoding::strip_bom_mut(&mut text); + text + } + }; + let text: Arc = Arc::from(text); + Ok(text) +} + #[cfg(test)] pub mod tests { use super::*; diff --git a/src/text_encoding.rs b/src/text_encoding.rs index ce4205103..b1433c6a6 100644 --- a/src/text_encoding.rs +++ b/src/text_encoding.rs @@ -1,5 +1,4 @@ use std::borrow::Cow; -use std::sync::Arc; pub const BOM_CHAR: char = '\u{FEFF}'; @@ -47,34 +46,6 @@ pub fn strip_bom_mut(text: &mut String) { } } -/// Converts an `Arc<[u8]>` to an `Arc`. -pub fn arc_bytes_to_text(bytes: Arc<[u8]>) -> Result, std::io::Error> { - let charset = detect_charset(bytes.as_ref()); - let text = match convert_to_utf8(bytes.as_ref(), charset)? { - Cow::Borrowed(text) => { - if text.starts_with(BOM_CHAR) { - text[..BOM_CHAR.len_utf8()].to_string() - } else { - return Ok( - // SAFETY: we know it's avalid utf-8 string at this point - unsafe { - let raw_ptr = Arc::into_raw(bytes); - Arc::from_raw(std::mem::transmute::<*const [u8], *const str>( - raw_ptr, - )) - }, - ); - } - } - Cow::Owned(mut text) => { - strip_bom_mut(&mut text); - text - } - }; - let text: Arc = Arc::from(text); - Ok(text) -} - #[cfg(test)] mod test { use super::*; From 1224b2d945dd27300a90641d681c57cd1221396b Mon Sep 17 00:00:00 2001 From: David Sherret Date: Wed, 31 Jan 2024 14:51:26 -0500 Subject: [PATCH 14/14] Move down test from CLI --- src/graph.rs | 22 ++--- src/source/mod.rs | 215 +++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 210 insertions(+), 27 deletions(-) diff --git a/src/graph.rs b/src/graph.rs index cad5ef75b..459eecfe7 100644 --- a/src/graph.rs +++ b/src/graph.rs @@ -1838,14 +1838,10 @@ pub(crate) fn parse_module( Some("json") )) { - let text = crate::source::decode_source( - specifier, - content, - maybe_charset.as_deref(), - ) - .map_err(|err| { - ModuleError::LoadingErr(specifier.clone(), None, Arc::new(err.into())) - })?; + let text = crate::source::decode_source(specifier, content, maybe_charset) + .map_err(|err| { + ModuleError::LoadingErr(specifier.clone(), None, Arc::new(err.into())) + })?; return Ok(Module::Json(JsonModule { maybe_cache_info: None, source: text, @@ -1884,9 +1880,8 @@ pub(crate) fn parse_module( | MediaType::Dts | MediaType::Dmts | MediaType::Dcts => { - let source = - new_source_with_text(specifier, content, maybe_charset.as_deref()) - .map_err(|err| *err)?; + let source = new_source_with_text(specifier, content, maybe_charset) + .map_err(|err| *err)?; match module_analyzer.analyze(specifier, source.clone(), media_type) { Ok(module_info) => { // Return the module as a valid module @@ -1908,9 +1903,8 @@ pub(crate) fn parse_module( } } MediaType::Unknown if is_root => { - let source = - new_source_with_text(specifier, content, maybe_charset.as_deref()) - .map_err(|err| *err)?; + let source = new_source_with_text(specifier, content, maybe_charset) + .map_err(|err| *err)?; match module_analyzer.analyze( specifier, source.clone(), diff --git a/src/source/mod.rs b/src/source/mod.rs index 058320f44..10fdc22a2 100644 --- a/src/source/mod.rs +++ b/src/source/mod.rs @@ -482,10 +482,10 @@ pub trait Reporter: fmt::Debug { /// Resolve a media type and optionally the charset from a module specifier and /// the value of a content type header. -pub fn resolve_media_type_and_charset_from_headers( +pub fn resolve_media_type_and_charset_from_headers<'a>( specifier: &ModuleSpecifier, - maybe_headers: Option<&HashMap>, -) -> (MediaType, Option) { + maybe_headers: Option<&'a HashMap>, +) -> (MediaType, Option<&'a str>) { resolve_media_type_and_charset_from_content_type( specifier, maybe_headers.and_then(|h| h.get("content-type")), @@ -494,18 +494,17 @@ pub fn resolve_media_type_and_charset_from_headers( /// Resolve a media type and optionally the charset from a module specifier and /// the value of a content type header. -pub fn resolve_media_type_and_charset_from_content_type( +pub fn resolve_media_type_and_charset_from_content_type<'a>( specifier: &ModuleSpecifier, - maybe_content_type: Option<&String>, -) -> (MediaType, Option) { + maybe_content_type: Option<&'a String>, +) -> (MediaType, Option<&'a str>) { if let Some(content_type) = maybe_content_type { let mut content_types = content_type.split(';'); let content_type = content_types.next().unwrap(); let media_type = MediaType::from_content_type(specifier, content_type); let charset = content_types .map(str::trim) - .find_map(|s| s.strip_prefix("charset=")) - .map(String::from); + .find_map(|s| s.strip_prefix("charset=")); (media_type, charset) } else { @@ -520,11 +519,13 @@ pub fn decode_source( bytes: Arc<[u8]>, maybe_charset: Option<&str>, ) -> Result, std::io::Error> { - let charset = if specifier.scheme() == "file" { - text_encoding::detect_charset(bytes.as_ref()) - } else { - maybe_charset.unwrap_or("utf-8") - }; + let charset = maybe_charset.unwrap_or_else(|| { + if specifier.scheme() == "file" { + text_encoding::detect_charset(bytes.as_ref()) + } else { + "utf-8" + } + }); decode_with_charset(bytes, charset) } @@ -639,4 +640,192 @@ pub mod tests { } ); } + + macro_rules! file_url { + ($path:expr) => { + if cfg!(target_os = "windows") { + concat!("file:///C:", $path) + } else { + concat!("file://", $path) + } + }; + } + + #[test] + fn test_resolve_media_type_and_charset_from_content_type() { + let fixtures = vec![ + // Extension only + (file_url!("/foo/bar.ts"), None, MediaType::TypeScript, None), + (file_url!("/foo/bar.tsx"), None, MediaType::Tsx, None), + (file_url!("/foo/bar.d.cts"), None, MediaType::Dcts, None), + (file_url!("/foo/bar.d.mts"), None, MediaType::Dmts, None), + (file_url!("/foo/bar.d.ts"), None, MediaType::Dts, None), + (file_url!("/foo/bar.js"), None, MediaType::JavaScript, None), + (file_url!("/foo/bar.jsx"), None, MediaType::Jsx, None), + (file_url!("/foo/bar.json"), None, MediaType::Json, None), + (file_url!("/foo/bar.wasm"), None, MediaType::Wasm, None), + (file_url!("/foo/bar.cjs"), None, MediaType::Cjs, None), + (file_url!("/foo/bar.mjs"), None, MediaType::Mjs, None), + (file_url!("/foo/bar.cts"), None, MediaType::Cts, None), + (file_url!("/foo/bar.mts"), None, MediaType::Mts, None), + (file_url!("/foo/bar"), None, MediaType::Unknown, None), + // Media type no extension + ( + "https://deno.land/x/mod", + Some("application/typescript".to_string()), + MediaType::TypeScript, + None, + ), + ( + "https://deno.land/x/mod", + Some("text/typescript".to_string()), + MediaType::TypeScript, + None, + ), + ( + "https://deno.land/x/mod", + Some("video/vnd.dlna.mpeg-tts".to_string()), + MediaType::TypeScript, + None, + ), + ( + "https://deno.land/x/mod", + Some("video/mp2t".to_string()), + MediaType::TypeScript, + None, + ), + ( + "https://deno.land/x/mod", + Some("application/x-typescript".to_string()), + MediaType::TypeScript, + None, + ), + ( + "https://deno.land/x/mod", + Some("application/javascript".to_string()), + MediaType::JavaScript, + None, + ), + ( + "https://deno.land/x/mod", + Some("text/javascript".to_string()), + MediaType::JavaScript, + None, + ), + ( + "https://deno.land/x/mod", + Some("application/ecmascript".to_string()), + MediaType::JavaScript, + None, + ), + ( + "https://deno.land/x/mod", + Some("text/ecmascript".to_string()), + MediaType::JavaScript, + None, + ), + ( + "https://deno.land/x/mod", + Some("application/x-javascript".to_string()), + MediaType::JavaScript, + None, + ), + ( + "https://deno.land/x/mod", + Some("application/node".to_string()), + MediaType::JavaScript, + None, + ), + ( + "https://deno.land/x/mod", + Some("text/jsx".to_string()), + MediaType::Jsx, + None, + ), + ( + "https://deno.land/x/mod", + Some("text/tsx".to_string()), + MediaType::Tsx, + None, + ), + ( + "https://deno.land/x/mod", + Some("text/json".to_string()), + MediaType::Json, + None, + ), + ( + "https://deno.land/x/mod", + Some("text/json; charset=utf-8".to_string()), + MediaType::Json, + Some("utf-8".to_string()), + ), + // Extension with media type + ( + "https://deno.land/x/mod.ts", + Some("text/plain".to_string()), + MediaType::TypeScript, + None, + ), + ( + "https://deno.land/x/mod.ts", + Some("foo/bar".to_string()), + MediaType::Unknown, + None, + ), + ( + "https://deno.land/x/mod.tsx", + Some("application/typescript".to_string()), + MediaType::Tsx, + None, + ), + ( + "https://deno.land/x/mod.tsx", + Some("application/javascript".to_string()), + MediaType::Tsx, + None, + ), + ( + "https://deno.land/x/mod.jsx", + Some("application/javascript".to_string()), + MediaType::Jsx, + None, + ), + ( + "https://deno.land/x/mod.jsx", + Some("application/x-typescript".to_string()), + MediaType::Jsx, + None, + ), + ( + "https://deno.land/x/mod.d.ts", + Some("application/javascript".to_string()), + MediaType::Dts, + None, + ), + ( + "https://deno.land/x/mod.d.ts", + Some("text/plain".to_string()), + MediaType::Dts, + None, + ), + ( + "https://deno.land/x/mod.d.ts", + Some("application/x-typescript".to_string()), + MediaType::Dts, + None, + ), + ]; + + for (specifier, maybe_content_type, media_type, maybe_charset) in fixtures { + let specifier = ModuleSpecifier::parse(specifier).unwrap(); + assert_eq!( + resolve_media_type_and_charset_from_content_type( + &specifier, + maybe_content_type.as_ref() + ), + (media_type, maybe_charset.as_deref()) + ); + } + } }