From f1c243e1f3cc85540dfc0e6fd706be5af6a20f2a Mon Sep 17 00:00:00 2001 From: Kitson Kelly Date: Mon, 14 Sep 2020 10:01:15 +1000 Subject: [PATCH 1/2] refactor: use ParsedModule and improve MediaTypes enum --- cli/ast.rs | 654 ++++++++++++++++++++++++++++++++++++ cli/errors.rs | 8 +- cli/file_fetcher.rs | 65 +--- cli/global_state.rs | 2 +- cli/info.rs | 2 +- cli/lint.rs | 9 +- cli/main.rs | 13 +- cli/module_graph.rs | 16 +- cli/msg.rs | 125 ++++++- cli/swc_util.rs | 445 ------------------------ cli/tsc.rs | 40 +-- cli/tsc/99_main_compiler.js | 18 +- 12 files changed, 820 insertions(+), 577 deletions(-) create mode 100644 cli/ast.rs delete mode 100644 cli/swc_util.rs diff --git a/cli/ast.rs b/cli/ast.rs new file mode 100644 index 00000000000000..bb25d5e9ec17fe --- /dev/null +++ b/cli/ast.rs @@ -0,0 +1,654 @@ +// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. + +use crate::msg::MediaType; + +use deno_core::ErrBox; +use deno_core::ModuleSpecifier; +use std::error::Error; +use std::fmt; +use std::rc::Rc; +use std::result; +use std::sync::Arc; +use std::sync::RwLock; +use swc_common::chain; +use swc_common::comments::Comment; +use swc_common::comments::SingleThreadedComments; +use swc_common::errors::Diagnostic; +use swc_common::errors::DiagnosticBuilder; +use swc_common::errors::Emitter; +use swc_common::errors::Handler; +use swc_common::errors::HandlerFlags; +use swc_common::FileName; +use swc_common::Globals; +use swc_common::Loc; +use swc_common::SourceMap; +use swc_common::Span; +use swc_ecmascript::ast::Module; +use swc_ecmascript::ast::Program; +use swc_ecmascript::codegen::text_writer::JsWriter; +use swc_ecmascript::codegen::Node; +use swc_ecmascript::dep_graph::analyze_dependencies; +use swc_ecmascript::dep_graph::DependencyDescriptor; +use swc_ecmascript::parser::lexer::Lexer; +use swc_ecmascript::parser::EsConfig; +use swc_ecmascript::parser::JscTarget; +use swc_ecmascript::parser::StringInput; +use swc_ecmascript::parser::Syntax; +use swc_ecmascript::parser::TsConfig; +use swc_ecmascript::transforms::fixer; +use swc_ecmascript::transforms::helpers; +use swc_ecmascript::transforms::pass::Optional; +use swc_ecmascript::transforms::proposals::decorators; +use swc_ecmascript::transforms::react; +use swc_ecmascript::transforms::typescript; +use swc_ecmascript::visit; +use swc_ecmascript::visit::FoldWith; +use swc_ecmascript::visit::Visit; + +type Result = result::Result; + +static TARGET: JscTarget = JscTarget::Es2020; + +#[derive(Debug, Clone, PartialEq)] +pub struct Location { + pub filename: String, + pub line: usize, + pub col: usize, +} + +impl Into for swc_common::Loc { + fn into(self) -> Location { + use swc_common::FileName::*; + + let filename = match &self.file.name { + Real(path_buf) => path_buf.to_string_lossy().to_string(), + Custom(str_) => str_.to_string(), + _ => panic!("invalid filename"), + }; + + Location { + filename, + line: self.line, + col: self.col_display, + } + } +} + +/// A buffer for collecting diagnostic messages from the AST parser. +#[derive(Debug)] +pub struct DiagnosticBuffer(Vec); + +impl Error for DiagnosticBuffer {} + +impl fmt::Display for DiagnosticBuffer { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let s = self.0.join(","); + f.pad(&s) + } +} + +impl DiagnosticBuffer { + pub fn from_error_buffer(error_buffer: ErrorBuffer, get_loc: F) -> Self + where + F: Fn(Span) -> Loc, + { + let s = error_buffer.0.read().unwrap().clone(); + let diagnostics = s + .iter() + .map(|d| { + let mut msg = d.message(); + + if let Some(span) = d.span.primary_span() { + let loc = get_loc(span); + let file_name = match &loc.file.name { + FileName::Custom(n) => n, + _ => unreachable!(), + }; + msg = format!( + "{} at {}:{}:{}", + msg, file_name, loc.line, loc.col_display + ); + } + + msg + }) + .collect::>(); + + Self(diagnostics) + } +} + +/// A buffer for collecting errors from the AST parser. +#[derive(Debug, Clone)] +pub struct ErrorBuffer(Arc>>); + +impl ErrorBuffer { + pub fn new() -> Self { + Self(Arc::new(RwLock::new(Vec::new()))) + } +} + +impl Emitter for ErrorBuffer { + fn emit(&mut self, db: &DiagnosticBuilder) { + self.0.write().unwrap().push((**db).clone()); + } +} + +fn get_es_config(jsx: bool) -> EsConfig { + EsConfig { + class_private_methods: true, + class_private_props: true, + class_props: true, + dynamic_import: true, + export_default_from: true, + export_namespace_from: true, + import_meta: true, + jsx, + nullish_coalescing: true, + num_sep: true, + optional_chaining: true, + top_level_await: true, + ..EsConfig::default() + } +} + +fn get_ts_config(tsx: bool, dts: bool) -> TsConfig { + TsConfig { + decorators: true, + dts, + dynamic_import: true, + tsx, + ..TsConfig::default() + } +} + +pub fn get_syntax(media_type: &MediaType) -> Syntax { + match media_type { + MediaType::JavaScript => Syntax::Es(get_es_config(false)), + MediaType::JSX => Syntax::Es(get_es_config(true)), + MediaType::TypeScript => Syntax::Typescript(get_ts_config(false, false)), + MediaType::Dts => Syntax::Typescript(get_ts_config(false, true)), + MediaType::TSX => Syntax::Typescript(get_ts_config(true, false)), + _ => Syntax::Es(get_es_config(false)), + } +} + +/// Visits a pattern node, recursively looking for any names that end up in the +/// local scope, pushing them onto the passed vector. +fn visit_pat(pat: &swc_ecmascript::ast::Pat, names: &mut Vec) { + match pat { + swc_ecmascript::ast::Pat::Ident(ident) => names.push(ident.sym.to_string()), + swc_ecmascript::ast::Pat::Array(array_pat) => { + for elem in array_pat.elems.iter() { + if let Some(pat) = elem { + visit_pat(pat, names); + } + } + } + swc_ecmascript::ast::Pat::Rest(rest_pat) => { + visit_pat(rest_pat.arg.as_ref(), names) + } + swc_ecmascript::ast::Pat::Object(object_pat) => { + for prop in object_pat.props.iter() { + match prop { + swc_ecmascript::ast::ObjectPatProp::Assign(assign_pat) => { + names.push(assign_pat.key.sym.to_string()) + } + swc_ecmascript::ast::ObjectPatProp::KeyValue(key_value) => { + visit_pat(key_value.value.as_ref(), names) + } + swc_ecmascript::ast::ObjectPatProp::Rest(rest_pat) => { + visit_pat(rest_pat.arg.as_ref(), names) + } + } + } + } + swc_ecmascript::ast::Pat::Assign(assign_pat) => { + visit_pat(assign_pat.left.as_ref(), names) + } + // Invalid and Expressions are noops + _ => {} + } +} + +/// A structure for collecting the named exports from a module. +#[derive(Default)] +struct ExportCollector { + pub names: Vec, + pub export_all_specifiers: Vec, +} + +impl Visit for ExportCollector { + fn visit_export_decl( + &mut self, + node: &swc_ecmascript::ast::ExportDecl, + _parent: &dyn visit::Node, + ) { + match &node.decl { + swc_ecmascript::ast::Decl::Class(class_decl) => { + self.names.push(class_decl.ident.sym.to_string()); + } + swc_ecmascript::ast::Decl::Fn(fn_decl) => { + self.names.push(fn_decl.ident.sym.to_string()); + } + swc_ecmascript::ast::Decl::Var(var_decl) => { + for decl in var_decl.decls.iter() { + visit_pat(&decl.name, &mut self.names); + } + } + swc_ecmascript::ast::Decl::TsEnum(ts_enum_decl) => { + self.names.push(ts_enum_decl.id.sym.to_string()); + } + // Interfaces, Type Aliases, and TS Module/Namespace decl are noops + _ => {} + } + } + + fn visit_named_export( + &mut self, + node: &swc_ecmascript::ast::NamedExport, + _parent: &dyn visit::Node, + ) { + for spec in node.specifiers.iter() { + match spec { + swc_ecmascript::ast::ExportSpecifier::Named(named_spec) => { + if let Some(ident) = &named_spec.exported { + self.names.push(ident.sym.to_string()); + } else { + self.names.push(named_spec.orig.sym.to_string()); + } + } + swc_ecmascript::ast::ExportSpecifier::Namespace(namespace_spec) => { + self.names.push(namespace_spec.name.sym.to_string()); + } + // Default is only proposed syntax, not current supported, so noop + _ => {} + } + } + } + + fn visit_export_default_decl( + &mut self, + node: &swc_ecmascript::ast::ExportDefaultDecl, + _parent: &dyn visit::Node, + ) { + match &node.decl { + swc_ecmascript::ast::DefaultDecl::Class(_) => { + self.names.push("default".to_string()) + } + swc_ecmascript::ast::DefaultDecl::Fn(_) => { + self.names.push("default".to_string()) + } + // Interface is a noop + _ => {} + } + } + + fn visit_export_default_expr( + &mut self, + _node: &swc_ecmascript::ast::ExportDefaultExpr, + _parent: &dyn visit::Node, + ) { + self.names.push("default".to_string()); + } + + fn visit_export_all( + &mut self, + node: &swc_ecmascript::ast::ExportAll, + _parent: &dyn visit::Node, + ) { + self.export_all_specifiers.push(node.src.value.to_string()); + } +} + +/// Options which can be adjusted when transpiling a module. +#[derive(Debug, Clone)] +pub struct TranspileOptions { + /// When emitting a legacy decorator, also emit experimental decorator meta + /// data. Defaults to `false`. + pub emit_metadata: bool, + /// Should the source map be inlined in the emitted code file, or provided + /// as a separate file. Defaults to `true`. + pub inline_source_map: bool, + /// When transforming JSX, what value should be used for the JSX factory. + /// Defaults to `React.createElement`. + pub jsx_factory: String, + /// When transforming JSX, what value should be used for the JSX fragment + /// factory. Defaults to `React.Fragment`. + pub jsx_fragment_factory: String, + /// Should JSX be transformed or preserved. Defaults to `true`. + pub transform_jsx: bool, +} + +impl Default for TranspileOptions { + fn default() -> Self { + TranspileOptions { + emit_metadata: false, + inline_source_map: true, + jsx_factory: "React.createElement".into(), + jsx_fragment_factory: "React.Fragment".into(), + transform_jsx: true, + } + } +} + +/// A logical structure to hold the value of a parsed module for further +/// processing. +pub struct ParsedModule { + comments: SingleThreadedComments, + pub leading_comments: Vec, + module: Module, + pub source_map: Rc, +} + +impl ParsedModule { + pub fn analyze_dependencies(&self) -> Vec { + analyze_dependencies(&self.module, &self.source_map, &self.comments) + } + + #[allow(dead_code)] // TODO(kitsonk) for bundling rewrite + pub fn analyze_exported_names(&self) -> (Vec, Vec) { + let mut collector = ExportCollector::default(); + collector.visit_module(&self.module, &self.module); + + (collector.names, collector.export_all_specifiers) + } + + pub fn transpile( + self, + options: &TranspileOptions, + ) -> Result<(String, Option)> { + let program = Program::Module(self.module); + + let jsx_pass = react::react( + self.source_map.clone(), + Some(&self.comments), + react::Options { + pragma: options.jsx_factory.clone(), + pragma_frag: options.jsx_fragment_factory.clone(), + // this will use `Object.assign()` instead of the `_extends` helper + // when spreading props. + use_builtins: true, + ..Default::default() + }, + ); + let mut passes = chain!( + Optional::new(jsx_pass, options.transform_jsx), + decorators::decorators(decorators::Config { + legacy: true, + emit_metadata: options.emit_metadata + }), + typescript::strip(), + fixer(Some(&self.comments)), + ); + + let program = swc_common::GLOBALS.set(&Globals::new(), || { + helpers::HELPERS.set(&helpers::Helpers::new(false), || { + program.fold_with(&mut passes) + }) + }); + + let mut src_map_buf = vec![]; + let mut buf = vec![]; + { + let writer = Box::new(JsWriter::new( + self.source_map.clone(), + "\n", + &mut buf, + Some(&mut src_map_buf), + )); + let config = swc_ecmascript::codegen::Config { minify: false }; + let mut emitter = swc_ecmascript::codegen::Emitter { + cfg: config, + comments: Some(&self.comments), + cm: self.source_map.clone(), + wr: writer, + }; + program.emit_with(&mut emitter)?; + } + let mut src = String::from_utf8(buf)?; + let mut map: Option = None; + { + let mut buf = Vec::new(); + self + .source_map + .build_source_map_from(&mut src_map_buf, None) + .to_writer(&mut buf)?; + + if options.inline_source_map { + src.push_str("//# sourceMappingURL=data:application/json;base64,"); + let encoded_map = base64::encode(buf); + src.push_str(&encoded_map); + } else { + map = Some(String::from_utf8(buf)?); + } + } + Ok((src, map)) + } +} + +/// For a given specifier, source, and media type, parse the source of the +/// module and return a representation which can be further processed. +/// +/// # Arguments +/// +/// - `specifier` - The module specifier for the module. +/// - `source` - The source code for the module. +/// - `media_type` - The media type for the module. +/// +pub fn parse( + specifier: &ModuleSpecifier, + source: &str, + media_type: &MediaType, +) -> Result { + let source_map = SourceMap::default(); + let source_file = source_map.new_source_file( + FileName::Custom(specifier.to_string()), + source.to_string(), + ); + let error_buffer = ErrorBuffer::new(); + let syntax = get_syntax(media_type); + let input = StringInput::from(&*source_file); + let comments = SingleThreadedComments::default(); + + let handler = Handler::with_emitter_and_flags( + Box::new(error_buffer.clone()), + HandlerFlags { + can_emit_warnings: true, + dont_buffer_diagnostics: true, + ..HandlerFlags::default() + }, + ); + + let lexer = Lexer::new(syntax, TARGET, input, Some(&comments)); + let mut parser = swc_ecmascript::parser::Parser::new_from(lexer); + + let sm = &source_map; + let module = parser.parse_module().map_err(move |err| { + let mut diagnostic = err.into_diagnostic(&handler); + diagnostic.emit(); + + ErrBox::from(DiagnosticBuffer::from_error_buffer(error_buffer, |span| { + sm.lookup_char_pos(span.lo) + })) + })?; + let leading_comments = + comments.with_leading(module.span.lo, |comments| comments.to_vec()); + + Ok(ParsedModule { + leading_comments, + module, + source_map: Rc::new(source_map), + comments, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use swc_ecmascript::dep_graph::DependencyKind; + + #[test] + fn test_parsed_module_analyze_dependencies() { + let specifier = + ModuleSpecifier::resolve_url_or_path("https://deno.land/x/mod.js") + .unwrap(); + let source = r#"import * as bar from "./test.ts"; + const foo = await import("./foo.ts"); + "#; + let parsed_module = parse(&specifier, source, &MediaType::JavaScript) + .expect("could not parse module"); + let actual = parsed_module.analyze_dependencies(); + assert_eq!( + actual, + vec![ + DependencyDescriptor { + kind: DependencyKind::Import, + is_dynamic: false, + leading_comments: Vec::new(), + col: 0, + line: 1, + specifier: "./test.ts".into() + }, + DependencyDescriptor { + kind: DependencyKind::Import, + is_dynamic: true, + leading_comments: Vec::new(), + col: 22, + line: 2, + specifier: "./foo.ts".into() + } + ] + ); + } + + #[test] + fn test_parsed_module_analyze_exported_names() { + let specifier = + ModuleSpecifier::resolve_url_or_path("https://deno.land/x/mod.ts") + .unwrap(); + let source = r#" + export * from "./a.ts"; + + export { a, b as c } from "./b.ts"; + + export default function () { + console.log("hello"); + } + + export enum C { + A, + B, + C, + } + + export const [d, e, ...f] = [1, 2, 3, 4, 5]; + + export const g = 1; + + export const { h, i: j, ...k } = { h: true, i: false, j: 1, k: 2 }; + + export class A {} + + export function l() {} + + export * as m from "./m.ts"; + "#; + let parsed_module = parse(&specifier, source, &MediaType::TypeScript) + .expect("could not parse module"); + let (names, export_all_specifiers) = parsed_module.analyze_exported_names(); + assert_eq!( + names, + vec![ + "a", "c", "default", "C", "d", "e", "f", "g", "h", "j", "k", "A", "l", + "m" + ] + ); + assert_eq!(export_all_specifiers, vec!["./a.ts"]); + } + + #[test] + fn test_transpile() { + let specifier = + ModuleSpecifier::resolve_url_or_path("https://deno.land/x/mod.ts") + .expect("could not resolve specifier"); + let source = r#" + enum D { + A, + B, + C, + } + + export class A { + private b: string; + protected c: number = 1; + e: "foo"; + constructor (public d = D.A) { + const e = "foo" as const; + this.e = e; + } + } + "#; + let module = parse(&specifier, source, &MediaType::TypeScript) + .expect("could not parse module"); + let (code, maybe_map) = module + .transpile(&TranspileOptions::default()) + .expect("could not strip types"); + assert!(code.starts_with("var D;\n(function(D) {\n")); + assert!( + code.contains("\n//# sourceMappingURL=data:application/json;base64,") + ); + assert!(maybe_map.is_none()); + } + + #[test] + fn test_transpile_tsx() { + let specifier = + ModuleSpecifier::resolve_url_or_path("https://deno.land/x/mod.ts") + .expect("could not resolve specifier"); + let source = r#" + export class A { + render() { + return
+ } + } + "#; + let module = parse(&specifier, source, &MediaType::TSX) + .expect("could not parse module"); + let (code, _) = module + .transpile(&TranspileOptions::default()) + .expect("could not strip types"); + assert!(code.contains("React.createElement(\"div\", null")); + } + + #[test] + fn test_transpile_decorators() { + let specifier = + ModuleSpecifier::resolve_url_or_path("https://deno.land/x/mod.ts") + .expect("could not resolve specifier"); + let source = r#" + function enumerable(value: boolean) { + return function ( + _target: any, + _propertyKey: string, + descriptor: PropertyDescriptor, + ) { + descriptor.enumerable = value; + }; + } + + export class A { + @enumerable(false) + a() { + Test.value; + } + } + "#; + let module = parse(&specifier, source, &MediaType::TypeScript) + .expect("could not parse module"); + let (code, _) = module + .transpile(&TranspileOptions::default()) + .expect("could not strip types"); + assert!(code.contains("_applyDecoratedDescriptor(")); + } +} diff --git a/cli/errors.rs b/cli/errors.rs index 041e8703c76c2f..31e6929a1f3abf 100644 --- a/cli/errors.rs +++ b/cli/errors.rs @@ -10,8 +10,8 @@ //! But Diagnostics are compile-time type errors, whereas JsErrors are runtime //! exceptions. +use crate::ast::DiagnosticBuffer; use crate::import_map::ImportMapError; -use crate::swc_util::SwcDiagnosticBuffer; use deno_core::ErrBox; use deno_core::ModuleResolutionError; use rustyline::error::ReadlineError; @@ -144,7 +144,7 @@ fn get_serde_json_error_class( } } -fn get_swc_diagnostic_class(_: &SwcDiagnosticBuffer) -> &'static str { +fn get_diagnostic_class(_: &DiagnosticBuffer) -> &'static str { "SyntaxError" } @@ -211,8 +211,8 @@ pub(crate) fn get_error_class_name(e: &ErrBox) -> &'static str { .map(get_serde_json_error_class) }) .or_else(|| { - e.downcast_ref::() - .map(get_swc_diagnostic_class) + e.downcast_ref::() + .map(get_diagnostic_class) }) .or_else(|| { e.downcast_ref::() diff --git a/cli/file_fetcher.rs b/cli/file_fetcher.rs index 8b3ca46a007b49..7a885d7b647309 100644 --- a/cli/file_fetcher.rs +++ b/cli/file_fetcher.rs @@ -552,23 +552,6 @@ impl SourceFileFetcher { } } -pub fn map_file_extension(path: &Path) -> msg::MediaType { - match path.extension() { - None => msg::MediaType::Unknown, - Some(os_str) => match os_str.to_str() { - Some("ts") => msg::MediaType::TypeScript, - Some("tsx") => msg::MediaType::TSX, - Some("js") => msg::MediaType::JavaScript, - Some("jsx") => msg::MediaType::JSX, - Some("mjs") => msg::MediaType::JavaScript, - Some("cjs") => msg::MediaType::JavaScript, - Some("json") => msg::MediaType::Json, - Some("wasm") => msg::MediaType::Wasm, - _ => msg::MediaType::Unknown, - }, - } -} - // convert a ContentType string into a enumerated MediaType + optional charset fn map_content_type( path: &Path, @@ -600,7 +583,7 @@ fn map_content_type( "application/json" | "text/json" => msg::MediaType::Json, "application/wasm" => msg::MediaType::Wasm, // Handle plain and possibly webassembly - "text/plain" | "application/octet-stream" => map_file_extension(path), + "text/plain" | "application/octet-stream" => msg::MediaType::from(path), _ => { debug!("unknown content type: {}", content_type); msg::MediaType::Unknown @@ -614,7 +597,7 @@ fn map_content_type( (media_type, charset) } - None => (map_file_extension(path), None), + None => (msg::MediaType::from(path), None), } } @@ -1603,50 +1586,6 @@ mod tests { .await; } - #[test] - fn test_map_file_extension() { - assert_eq!( - map_file_extension(Path::new("foo/bar.ts")), - msg::MediaType::TypeScript - ); - assert_eq!( - map_file_extension(Path::new("foo/bar.tsx")), - msg::MediaType::TSX - ); - assert_eq!( - map_file_extension(Path::new("foo/bar.d.ts")), - msg::MediaType::TypeScript - ); - assert_eq!( - map_file_extension(Path::new("foo/bar.js")), - msg::MediaType::JavaScript - ); - assert_eq!( - map_file_extension(Path::new("foo/bar.jsx")), - msg::MediaType::JSX - ); - assert_eq!( - map_file_extension(Path::new("foo/bar.json")), - msg::MediaType::Json - ); - assert_eq!( - map_file_extension(Path::new("foo/bar.wasm")), - msg::MediaType::Wasm - ); - assert_eq!( - map_file_extension(Path::new("foo/bar.cjs")), - msg::MediaType::JavaScript - ); - assert_eq!( - map_file_extension(Path::new("foo/bar.txt")), - msg::MediaType::Unknown - ); - assert_eq!( - map_file_extension(Path::new("foo/bar")), - msg::MediaType::Unknown - ); - } - #[test] fn test_map_content_type_extension_only() { // Extension only diff --git a/cli/global_state.rs b/cli/global_state.rs index 2dfec4a72274ae..77f48690497586 100644 --- a/cli/global_state.rs +++ b/cli/global_state.rs @@ -314,8 +314,8 @@ fn thread_safe() { #[test] fn test_should_allow_js() { + use crate::ast::Location; use crate::module_graph::ImportDescriptor; - use crate::swc_util::Location; assert!(should_allow_js(&[ &ModuleGraphFile { diff --git a/cli/info.rs b/cli/info.rs index c876c57d5b1eb2..f997fbdea2b39c 100644 --- a/cli/info.rs +++ b/cli/info.rs @@ -349,8 +349,8 @@ pub fn human_size(bytse: f64) -> String { #[cfg(test)] mod test { use super::*; + use crate::ast::Location; use crate::module_graph::ImportDescriptor; - use crate::swc_util::Location; use crate::MediaType; #[test] diff --git a/cli/lint.rs b/cli/lint.rs index c52aff40883cb0..3ff13f020dcda4 100644 --- a/cli/lint.rs +++ b/cli/lint.rs @@ -6,13 +6,12 @@ //! At the moment it is only consumed using CLI but in //! the future it can be easily extended to provide //! the same functions as ops available in JS runtime. +use crate::ast; use crate::colors; -use crate::file_fetcher::map_file_extension; use crate::fmt::collect_files; use crate::fmt::run_parallelized; use crate::fmt_errors; use crate::msg; -use crate::swc_util; use deno_core::ErrBox; use deno_lint::diagnostic::LintDiagnostic; use deno_lint::linter::Linter; @@ -131,8 +130,8 @@ fn lint_file( ) -> Result<(Vec, String), ErrBox> { let file_name = file_path.to_string_lossy().to_string(); let source_code = fs::read_to_string(&file_path)?; - let media_type = map_file_extension(&file_path); - let syntax = swc_util::get_syntax_for_media_type(media_type); + let media_type = msg::MediaType::from(&file_path); + let syntax = ast::get_syntax(&media_type); let lint_rules = rules::get_recommended_rules(); let mut linter = create_linter(syntax, lint_rules); @@ -158,7 +157,7 @@ fn lint_stdin(json: bool) -> Result<(), ErrBox> { }; let mut reporter = create_reporter(reporter_kind); let lint_rules = rules::get_recommended_rules(); - let syntax = swc_util::get_syntax_for_media_type(msg::MediaType::TypeScript); + let syntax = ast::get_syntax(&msg::MediaType::TypeScript); let mut linter = create_linter(syntax, lint_rules); let mut has_error = false; let pseudo_file_name = "_stdin.ts"; diff --git a/cli/main.rs b/cli/main.rs index 51e768caf8aed9..5f794644852acd 100644 --- a/cli/main.rs +++ b/cli/main.rs @@ -22,6 +22,7 @@ extern crate serde_derive; extern crate tokio; extern crate url; +mod ast; mod checksum; pub mod colors; mod coverage; @@ -59,7 +60,6 @@ pub mod resolve_addr; pub mod signal; pub mod source_maps; pub mod state; -mod swc_util; mod test_runner; mod text_encoding; mod tokio_util; @@ -72,7 +72,6 @@ pub mod worker; use crate::coverage::CoverageCollector; use crate::coverage::PrettyCoverageReporter; -use crate::file_fetcher::map_file_extension; use crate::file_fetcher::SourceFile; use crate::file_fetcher::SourceFileFetcher; use crate::file_fetcher::TextDocument; @@ -376,7 +375,7 @@ async fn doc_command( let doc_parser = doc::DocParser::new(loader, private); let parse_result = if source_file == "--builtin" { - let syntax = swc_util::get_syntax_for_dts(); + let syntax = ast::get_syntax(&msg::MediaType::Dts); doc_parser.parse_source( "lib.deno.d.ts", syntax, @@ -384,12 +383,8 @@ async fn doc_command( ) } else { let path = PathBuf::from(&source_file); - let syntax = if path.ends_with("d.ts") { - swc_util::get_syntax_for_dts() - } else { - let media_type = map_file_extension(&path); - swc_util::get_syntax_for_media_type(media_type) - }; + let media_type = MediaType::from(&path); + let syntax = ast::get_syntax(&media_type); let module_specifier = ModuleSpecifier::resolve_url_or_path(&source_file).unwrap(); doc_parser diff --git a/cli/module_graph.rs b/cli/module_graph.rs index 40147c44c1c00c..2c00305297f754 100644 --- a/cli/module_graph.rs +++ b/cli/module_graph.rs @@ -1,13 +1,12 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. +use crate::ast::Location; use crate::checksum; -use crate::file_fetcher::map_file_extension; use crate::file_fetcher::SourceFile; use crate::file_fetcher::SourceFileFetcher; use crate::import_map::ImportMap; use crate::msg::MediaType; use crate::permissions::Permissions; -use crate::swc_util::Location; use crate::tsc::pre_process_file; use crate::tsc::ImportDesc; use crate::tsc::TsReferenceDesc; @@ -24,7 +23,6 @@ use serde::Serialize; use serde::Serializer; use std::collections::HashMap; use std::collections::HashSet; -use std::path::PathBuf; use std::pin::Pin; // TODO(bartlomieju): it'd be great if this function returned @@ -348,7 +346,7 @@ impl ModuleGraphLoader { let (raw_imports, raw_references) = pre_process_file( &module_specifier.to_string(), - map_file_extension(&PathBuf::from(&specifier)), + MediaType::from(&specifier), &source_code, self.analyze_dynamic_imports, )?; @@ -380,7 +378,7 @@ impl ModuleGraphLoader { url: specifier.to_string(), redirect: None, version_hash: "".to_string(), - media_type: map_file_extension(&PathBuf::from(specifier.clone())), + media_type: MediaType::from(&specifier), filename: specifier, source_code, imports, @@ -931,6 +929,8 @@ console.log(qat.qat); // According to TS docs (https://www.typescriptlang.org/docs/handbook/triple-slash-directives.html) // directives that are not at the top of the file are ignored, so only // 3 references should be captured instead of 4. + let file_specifier = + ModuleSpecifier::resolve_url_or_path("some/file.ts").unwrap(); assert_eq!( references, vec![ @@ -938,7 +938,7 @@ console.log(qat.qat); specifier: "dom".to_string(), kind: TsReferenceKind::Lib, location: Location { - filename: "some/file.ts".to_string(), + filename: file_specifier.to_string(), line: 5, col: 0, }, @@ -947,7 +947,7 @@ console.log(qat.qat); specifier: "./type_reference.d.ts".to_string(), kind: TsReferenceKind::Types, location: Location { - filename: "some/file.ts".to_string(), + filename: file_specifier.to_string(), line: 6, col: 0, }, @@ -956,7 +956,7 @@ console.log(qat.qat); specifier: "./type_reference/dep.ts".to_string(), kind: TsReferenceKind::Path, location: Location { - filename: "some/file.ts".to_string(), + filename: file_specifier.to_string(), line: 7, col: 0, }, diff --git a/cli/msg.rs b/cli/msg.rs index 3e5000296ca995..520d46fc2de5a0 100644 --- a/cli/msg.rs +++ b/cli/msg.rs @@ -1,10 +1,12 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. -// Warning! The values in this enum are duplicated in js/compiler.ts -// Update carefully! use serde::Serialize; use serde::Serializer; +use std::path::Path; +use std::path::PathBuf; +// Warning! The values in this enum are duplicated in tsc/99_main_compiler.js +// Update carefully! #[allow(non_camel_case_types)] #[repr(i32)] #[derive(Clone, Copy, PartialEq, Debug)] @@ -12,10 +14,73 @@ pub enum MediaType { JavaScript = 0, JSX = 1, TypeScript = 2, - TSX = 3, - Json = 4, - Wasm = 5, - Unknown = 6, + Dts = 3, + TSX = 4, + Json = 5, + Wasm = 6, + BuildInfo = 7, + Unknown = 8, +} + +impl<'a> From<&'a Path> for MediaType { + fn from(path: &'a Path) -> Self { + MediaType::from_path(path) + } +} + +impl<'a> From<&'a PathBuf> for MediaType { + fn from(path: &'a PathBuf) -> Self { + MediaType::from_path(path) + } +} + +impl<'a> From<&'a String> for MediaType { + fn from(specifier: &'a String) -> Self { + MediaType::from_path(&PathBuf::from(specifier)) + } +} + +impl MediaType { + fn from_path(path: &Path) -> Self { + match path.extension() { + None => MediaType::Unknown, + Some(os_str) => match os_str.to_str() { + Some("ts") => MediaType::TypeScript, + Some("tsx") => MediaType::TSX, + Some("js") => MediaType::JavaScript, + Some("jsx") => MediaType::JSX, + Some("mjs") => MediaType::JavaScript, + Some("cjs") => MediaType::JavaScript, + Some("json") => MediaType::Json, + Some("wasm") => MediaType::Wasm, + _ => MediaType::Unknown, + }, + } + } + + /// Convert a MediaType to a `ts.Extension`. + /// + /// *NOTE* This is defined in TypeScript as a string based enum. Changes to + /// that enum in TypeScript should be reflected here. + pub fn as_ts_extension(&self) -> &str { + match self { + MediaType::JavaScript => ".js", + MediaType::JSX => ".jsx", + MediaType::TypeScript => ".ts", + MediaType::Dts => ".d.ts", + MediaType::TSX => ".tsx", + MediaType::Json => ".json", + // TypeScript doesn't have an "unknown", so we will treat WASM as JS for + // mapping purposes, though in reality, it is unlikely to ever be passed + // to the compiler. + MediaType::Wasm => ".js", + MediaType::BuildInfo => ".tsbuildinfo", + // TypeScript doesn't have an "unknown", so we will treat WASM as JS for + // mapping purposes, though in reality, it is unlikely to ever be passed + // to the compiler. + MediaType::Unknown => ".js", + } + } } impl Serialize for MediaType { @@ -23,14 +88,16 @@ impl Serialize for MediaType { where S: Serializer, { - let value: i32 = match self { + let value = match self { MediaType::JavaScript => 0 as i32, MediaType::JSX => 1 as i32, MediaType::TypeScript => 2 as i32, - MediaType::TSX => 3 as i32, - MediaType::Json => 4 as i32, - MediaType::Wasm => 5 as i32, - MediaType::Unknown => 6 as i32, + MediaType::Dts => 3 as i32, + MediaType::TSX => 4 as i32, + MediaType::Json => 5 as i32, + MediaType::Wasm => 6 as i32, + MediaType::BuildInfo => 7 as i32, + MediaType::Unknown => 8 as i32, }; Serialize::serialize(&value, serializer) } @@ -41,9 +108,11 @@ pub fn enum_name_media_type(mt: MediaType) -> &'static str { MediaType::JavaScript => "JavaScript", MediaType::JSX => "JSX", MediaType::TypeScript => "TypeScript", + MediaType::Dts => "Dts", MediaType::TSX => "TSX", MediaType::Json => "Json", MediaType::Wasm => "Wasm", + MediaType::BuildInfo => "BuildInfo", MediaType::Unknown => "Unknown", } } @@ -76,3 +145,37 @@ impl Serialize for CompilerRequestType { Serialize::serialize(&value, serializer) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_map_file_extension() { + assert_eq!( + MediaType::from(Path::new("foo/bar.ts")), + MediaType::TypeScript + ); + assert_eq!(MediaType::from(Path::new("foo/bar.tsx")), MediaType::TSX); + assert_eq!( + MediaType::from(Path::new("foo/bar.d.ts")), + MediaType::TypeScript + ); + assert_eq!( + MediaType::from(Path::new("foo/bar.js")), + MediaType::JavaScript + ); + assert_eq!(MediaType::from(Path::new("foo/bar.jsx")), MediaType::JSX); + assert_eq!(MediaType::from(Path::new("foo/bar.json")), MediaType::Json); + assert_eq!(MediaType::from(Path::new("foo/bar.wasm")), MediaType::Wasm); + assert_eq!( + MediaType::from(Path::new("foo/bar.cjs")), + MediaType::JavaScript + ); + assert_eq!( + MediaType::from(Path::new("foo/bar.txt")), + MediaType::Unknown + ); + assert_eq!(MediaType::from(Path::new("foo/bar")), MediaType::Unknown); + } +} diff --git a/cli/swc_util.rs b/cli/swc_util.rs deleted file mode 100644 index f54f187e36c7da..00000000000000 --- a/cli/swc_util.rs +++ /dev/null @@ -1,445 +0,0 @@ -// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. - -use crate::msg::MediaType; -use deno_core::ErrBox; -use serde::Serialize; -use std::error::Error; -use std::fmt; -use std::rc::Rc; -use std::sync::Arc; -use std::sync::RwLock; -use swc_common::chain; -use swc_common::comments::SingleThreadedComments; -use swc_common::errors::Diagnostic; -use swc_common::errors::DiagnosticBuilder; -use swc_common::errors::Emitter; -use swc_common::errors::Handler; -use swc_common::errors::HandlerFlags; -use swc_common::FileName; -use swc_common::Globals; -use swc_common::SourceMap; -use swc_common::Span; -use swc_ecmascript::ast::Program; -use swc_ecmascript::codegen::text_writer::JsWriter; -use swc_ecmascript::codegen::Node; -use swc_ecmascript::parser::lexer::Lexer; -use swc_ecmascript::parser::EsConfig; -use swc_ecmascript::parser::JscTarget; -use swc_ecmascript::parser::Parser; -use swc_ecmascript::parser::StringInput; -use swc_ecmascript::parser::Syntax; -use swc_ecmascript::parser::TsConfig; -use swc_ecmascript::transforms::fixer; -use swc_ecmascript::transforms::helpers; -use swc_ecmascript::transforms::pass::Optional; -use swc_ecmascript::transforms::proposals::decorators; -use swc_ecmascript::transforms::react; -use swc_ecmascript::transforms::typescript; -use swc_ecmascript::visit::FoldWith; - -#[derive(Debug, Serialize, Clone, PartialEq)] -pub struct Location { - pub filename: String, - pub line: usize, - pub col: usize, -} - -impl Into for swc_common::Loc { - fn into(self) -> Location { - use swc_common::FileName::*; - - let filename = match &self.file.name { - Real(path_buf) => path_buf.to_string_lossy().to_string(), - Custom(str_) => str_.to_string(), - _ => panic!("invalid filename"), - }; - - Location { - filename, - line: self.line, - col: self.col_display, - } - } -} - -fn get_default_es_config() -> EsConfig { - let mut config = EsConfig::default(); - config.num_sep = true; - config.class_private_props = true; - config.class_private_methods = true; - config.class_props = true; - config.export_default_from = true; - config.export_namespace_from = true; - config.dynamic_import = true; - config.nullish_coalescing = true; - config.optional_chaining = true; - config.import_meta = true; - config.top_level_await = true; - config -} - -fn get_default_ts_config() -> TsConfig { - let mut ts_config = TsConfig::default(); - ts_config.dynamic_import = true; - ts_config.decorators = true; - ts_config -} - -pub fn get_syntax_for_dts() -> Syntax { - let mut ts_config = TsConfig::default(); - ts_config.dts = true; - Syntax::Typescript(ts_config) -} - -pub fn get_syntax_for_media_type(media_type: MediaType) -> Syntax { - match media_type { - MediaType::JavaScript => Syntax::Es(get_default_es_config()), - MediaType::JSX => { - let mut config = get_default_es_config(); - config.jsx = true; - Syntax::Es(config) - } - MediaType::TypeScript => Syntax::Typescript(get_default_ts_config()), - MediaType::TSX => { - let mut config = get_default_ts_config(); - config.tsx = true; - Syntax::Typescript(config) - } - _ => Syntax::Es(get_default_es_config()), - } -} - -#[derive(Clone, Debug)] -pub struct SwcDiagnosticBuffer { - pub diagnostics: Vec, -} - -impl Error for SwcDiagnosticBuffer {} - -impl fmt::Display for SwcDiagnosticBuffer { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let msg = self.diagnostics.join(","); - - f.pad(&msg) - } -} - -impl SwcDiagnosticBuffer { - pub fn from_swc_error( - error_buffer: SwcErrorBuffer, - parser: &AstParser, - ) -> Self { - let s = error_buffer.0.read().unwrap().clone(); - - let diagnostics = s - .iter() - .map(|d| { - let mut msg = d.message(); - - if let Some(span) = d.span.primary_span() { - let location = parser.get_span_location(span); - let filename = match &location.file.name { - FileName::Custom(n) => n, - _ => unreachable!(), - }; - msg = format!( - "{} at {}:{}:{}", - msg, filename, location.line, location.col_display - ); - } - - msg - }) - .collect::>(); - - Self { diagnostics } - } -} - -#[derive(Clone)] -pub struct SwcErrorBuffer(Arc>>); - -impl SwcErrorBuffer { - pub fn default() -> Self { - Self(Arc::new(RwLock::new(vec![]))) - } -} - -impl Emitter for SwcErrorBuffer { - fn emit(&mut self, db: &DiagnosticBuilder) { - self.0.write().unwrap().push((**db).clone()); - } -} - -/// Low-level utility structure with common AST parsing functions. -/// -/// Allows to build more complicated parser by providing a callback -/// to `parse_module`. -pub struct AstParser { - pub buffered_error: SwcErrorBuffer, - pub source_map: Rc, - pub handler: Handler, - pub comments: SingleThreadedComments, - pub globals: Globals, -} - -impl AstParser { - pub fn default() -> Self { - let buffered_error = SwcErrorBuffer::default(); - - let handler = Handler::with_emitter_and_flags( - Box::new(buffered_error.clone()), - HandlerFlags { - dont_buffer_diagnostics: true, - can_emit_warnings: true, - ..Default::default() - }, - ); - - AstParser { - buffered_error, - source_map: Rc::new(SourceMap::default()), - handler, - comments: SingleThreadedComments::default(), - globals: Globals::new(), - } - } - - pub fn parse_module( - &self, - file_name: &str, - media_type: MediaType, - source_code: &str, - ) -> Result { - let swc_source_file = self.source_map.new_source_file( - FileName::Custom(file_name.to_string()), - source_code.to_string(), - ); - - let buffered_err = self.buffered_error.clone(); - let syntax = get_syntax_for_media_type(media_type); - - let lexer = Lexer::new( - syntax, - JscTarget::Es2019, - StringInput::from(&*swc_source_file), - Some(&self.comments), - ); - - let mut parser = Parser::new_from(lexer); - - parser.parse_module().map_err(move |err| { - let mut diagnostic = err.into_diagnostic(&self.handler); - diagnostic.emit(); - SwcDiagnosticBuffer::from_swc_error(buffered_err, self) - }) - } - - pub fn get_span_location(&self, span: Span) -> swc_common::Loc { - self.source_map.lookup_char_pos(span.lo()) - } - - pub fn get_span_comments( - &self, - span: Span, - ) -> Vec { - self - .comments - .with_leading(span.lo(), |comments| comments.to_vec()) - } -} - -#[derive(Debug, Clone)] -pub struct EmitTranspileOptions { - /// When emitting a legacy decorator, also emit experimental decorator meta - /// data. Defaults to `false`. - pub emit_metadata: bool, - /// Should the source map be inlined in the emitted code file, or provided - /// as a separate file. Defaults to `true`. - pub inline_source_map: bool, - /// When transforming JSX, what value should be used for the JSX factory. - /// Defaults to `React.createElement`. - pub jsx_factory: String, - /// When transforming JSX, what value should be used for the JSX fragment - /// factory. Defaults to `React.Fragment`. - pub jsx_fragment_factory: String, - /// Should JSX be transformed or preserved. Defaults to `true`. - pub transform_jsx: bool, -} - -impl Default for EmitTranspileOptions { - fn default() -> Self { - EmitTranspileOptions { - emit_metadata: false, - inline_source_map: true, - jsx_factory: "React.createElement".into(), - jsx_fragment_factory: "React.Fragment".into(), - transform_jsx: true, - } - } -} - -pub fn transpile( - file_name: &str, - media_type: MediaType, - source_code: &str, - options: &EmitTranspileOptions, -) -> Result<(String, Option), ErrBox> { - let ast_parser = AstParser::default(); - let module = ast_parser.parse_module(file_name, media_type, source_code)?; - let program = Program::Module(module); - - let jsx_pass = react::react( - ast_parser.source_map.clone(), - Some(&ast_parser.comments), - react::Options { - pragma: options.jsx_factory.clone(), - pragma_frag: options.jsx_fragment_factory.clone(), - // this will use `Object.assign()` instead of the `_extends` helper - // when spreading props. - use_builtins: true, - ..Default::default() - }, - ); - let mut passes = chain!( - Optional::new(jsx_pass, options.transform_jsx), - decorators::decorators(decorators::Config { - legacy: true, - emit_metadata: options.emit_metadata - }), - typescript::strip(), - fixer(Some(&ast_parser.comments)), - ); - - let program = swc_common::GLOBALS.set(&Globals::new(), || { - helpers::HELPERS.set(&helpers::Helpers::new(false), || { - program.fold_with(&mut passes) - }) - }); - - let mut src_map_buf = vec![]; - let mut buf = vec![]; - { - let writer = Box::new(JsWriter::new( - ast_parser.source_map.clone(), - "\n", - &mut buf, - Some(&mut src_map_buf), - )); - let config = swc_ecmascript::codegen::Config { minify: false }; - let mut emitter = swc_ecmascript::codegen::Emitter { - cfg: config, - comments: Some(&ast_parser.comments), - cm: ast_parser.source_map.clone(), - wr: writer, - }; - program.emit_with(&mut emitter)?; - } - let mut src = String::from_utf8(buf)?; - let mut map: Option = None; - { - let mut buf = Vec::new(); - ast_parser - .source_map - .build_source_map_from(&mut src_map_buf, None) - .to_writer(&mut buf)?; - - if options.inline_source_map { - src.push_str("//# sourceMappingURL=data:application/json;base64,"); - let encoded_map = base64::encode(buf); - src.push_str(&encoded_map); - } else { - map = Some(String::from_utf8(buf)?); - } - } - Ok((src, map)) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_transpile() { - let source = r#" - enum D { - A, - B, - C, - } - export class A { - private b: string; - protected c: number = 1; - e: "foo"; - constructor (public d = D.A) { - const e = "foo" as const; - this.e = e; - } - } - "#; - let result = transpile( - "test.ts", - MediaType::TypeScript, - source, - &EmitTranspileOptions::default(), - ) - .unwrap(); - let (code, maybe_map) = result; - assert!(code.starts_with("var D;\n(function(D) {\n")); - assert!( - code.contains("\n//# sourceMappingURL=data:application/json;base64,") - ); - assert!(maybe_map.is_none()); - } - - #[test] - fn test_transpile_tsx() { - let source = r#" - export class A { - render() { - return
- } - } - "#; - let result = transpile( - "test.ts", - MediaType::TSX, - source, - &EmitTranspileOptions::default(), - ) - .unwrap(); - let (code, _maybe_source_map) = result; - assert!(code.contains("React.createElement(\"div\", null")); - } - - #[test] - fn test_transpile_decorators() { - let source = r#" - function enumerable(value: boolean) { - return function ( - _target: any, - _propertyKey: string, - descriptor: PropertyDescriptor, - ) { - descriptor.enumerable = value; - }; - } - - export class A { - @enumerable(false) - a() { - Test.value; - } - } - "#; - let result = transpile( - "test.ts", - MediaType::TypeScript, - source, - &EmitTranspileOptions::default(), - ) - .unwrap(); - let (code, _maybe_source_map) = result; - assert!(code.contains("_applyDecoratedDescriptor(")); - } -} diff --git a/cli/tsc.rs b/cli/tsc.rs index 98e73ae21f5f74..3d2fe880499162 100644 --- a/cli/tsc.rs +++ b/cli/tsc.rs @@ -1,5 +1,8 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. +use crate::ast::parse; +use crate::ast::Location; +use crate::ast::TranspileOptions; use crate::colors; use crate::diagnostics::Diagnostics; use crate::disk_cache::DiskCache; @@ -17,10 +20,6 @@ use crate::ops; use crate::permissions::Permissions; use crate::source_maps::SourceMapGetter; use crate::state::State; -use crate::swc_util; -use crate::swc_util::AstParser; -use crate::swc_util::Location; -use crate::swc_util::SwcDiagnosticBuffer; use crate::tsc_config; use crate::version; use crate::worker::Worker; @@ -819,20 +818,20 @@ impl TsCompiler { let compiler_options: TranspileTsOptions = serde_json::from_value(compiler_options)?; - let transpile_options = swc_util::EmitTranspileOptions { + let transpile_options = TranspileOptions { emit_metadata: compiler_options.emit_decorator_metadata, inline_source_map: true, jsx_factory: compiler_options.jsx_factory, jsx_fragment_factory: compiler_options.jsx_fragment_factory, transform_jsx: compiler_options.jsx == "react", }; + let media_type = MediaType::TypeScript; for source_file in source_files { - let (stripped_source, _maybe_source_map) = swc_util::transpile( - &source_file.file_name, - MediaType::TypeScript, - &source_file.source_code, - &transpile_options, - )?; + let specifier = + ModuleSpecifier::resolve_url_or_path(&source_file.file_name)?; + let parsed_module = + parse(&specifier, &source_file.source_code, &media_type)?; + let (stripped_source, _) = parsed_module.transpile(&transpile_options)?; // TODO(bartlomieju): this is superfluous, just to make caching function happy let emitted_filename = PathBuf::from(&source_file.file_name) @@ -1467,16 +1466,11 @@ pub fn pre_process_file( media_type: MediaType, source_code: &str, analyze_dynamic_imports: bool, -) -> Result<(Vec, Vec), SwcDiagnosticBuffer> { - let parser = AstParser::default(); - let parse_result = parser.parse_module(file_name, media_type, source_code); - let module = parse_result?; - - let dependency_descriptors = dep_graph::analyze_dependencies( - &module, - &parser.source_map, - &parser.comments, - ); +) -> Result<(Vec, Vec), ErrBox> { + let specifier = ModuleSpecifier::resolve_url_or_path(file_name)?; + let module = parse(&specifier, source_code, &media_type)?; + + let dependency_descriptors = module.analyze_dependencies(); // for each import check if there's relevant @deno-types directive let imports = dependency_descriptors @@ -1503,7 +1497,7 @@ pub fn pre_process_file( .collect(); // analyze comment from beginning of the file and find TS directives - let comments = parser.get_span_comments(module.span); + let comments = module.leading_comments; let mut references = vec![]; for comment in comments { @@ -1513,7 +1507,7 @@ pub fn pre_process_file( let text = comment.text.to_string(); if let Some((kind, specifier)) = parse_ts_reference(text.trim()) { - let location = parser.get_span_location(comment.span); + let location = module.source_map.lookup_char_pos(comment.span.lo); references.push(TsReferenceDesc { kind, specifier, diff --git a/cli/tsc/99_main_compiler.js b/cli/tsc/99_main_compiler.js index 41b52d283c4775..80f7b22336bfe8 100644 --- a/cli/tsc/99_main_compiler.js +++ b/cli/tsc/99_main_compiler.js @@ -131,16 +131,20 @@ delete Object.prototype.__proto__; 0: "JavaScript", 1: "JSX", 2: "TypeScript", - 3: "TSX", - 4: "Json", - 5: "Wasm", - 6: "Unknown", + 3: "Dts", + 4: "TSX", + 5: "Json", + 6: "Wasm", + 7: "BuildInfo", + 8: "Unknown", JavaScript: 0, JSX: 1, TypeScript: 2, - TSX: 3, - Json: 4, - Wasm: 5, + Dts: 3, + TSX: 4, + Json: 5, + Wasm: 6, + BuildInfo: 7, Unknown: 6, }; From 8eb28d97a822a9533695fe6ef33ca2088040e359 Mon Sep 17 00:00:00 2001 From: Kitson Kelly Date: Mon, 14 Sep 2020 21:27:51 +1000 Subject: [PATCH 2/2] address feedback --- cli/ast.rs | 197 ++++------------------------------------------------- cli/tsc.rs | 6 +- 2 files changed, 18 insertions(+), 185 deletions(-) diff --git a/cli/ast.rs b/cli/ast.rs index bb25d5e9ec17fe..21dd51c5e442cb 100644 --- a/cli/ast.rs +++ b/cli/ast.rs @@ -41,9 +41,7 @@ use swc_ecmascript::transforms::pass::Optional; use swc_ecmascript::transforms::proposals::decorators; use swc_ecmascript::transforms::react; use swc_ecmascript::transforms::typescript; -use swc_ecmascript::visit; use swc_ecmascript::visit::FoldWith; -use swc_ecmascript::visit::Visit; type Result = result::Result; @@ -173,134 +171,6 @@ pub fn get_syntax(media_type: &MediaType) -> Syntax { } } -/// Visits a pattern node, recursively looking for any names that end up in the -/// local scope, pushing them onto the passed vector. -fn visit_pat(pat: &swc_ecmascript::ast::Pat, names: &mut Vec) { - match pat { - swc_ecmascript::ast::Pat::Ident(ident) => names.push(ident.sym.to_string()), - swc_ecmascript::ast::Pat::Array(array_pat) => { - for elem in array_pat.elems.iter() { - if let Some(pat) = elem { - visit_pat(pat, names); - } - } - } - swc_ecmascript::ast::Pat::Rest(rest_pat) => { - visit_pat(rest_pat.arg.as_ref(), names) - } - swc_ecmascript::ast::Pat::Object(object_pat) => { - for prop in object_pat.props.iter() { - match prop { - swc_ecmascript::ast::ObjectPatProp::Assign(assign_pat) => { - names.push(assign_pat.key.sym.to_string()) - } - swc_ecmascript::ast::ObjectPatProp::KeyValue(key_value) => { - visit_pat(key_value.value.as_ref(), names) - } - swc_ecmascript::ast::ObjectPatProp::Rest(rest_pat) => { - visit_pat(rest_pat.arg.as_ref(), names) - } - } - } - } - swc_ecmascript::ast::Pat::Assign(assign_pat) => { - visit_pat(assign_pat.left.as_ref(), names) - } - // Invalid and Expressions are noops - _ => {} - } -} - -/// A structure for collecting the named exports from a module. -#[derive(Default)] -struct ExportCollector { - pub names: Vec, - pub export_all_specifiers: Vec, -} - -impl Visit for ExportCollector { - fn visit_export_decl( - &mut self, - node: &swc_ecmascript::ast::ExportDecl, - _parent: &dyn visit::Node, - ) { - match &node.decl { - swc_ecmascript::ast::Decl::Class(class_decl) => { - self.names.push(class_decl.ident.sym.to_string()); - } - swc_ecmascript::ast::Decl::Fn(fn_decl) => { - self.names.push(fn_decl.ident.sym.to_string()); - } - swc_ecmascript::ast::Decl::Var(var_decl) => { - for decl in var_decl.decls.iter() { - visit_pat(&decl.name, &mut self.names); - } - } - swc_ecmascript::ast::Decl::TsEnum(ts_enum_decl) => { - self.names.push(ts_enum_decl.id.sym.to_string()); - } - // Interfaces, Type Aliases, and TS Module/Namespace decl are noops - _ => {} - } - } - - fn visit_named_export( - &mut self, - node: &swc_ecmascript::ast::NamedExport, - _parent: &dyn visit::Node, - ) { - for spec in node.specifiers.iter() { - match spec { - swc_ecmascript::ast::ExportSpecifier::Named(named_spec) => { - if let Some(ident) = &named_spec.exported { - self.names.push(ident.sym.to_string()); - } else { - self.names.push(named_spec.orig.sym.to_string()); - } - } - swc_ecmascript::ast::ExportSpecifier::Namespace(namespace_spec) => { - self.names.push(namespace_spec.name.sym.to_string()); - } - // Default is only proposed syntax, not current supported, so noop - _ => {} - } - } - } - - fn visit_export_default_decl( - &mut self, - node: &swc_ecmascript::ast::ExportDefaultDecl, - _parent: &dyn visit::Node, - ) { - match &node.decl { - swc_ecmascript::ast::DefaultDecl::Class(_) => { - self.names.push("default".to_string()) - } - swc_ecmascript::ast::DefaultDecl::Fn(_) => { - self.names.push("default".to_string()) - } - // Interface is a noop - _ => {} - } - } - - fn visit_export_default_expr( - &mut self, - _node: &swc_ecmascript::ast::ExportDefaultExpr, - _parent: &dyn visit::Node, - ) { - self.names.push("default".to_string()); - } - - fn visit_export_all( - &mut self, - node: &swc_ecmascript::ast::ExportAll, - _parent: &dyn visit::Node, - ) { - self.export_all_specifiers.push(node.src.value.to_string()); - } -} - /// Options which can be adjusted when transpiling a module. #[derive(Debug, Clone)] pub struct TranspileOptions { @@ -336,24 +206,32 @@ impl Default for TranspileOptions { /// processing. pub struct ParsedModule { comments: SingleThreadedComments, - pub leading_comments: Vec, + leading_comments: Vec, module: Module, - pub source_map: Rc, + source_map: Rc, } impl ParsedModule { + /// Return a vector of dependencies for the module. pub fn analyze_dependencies(&self) -> Vec { analyze_dependencies(&self.module, &self.source_map, &self.comments) } - #[allow(dead_code)] // TODO(kitsonk) for bundling rewrite - pub fn analyze_exported_names(&self) -> (Vec, Vec) { - let mut collector = ExportCollector::default(); - collector.visit_module(&self.module, &self.module); + /// Get the module's leading comments, where triple slash directives might + /// be located. + pub fn get_leading_comments(&self) -> Vec { + self.leading_comments.clone() + } - (collector.names, collector.export_all_specifiers) + /// Get a location for a given span within the module. + pub fn get_location(&self, span: &Span) -> Location { + self.source_map.lookup_char_pos(span.lo).into() } + /// Transform a TypeScript file into a JavaScript file, based on the supplied + /// options. + /// + /// The result is a tuple of the code and optional source map as strings. pub fn transpile( self, options: &TranspileOptions, @@ -522,51 +400,6 @@ mod tests { ); } - #[test] - fn test_parsed_module_analyze_exported_names() { - let specifier = - ModuleSpecifier::resolve_url_or_path("https://deno.land/x/mod.ts") - .unwrap(); - let source = r#" - export * from "./a.ts"; - - export { a, b as c } from "./b.ts"; - - export default function () { - console.log("hello"); - } - - export enum C { - A, - B, - C, - } - - export const [d, e, ...f] = [1, 2, 3, 4, 5]; - - export const g = 1; - - export const { h, i: j, ...k } = { h: true, i: false, j: 1, k: 2 }; - - export class A {} - - export function l() {} - - export * as m from "./m.ts"; - "#; - let parsed_module = parse(&specifier, source, &MediaType::TypeScript) - .expect("could not parse module"); - let (names, export_all_specifiers) = parsed_module.analyze_exported_names(); - assert_eq!( - names, - vec![ - "a", "c", "default", "C", "d", "e", "f", "g", "h", "j", "k", "A", "l", - "m" - ] - ); - assert_eq!(export_all_specifiers, vec!["./a.ts"]); - } - #[test] fn test_transpile() { let specifier = diff --git a/cli/tsc.rs b/cli/tsc.rs index 3d2fe880499162..b3d43559b420ed 100644 --- a/cli/tsc.rs +++ b/cli/tsc.rs @@ -1497,7 +1497,7 @@ pub fn pre_process_file( .collect(); // analyze comment from beginning of the file and find TS directives - let comments = module.leading_comments; + let comments = module.get_leading_comments(); let mut references = vec![]; for comment in comments { @@ -1507,11 +1507,11 @@ pub fn pre_process_file( let text = comment.text.to_string(); if let Some((kind, specifier)) = parse_ts_reference(text.trim()) { - let location = module.source_map.lookup_char_pos(comment.span.lo); + let location = module.get_location(&comment.span); references.push(TsReferenceDesc { kind, specifier, - location: location.into(), + location, }); } }