From 5265935b3e5c76f1a9c69b364836e328a4be1c5b Mon Sep 17 00:00:00 2001 From: Abdulaziz Ghuloum Date: Tue, 24 Dec 2024 03:39:22 +0300 Subject: [PATCH] parsing most of vitest dts file --- src/expander.ts | 18 +++ src/library-manager.ts | 259 ++++++++++++++++++++++------------------- src/parse-dts.ts | 177 ++++++++++++++++++++++++++++ src/parse.ts | 39 +++++++ src/tags.ts | 18 +++ 5 files changed, 392 insertions(+), 119 deletions(-) create mode 100644 src/parse-dts.ts diff --git a/src/expander.ts b/src/expander.ts index b698e14..4ea2f14 100644 --- a/src/expander.ts +++ b/src/expander.ts @@ -262,6 +262,24 @@ const list_handlers_table: { [tag in list_tag]: "descend" | "stop" | "todo" } = type_parameter: "todo", type_parameters: "todo", type_query: "todo", + function_type: "todo", + interface_declaration: "todo", + module_block: "todo", + module_declaration: "todo", + array_type: "todo", + call_signature: "todo", + conditional_type: "todo", + expression_with_type_arguments: "todo", + function_declaration: "todo", + heritage_clause: "todo", + indexed_access_type: "todo", + mapped_type: "todo", + method_signature: "todo", + parenthesized_type: "todo", + type_literal: "todo", + type_operator: "todo", + type_predicate: "todo", + type_reference: "todo", syntax_list: "descend", }; diff --git a/src/library-manager.ts b/src/library-manager.ts index 0678d96..0b4d143 100644 --- a/src/library-manager.ts +++ b/src/library-manager.ts @@ -18,6 +18,7 @@ import { normalize } from "node:path"; import { Binding, CompilationUnit, Context, Loc, Rib } from "./syntax-structures"; import stringify from "json-stringify-pretty-compact"; import { init_global_context } from "./global-module"; +import { parse_dts } from "./parse-dts"; const cookie = "rewrite-ts-016"; @@ -44,15 +45,17 @@ type module_state = } | { type: "error"; reason: string }; -class RtsModule implements imported_module { - private path: string; - private library_manager: LibraryManager; - private state: module_state = { type: "initial" }; - private libs: string[]; - private global_unit: CompilationUnit; - private global_context: Context; - public imported_modules: imported_module[] = []; - public dependant_modules: imported_module[] = []; +abstract class Module implements imported_module { + path: string; + library_manager: LibraryManager; + state: module_state = { type: "initial" }; + libs: string[]; + global_unit: CompilationUnit; + global_context: Context; + imported_modules: imported_module[] = []; + dependant_modules: imported_module[] = []; + + abstract do_recompile(): Promise; constructor( path: string, @@ -88,22 +91,10 @@ class RtsModule implements imported_module { } } - private get_json_path(): string { + get_json_path(): string { return join(dirname(this.path), ".rts", basename(this.path) + ".json"); } - private get_generated_code_absolute_path(): string { - return join(dirname(this.path), ".rts", basename(this.path) + ".ts"); - } - - private get_generated_code_relative_path(): string { - return "./.rts/" + basename(this.path) + ".ts"; - } - - private get_proxy_path(): string { - return this.path + ".ts"; - } - private async do_initialize() { assert(this.state.type === "initial", `invalid state ${this.state.type}`); const [pkg, pkg_relative_path] = await this.library_manager.findPackage(this.path); @@ -172,98 +163,6 @@ class RtsModule implements imported_module { } } - async do_recompile() { - const state = this.state; - assert(state.type === "stale"); - console.log(`recompiling ${state.cid} ...`); - const code = await fs.readFile(this.path, { encoding: "utf-8" }); - const my_pkg = state.pkg; - const my_path = state.pkg_relative_path; - const source_file: source_file = { - package: { name: my_pkg.name, version: my_pkg.version }, - path: my_path, - }; - const [_loc0, expand] = initial_step(parse(code, source_file), state.cid, this.libs); - try { - const helpers: preexpand_helpers = { - manager: { - resolve_import: async (loc) => { - assert(loc.t.tag === "string"); - const import_path = JSON.parse(loc.t.content); - const mod = this.get_imported_modules_for_path(import_path, loc); - return mod; - }, - resolve_label: async (label) => { - const mod = this.find_module_by_cid(label.cuid); - if (!mod) throw new Error(`cannot find module with cuid = ${label.cuid}`); - return mod.resolve_label(label.name); - }, - get_import_path: async (cuid) => { - const mod = this.find_module_by_cid(cuid); - if (!mod) throw new Error(`cannot find module with cuid = ${cuid}`); - const [mod_pkg, mod_path] = mod.get_pkg_and_path(); - if (mod_pkg === my_pkg) { - const dir0 = join(dirname(my_path), ".rts"); - const dir1 = join(dirname(mod_path), ".rts"); - if (dir0 !== dir1) throw new Error(`TODO relative path imports`); - return `./${basename(mod_path)}.ts`; - } else { - throw new Error(`TODO cross package imports`); - } - }, - resolve_rib: (rib_id, cuid) => { - const mod = this.find_module_by_cid(cuid); - if (!mod) throw new Error(`cannot find module with cuid = ${cuid}`); - return mod.resolve_rib(rib_id); - }, - }, - global_unit: this.global_unit, - global_context: this.global_context, - inspect(_loc, _reason, k) { - return k(); - }, - }; - const { loc, unit, context, modular } = await expand(helpers); - assert(modular.extensible); - const proxy_code = generate_proxy_code( - this.get_generated_code_relative_path(), - modular, - context, - ); - const exported_identifiers = get_exported_identifiers_from_rib( - modular.explicit, - state.cid, - context, - ); - const json_content = { - cid: state.cid, - cookie, - imports: this.imported_modules.map((x) => { - const [pkg, path] = x.get_pkg_and_path(); - return { pkg: { name: pkg.name, version: pkg.version }, pkg_relative_path: path }; - }), - exported_identifiers, - context, - unit, - }; - const code_path = this.get_generated_code_absolute_path(); - await fs.mkdir(dirname(code_path), { recursive: true }); - await fs.writeFile(code_path, await pprint(loc)); - await fs.writeFile(this.get_proxy_path(), proxy_code); - const mtime = Date.now(); - await fs.writeFile(this.get_json_path(), stringify(json_content)); - this.state = { ...state, type: "fresh", ...json_content, mtime }; - console.log(`up to date ${state.cid}`); - } catch (error) { - this.state = { type: "error", reason: String(error) }; - if (error instanceof StxError) { - await print_stx_error(error, this.library_manager); - } else { - console.error(error); - } - } - } - async recompile() { const state = this.state; switch (state.type) { @@ -377,10 +276,7 @@ class RtsModule implements imported_module { return [this.state.pkg, this.state.pkg_relative_path]; } - private async get_imported_modules_for_path( - import_path: string, - loc: Loc, - ): Promise { + async get_imported_modules_for_path(import_path: string, loc: Loc): Promise { const mod = await this.library_manager.do_import(import_path, this.path); if (this.imported_modules.includes(mod)) return mod; const self = this; @@ -403,6 +299,131 @@ class RtsModule implements imported_module { assert(rib !== undefined); return rib; } + + get_preexpand_helpers(my_pkg: Package, my_path: string): preexpand_helpers { + const helpers: preexpand_helpers = { + manager: { + resolve_import: async (loc) => { + assert(loc.t.tag === "string"); + const import_path = JSON.parse(loc.t.content); + const mod = this.get_imported_modules_for_path(import_path, loc); + return mod; + }, + resolve_label: async (label) => { + const mod = this.find_module_by_cid(label.cuid); + if (!mod) throw new Error(`cannot find module with cuid = ${label.cuid}`); + return mod.resolve_label(label.name); + }, + get_import_path: async (cuid) => { + const mod = this.find_module_by_cid(cuid); + if (!mod) throw new Error(`cannot find module with cuid = ${cuid}`); + const [mod_pkg, mod_path] = mod.get_pkg_and_path(); + if (mod_pkg === my_pkg) { + const dir0 = join(dirname(my_path), ".rts"); + const dir1 = join(dirname(mod_path), ".rts"); + if (dir0 !== dir1) throw new Error(`TODO relative path imports`); + return `./${basename(mod_path)}.ts`; + } else { + throw new Error(`TODO cross package imports`); + } + }, + resolve_rib: (rib_id, cuid) => { + const mod = this.find_module_by_cid(cuid); + if (!mod) throw new Error(`cannot find module with cuid = ${cuid}`); + return mod.resolve_rib(rib_id); + }, + }, + global_unit: this.global_unit, + global_context: this.global_context, + inspect(_loc, _reason, k) { + return k(); + }, + }; + return helpers; + } +} + +class RtsModule extends Module { + private get_generated_code_absolute_path(): string { + return join(dirname(this.path), ".rts", basename(this.path) + ".ts"); + } + + private get_generated_code_relative_path(): string { + return "./.rts/" + basename(this.path) + ".ts"; + } + + private get_proxy_path(): string { + return this.path + ".ts"; + } + + async do_recompile() { + const state = this.state; + assert(state.type === "stale"); + console.log(`recompiling ${state.cid} ...`); + const code = await fs.readFile(this.path, { encoding: "utf-8" }); + const my_pkg = state.pkg; + const my_path = state.pkg_relative_path; + const source_file: source_file = { + package: { name: my_pkg.name, version: my_pkg.version }, + path: my_path, + }; + const [_loc0, expand] = initial_step(parse(code, source_file), state.cid, this.libs); + try { + const helpers = this.get_preexpand_helpers(my_pkg, my_path); + const { loc, unit, context, modular } = await expand(helpers); + assert(modular.extensible); + const proxy_code = generate_proxy_code( + this.get_generated_code_relative_path(), + modular, + context, + ); + const exported_identifiers = get_exported_identifiers_from_rib( + modular.explicit, + state.cid, + context, + ); + const json_content = { + cid: state.cid, + cookie, + imports: this.imported_modules.map((x) => { + const [pkg, path] = x.get_pkg_and_path(); + return { pkg: { name: pkg.name, version: pkg.version }, pkg_relative_path: path }; + }), + exported_identifiers, + context, + unit, + }; + const code_path = this.get_generated_code_absolute_path(); + await fs.mkdir(dirname(code_path), { recursive: true }); + await fs.writeFile(code_path, await pprint(loc)); + await fs.writeFile(this.get_proxy_path(), proxy_code); + const mtime = Date.now(); + await fs.writeFile(this.get_json_path(), stringify(json_content)); + this.state = { ...state, type: "fresh", ...json_content, mtime }; + console.log(`up to date ${state.cid}`); + } catch (error) { + this.state = { type: "error", reason: String(error) }; + if (error instanceof StxError) { + await print_stx_error(error, this.library_manager); + } else { + console.error(error); + } + } + } +} + +class DtsModule extends Module { + async do_recompile() { + const state = this.state; + assert(state.type === "stale"); + console.log(`recompiling ${state.cid} ...`); + const my_pkg = state.pkg; + const my_path = state.pkg_relative_path; + const code = await fs.readFile(this.path, { encoding: "utf-8" }); + const stuff = parse_dts(code, my_path); + console.log(stuff); + throw new Error("not yet"); + } } type package_props = { types_file?: string }; @@ -452,7 +473,7 @@ export class LibraryManager { if (path.endsWith(".rts")) { return new RtsModule(path, this, this.libs, this.global_unit, this.global_context); } else if (path.endsWith(".d.ts")) { - throw new Error(`TODO import .d.ts ${path}`); + return new DtsModule(path, this, this.libs, this.global_unit, this.global_context); } else { throw new Error(`don't know how to import ${path}`); } diff --git a/src/parse-dts.ts b/src/parse-dts.ts new file mode 100644 index 0000000..27716dc --- /dev/null +++ b/src/parse-dts.ts @@ -0,0 +1,177 @@ +import TS from "typescript"; +import { assert } from "./assert"; + +function parse_statements(statements: TS.NodeArray) { + const types: { [name: string]: { defined: boolean; exported: boolean } } = {}; + const lexicals: { [name: string]: { declared: boolean; exported: boolean } } = {}; + const imported: { [name: string]: { exported_name: string; exporting_module: string } } = {}; + const namespace_imported: { [name: string]: { exporting_module: string } } = {}; + const exported: { [name: string]: { local_name: string; module_name: string | undefined } } = {}; + function push_lexical(name: string, props: { declared: boolean; exported: boolean }) { + assert(!lexicals[name]); + lexicals[name] = props; + } + function push_type(name: string, props: { defined: boolean; exported: boolean }) { + assert(!types[name]); + types[name] = props; + } + function push_imported(name: string, exported_name: string, exporting_module: string) { + assert(!imported[name]); + imported[name] = { exported_name, exporting_module }; + } + function push_namespace_imported(name: string, exporting_module: string) { + assert(!namespace_imported[name]); + namespace_imported[name] = { exporting_module }; + } + function push_exported(local_name: string, name: string, module_name: string | undefined) { + assert(!exported[name], `duplicate export ${local_name} as ${name}`); + exported[name] = { local_name, module_name }; + } + function handle_import_declaration(decl: TS.ImportDeclaration) { + const specifier = decl.moduleSpecifier; + assert(specifier.kind === TS.SyntaxKind.StringLiteral); + const module_name = (specifier as TS.StringLiteral).text; + function handle_clause_name(name: string) { + throw new Error(`import ${name} from ${module_name}`); + } + function handle_import_specifier(spec: TS.ImportSpecifier) { + const name = spec.name.text; + const exported_name = spec.propertyName?.text; + if (exported_name === undefined) { + push_imported(name, name, module_name); + } else { + push_imported(name, exported_name, module_name); + } + } + function handle_clause_bindings(bindings: TS.NamedImportBindings) { + switch (bindings.kind) { + case TS.SyntaxKind.NamedImports: + return bindings.elements.forEach(handle_import_specifier); + case TS.SyntaxKind.NamespaceImport: + return push_namespace_imported(bindings.name.text, module_name); + default: + const invalid: never = bindings; + throw invalid; + } + } + const clause = decl.importClause; + if (clause) { + if (clause.name) { + handle_clause_name(clause.name.text); + } + if (clause.namedBindings) { + handle_clause_bindings(clause.namedBindings); + } + } + } + function handle_export_declaration(decl: TS.ExportDeclaration) { + const module_name = decl.moduleSpecifier + ? (decl.moduleSpecifier as TS.StringLiteral).text + : undefined; + assert(module_name === undefined || typeof module_name === "string"); + function handle_export_specifier(spec: TS.ExportSpecifier) { + const name = spec.name.text; + const specname = spec.propertyName?.text; + if (specname) { + push_exported(specname, name, module_name); + } else { + push_exported(name, name, module_name); + } + } + function handle_named_exports(named_exports: TS.NamedExports) { + named_exports.elements.forEach(handle_export_specifier); + } + function handle_named_export_bindings(bindings: TS.NamedExportBindings) { + switch (bindings.kind) { + case TS.SyntaxKind.NamedExports: + return handle_named_exports(bindings); + default: + throw new Error(`unhandled named export binding type '${TS.SyntaxKind[bindings.kind]}'`); + } + } + assert(!decl.isTypeOnly); + const name = decl.name?.text; + if (name) { + throw new Error(`export name '${name}'`); + } + const clause = decl.exportClause; + if (clause) { + handle_named_export_bindings(clause); + } + } + function handle_module_declaration(decl: TS.ModuleDeclaration) { + const module_name = decl.name.text; + throw new Error(`module declaration '${module_name}'`); + } + function handle_interface_declaration(decl: TS.InterfaceDeclaration) { + const name = decl.name.text; + const exported = decl.modifiers?.some((x) => x.kind === TS.SyntaxKind.ExportKeyword) ?? false; + const declared = decl.modifiers?.some((x) => x.kind === TS.SyntaxKind.DeclareKeyword) ?? false; + assert(!declared); + push_type(name, { defined: true, exported }); + } + function handle_type_alias_declaration(decl: TS.TypeAliasDeclaration) { + const exported = decl.modifiers?.some((x) => x.kind === TS.SyntaxKind.ExportKeyword) ?? false; + const declared = decl.modifiers?.some((x) => x.kind === TS.SyntaxKind.DeclareKeyword) ?? false; + assert(!declared); + const name = decl.name.text; + push_type(name, { defined: true, exported }); + } + function handle_function_declaration(decl: TS.FunctionDeclaration) { + const name = decl.name?.text; + const exported = decl.modifiers?.some((x) => x.kind === TS.SyntaxKind.ExportKeyword) ?? false; + const declared = decl.modifiers?.some((x) => x.kind === TS.SyntaxKind.DeclareKeyword) ?? false; + assert(name !== undefined); + assert(declared); + push_lexical(name, { declared, exported }); + } + function handle_variable_statement(decl: TS.VariableStatement) { + const exported = decl.modifiers?.some((x) => x.kind === TS.SyntaxKind.ExportKeyword) ?? false; + const declared = decl.modifiers?.some((x) => x.kind === TS.SyntaxKind.DeclareKeyword) ?? false; + assert(declared); + function handle_binding_name(binding: TS.BindingName) { + switch (binding.kind) { + case TS.SyntaxKind.Identifier: + return push_lexical(binding.text, { declared, exported }); + default: + throw new Error(`unhandled binding type '${TS.SyntaxKind[binding.kind]}'`); + } + } + function handle_decl(decl: TS.VariableDeclaration) { + handle_binding_name(decl.name); + } + decl.declarationList.declarations.forEach(handle_decl); + } + function handle_statement(stmt: TS.Statement) { + switch (stmt.kind) { + case TS.SyntaxKind.ImportDeclaration: + return handle_import_declaration(stmt as TS.ImportDeclaration); + case TS.SyntaxKind.ExportDeclaration: + return handle_export_declaration(stmt as TS.ExportDeclaration); + case TS.SyntaxKind.ModuleDeclaration: + return handle_module_declaration(stmt as TS.ModuleDeclaration); + case TS.SyntaxKind.InterfaceDeclaration: + return handle_interface_declaration(stmt as TS.InterfaceDeclaration); + case TS.SyntaxKind.TypeAliasDeclaration: + return handle_type_alias_declaration(stmt as TS.TypeAliasDeclaration); + case TS.SyntaxKind.FunctionDeclaration: + return handle_function_declaration(stmt as TS.FunctionDeclaration); + case TS.SyntaxKind.VariableStatement: + return handle_variable_statement(stmt as TS.VariableStatement); + default: + throw new Error(`unhandled statement in d.ts file '${TS.SyntaxKind[stmt.kind]}'`); + } + } + statements.forEach(handle_statement); + return { types, lexicals, imported, namespace_imported, exported }; +} + +export function parse_dts(code: string, my_path: string) { + const options: TS.CreateSourceFileOptions = { + languageVersion: TS.ScriptTarget.ESNext, + jsDocParsingMode: TS.JSDocParsingMode.ParseNone, + }; + const src = TS.createSourceFile(my_path, code, options); + if (src.libReferenceDirectives.length !== 0) throw new Error("not handled"); + const data = parse_statements(src.statements); +} diff --git a/src/parse.ts b/src/parse.ts index fff3fa7..596c4f4 100644 --- a/src/parse.ts +++ b/src/parse.ts @@ -17,6 +17,26 @@ const pass_through: { [k in SyntaxKind]?: list_tag } = { [SyntaxKind.ImportClause]: "import_clause", [SyntaxKind.ImportDeclaration]: "import_declaration", [SyntaxKind.NamespaceImport]: "namespace_import", + [SyntaxKind.FunctionType]: "function_type", + [SyntaxKind.PropertySignature]: "property_signature", + [SyntaxKind.InterfaceDeclaration]: "interface_declaration", + [SyntaxKind.ModuleBlock]: "module_block", + [SyntaxKind.ModuleDeclaration]: "module_declaration", + [SyntaxKind.TypeOperator]: "type_operator", + [SyntaxKind.MappedType]: "mapped_type", + [SyntaxKind.TypeReference]: "type_reference", + [SyntaxKind.CallSignature]: "call_signature", + [SyntaxKind.ArrayType]: "array_type", + [SyntaxKind.TypeLiteral]: "type_literal", + [SyntaxKind.FunctionDeclaration]: "function_declaration", + [SyntaxKind.IndexedAccessType]: "indexed_access_type", + [SyntaxKind.ExpressionWithTypeArguments]: "expression_with_type_arguments", + [SyntaxKind.HeritageClause]: "heritage_clause", + [SyntaxKind.ConditionalType]: "conditional_type", + [SyntaxKind.TypeQuery]: "type_query", + [SyntaxKind.MethodSignature]: "method_signature", + [SyntaxKind.ParenthesizedType]: "parenthesized_type", + [SyntaxKind.TypePredicate]: "type_predicate", [SyntaxKind.SyntaxList]: "syntax_list", }; @@ -103,6 +123,25 @@ function absurdly(node: TS.Node, source: TS.SourceFile, f: source_file): AST { case SyntaxKind.NullKeyword: case SyntaxKind.StringKeyword: case SyntaxKind.NumberKeyword: + case SyntaxKind.DeclareKeyword: + case SyntaxKind.NamespaceKeyword: + case SyntaxKind.InterfaceKeyword: + case SyntaxKind.AnyKeyword: + case SyntaxKind.VoidKeyword: + case SyntaxKind.InKeyword: + case SyntaxKind.KeyOfKeyword: + case SyntaxKind.ModuleKeyword: + case SyntaxKind.NeverKeyword: + case SyntaxKind.BooleanKeyword: + case SyntaxKind.UndefinedKeyword: + case SyntaxKind.UnknownKeyword: + case SyntaxKind.FunctionKeyword: + case SyntaxKind.FalseKeyword: + case SyntaxKind.TrueKeyword: + case SyntaxKind.TypeOfKeyword: + case SyntaxKind.IsKeyword: + case SyntaxKind.SymbolKeyword: + case SyntaxKind.ReadonlyKeyword: return { type: "atom", tag: "other", content, src }; case SyntaxKind.EndOfFileToken: return { type: "atom", tag: "other", content, src }; diff --git a/src/tags.ts b/src/tags.ts index 5d0f377..41ff175 100644 --- a/src/tags.ts +++ b/src/tags.ts @@ -50,5 +50,23 @@ export type list_tag = | "array_pattern" | "object_pattern" | "slice" + | "function_type" + | "interface_declaration" + | "module_block" + | "module_declaration" + | "type_operator" + | "mapped_type" + | "type_reference" + | "call_signature" + | "array_type" + | "type_literal" + | "function_declaration" + | "indexed_access_type" + | "expression_with_type_arguments" + | "heritage_clause" + | "conditional_type" + | "method_signature" + | "parenthesized_type" + | "type_predicate" | "syntax_list" | "ERROR";