From 73cf66e831eef3a1a081ee818a59992dd530dc69 Mon Sep 17 00:00:00 2001 From: Abdulaziz Ghuloum Date: Sun, 22 Dec 2024 04:22:49 +0300 Subject: [PATCH 1/4] saving and loading import dependencies --- src/library-manager.ts | 88 ++++++++++++++++++++++++++------- src/preexpand-helpers.ts | 1 + test-project/.rts/main.rts.json | 8 ++- test-project/.rts/mod.rts.json | 3 +- 4 files changed, 81 insertions(+), 19 deletions(-) diff --git a/src/library-manager.ts b/src/library-manager.ts index 326bc36..03eda69 100644 --- a/src/library-manager.ts +++ b/src/library-manager.ts @@ -19,10 +19,11 @@ import { Binding, CompilationUnit, Context, Loc, Rib } from "./syntax-structures import stringify from "json-stringify-pretty-compact"; import { init_global_context } from "./global-module"; -const cookie = "rewrite-ts-013"; +const cookie = "rewrite-ts-016"; type module_state = | { type: "initial" } + | { type: "initializing"; promise: Promise } | { type: "stale"; cid: string; pkg: Package; pkg_relative_path: string } | { type: "fresh"; @@ -32,6 +33,7 @@ type module_state = exported_identifiers: { [name: string]: import_resolution[] }; context: Context; unit: CompilationUnit; + mtime: number; } | { type: "error"; reason: string }; @@ -64,6 +66,7 @@ class RtsModule implements imported_module { return this.initialize().then(() => this.ensureUpToDate()); case "stale": return this.recompile().then(() => this.ensureUpToDate()); + case "initializing": case "fresh": case "error": return; @@ -90,11 +93,11 @@ class RtsModule implements imported_module { return this.path + ".ts"; } - async initialize() { + private async do_initialize() { assert(this.state.type === "initial"); - console.log(`initializing ${this.path}`); const [pkg, pkg_relative_path] = await this.library_manager.findPackage(this.path); const cid = `${pkg_relative_path} ${pkg.name} ${pkg.version}`; + console.log(`initializing ${cid}`); const json_path = this.get_json_path(); const json_mtime = await mtime(json_path); const my_mtime = await mtime(this.path); @@ -111,7 +114,28 @@ class RtsModule implements imported_module { assert(json.cid === cid); assert(json.exported_identifiers !== undefined, `no exported_identifiers in ${json_path}`); assert(json.context !== undefined, `no exported_identifiers in ${json_path}`); - console.error("TODO: check dependencies"); + assert(json.imports); + const imported_modules: imported_module[] = await Promise.all( + json.imports.map( + (x: { pkg: { name: string; version: string }; pkg_relative_path: string }) => { + const { + pkg: { name, version }, + pkg_relative_path, + } = x; + const pkg = this.library_manager.get_package(name, version); + assert(pkg !== undefined); + const path = join(pkg.dir, pkg_relative_path); + const mod = this.library_manager.ensureUpToDate(path); + return mod; + }, + ), + ); + if (imported_modules.some((x) => x.get_mtime() > my_mtime)) { + this.state = { type: "stale", cid, pkg, pkg_relative_path }; + return; + } + this.imported_modules = imported_modules; + console.log(`up to date ${cid}`); this.state = { type: "fresh", cid, @@ -120,12 +144,25 @@ class RtsModule implements imported_module { exported_identifiers: json.exported_identifiers, context: json.context, unit: json.unit, + mtime: my_mtime, }; } + async initialize() { + switch (this.state.type) { + case "initial": { + const promise = this.do_initialize(); + this.state = { type: "initializing", promise }; + return promise; + } + case "initializing": + return this.state.promise; + } + } + async recompile() { assert(this.state.type === "stale"); - console.log(`expanding ${this.state.cid}`); + console.log(`recompiling ${this.state.cid} ...`); const code = await fs.readFile(this.path, { encoding: "utf-8" }); const my_pkg = this.state.pkg; const my_path = this.state.pkg_relative_path; @@ -188,9 +225,14 @@ class RtsModule implements imported_module { const json_content = { cid: this.state.cid, cookie, + imports: this.imported_modules.map((x) => { + const [pkg, path] = x.get_pkg_and_path(); + return { pkg: { name: pkg.name, version: pkg.version }, pkg_relative_path: path }; + }), exported_identifiers, context, unit, + mtime: Date.now(), }; const code_path = this.get_generated_code_absolute_path(); await fs.mkdir(dirname(code_path), { recursive: true }); @@ -231,6 +273,15 @@ class RtsModule implements imported_module { } } + get_mtime(): number { + switch (this.state.type) { + case "fresh": + return this.state.mtime; + default: + throw new Error(`invalid state`); + } + } + find_module_by_cid(cid: string): imported_module | undefined { if (this.get_cid() === cid) return this; for (const m of this.imported_modules) { @@ -265,8 +316,11 @@ class RtsModule implements imported_module { return [this.state.pkg, this.state.pkg_relative_path]; } - private get_imported_modules_for_path(import_path: string, loc: Loc): imported_module { - const mod = this.library_manager.do_import(import_path, this.path); + private async get_imported_modules_for_path( + import_path: string, + loc: Loc, + ): Promise { + const mod = await this.library_manager.do_import(import_path, this.path); if (this.imported_modules.includes(mod)) return mod; const self = this; function check(mod: imported_module) { @@ -305,7 +359,8 @@ export class LibraryManager { private libs: string[]; private global_unit: CompilationUnit; private global_context: Context; - private modules: { [path: string]: imported_module } = {}; + private modules_by_path: { [path: string]: imported_module } = {}; + private modules_by_cid: { [cid: string]: imported_module } = {}; private packages: { [dir: string]: Package } = {}; constructor(patterns: { [k: string]: AST }, globals: string[], libs: string[]) { @@ -315,19 +370,18 @@ export class LibraryManager { this.global_context = global_context; } - private get_or_create_module(path: string) { - const mod = (this.modules[path] ??= new RtsModule( - path, - this, - this.libs, - this.global_unit, - this.global_context, - )); + private async get_or_create_module(path: string) { + const existing = this.modules_by_path[path]; + if (existing) return existing; + const mod = new RtsModule(path, this, this.libs, this.global_unit, this.global_context); + this.modules_by_path[path] = mod; + await mod.initialize(); + this.modules_by_cid[mod.get_cid()] = mod; return mod; } async ensureUpToDate(path: string) { - const mod = this.get_or_create_module(path); + const mod = await this.get_or_create_module(path); await mod.ensureUpToDate(); return mod; } diff --git a/src/preexpand-helpers.ts b/src/preexpand-helpers.ts index a65c964..994a984 100644 --- a/src/preexpand-helpers.ts +++ b/src/preexpand-helpers.ts @@ -17,6 +17,7 @@ export type imported_module = { resolve_label(name: string): Promise; get_pkg_and_path(): [{ name: string; version: string }, string]; resolve_rib: (rib_id: string) => Rib; + get_mtime(): number; }; export type manager = { diff --git a/test-project/.rts/main.rts.json b/test-project/.rts/main.rts.json index 4a1f1db..0aaf239 100644 --- a/test-project/.rts/main.rts.json +++ b/test-project/.rts/main.rts.json @@ -1,6 +1,12 @@ { "cid": "test-project/main.rts rewrite-ts-visualized 0.0.0", - "cookie": "rewrite-ts-013", + "cookie": "rewrite-ts-016", + "imports": [ + { + "pkg": {"name": "rewrite-ts-visualized", "version": "0.0.0"}, + "pkg_relative_path": "test-project/mod.rts" + } + ], "exported_identifiers": {}, "context": {"l1": {"type": "lexical", "name": "y_1"}}, "unit": { diff --git a/test-project/.rts/mod.rts.json b/test-project/.rts/mod.rts.json index 491c564..74f50cf 100644 --- a/test-project/.rts/mod.rts.json +++ b/test-project/.rts/mod.rts.json @@ -1,6 +1,7 @@ { "cid": "test-project/mod.rts rewrite-ts-visualized 0.0.0", - "cookie": "rewrite-ts-013", + "cookie": "rewrite-ts-016", + "imports": [], "exported_identifiers": { "x": [ { From e34f5372e8e09e2e9ead5fc6a52882efbacee21d Mon Sep 17 00:00:00 2001 From: Abdulaziz Ghuloum Date: Sun, 22 Dec 2024 04:51:11 +0300 Subject: [PATCH 2/4] avoiding reinitialization on concurrent loading --- src/library-manager.ts | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/src/library-manager.ts b/src/library-manager.ts index 03eda69..2849a9a 100644 --- a/src/library-manager.ts +++ b/src/library-manager.ts @@ -64,9 +64,10 @@ class RtsModule implements imported_module { switch (this.state.type) { case "initial": return this.initialize().then(() => this.ensureUpToDate()); + case "initializing": + return this.state.promise.then(() => this.ensureUpToDate()); case "stale": return this.recompile().then(() => this.ensureUpToDate()); - case "initializing": case "fresh": case "error": return; @@ -102,7 +103,7 @@ class RtsModule implements imported_module { const json_mtime = await mtime(json_path); const my_mtime = await mtime(this.path); assert(my_mtime !== undefined); - if (my_mtime >= (json_mtime ?? 0)) { + if (json_mtime === undefined || my_mtime >= json_mtime) { this.state = { type: "stale", cid, pkg, pkg_relative_path }; return; } @@ -115,9 +116,9 @@ class RtsModule implements imported_module { assert(json.exported_identifiers !== undefined, `no exported_identifiers in ${json_path}`); assert(json.context !== undefined, `no exported_identifiers in ${json_path}`); assert(json.imports); - const imported_modules: imported_module[] = await Promise.all( - json.imports.map( - (x: { pkg: { name: string; version: string }; pkg_relative_path: string }) => { + const imported_modules = await Promise.all( + (json.imports as { pkg: { name: string; version: string }; pkg_relative_path: string }[]).map( + (x) => { const { pkg: { name, version }, pkg_relative_path, @@ -130,7 +131,7 @@ class RtsModule implements imported_module { }, ), ); - if (imported_modules.some((x) => x.get_mtime() > my_mtime)) { + if (imported_modules.some((x) => x.get_mtime() > json_mtime)) { this.state = { type: "stale", cid, pkg, pkg_relative_path }; return; } @@ -144,7 +145,7 @@ class RtsModule implements imported_module { exported_identifiers: json.exported_identifiers, context: json.context, unit: json.unit, - mtime: my_mtime, + mtime: json_mtime, }; } @@ -232,14 +233,14 @@ class RtsModule implements imported_module { exported_identifiers, context, unit, - mtime: Date.now(), }; const code_path = this.get_generated_code_absolute_path(); await fs.mkdir(dirname(code_path), { recursive: true }); await fs.writeFile(code_path, await pprint(loc)); await fs.writeFile(this.get_proxy_path(), proxy_code); + const mtime = Date.now(); await fs.writeFile(this.get_json_path(), stringify(json_content)); - this.state = { ...this.state, type: "fresh", ...json_content }; + this.state = { ...this.state, type: "fresh", ...json_content, mtime }; } catch (error) { if (error instanceof StxError) { await print_stx_error(error, this.library_manager); @@ -320,7 +321,7 @@ class RtsModule implements imported_module { import_path: string, loc: Loc, ): Promise { - const mod = await this.library_manager.do_import(import_path, this.path); + const mod = this.library_manager.do_import(import_path, this.path); if (this.imported_modules.includes(mod)) return mod; const self = this; function check(mod: imported_module) { @@ -359,8 +360,7 @@ export class LibraryManager { private libs: string[]; private global_unit: CompilationUnit; private global_context: Context; - private modules_by_path: { [path: string]: imported_module } = {}; - private modules_by_cid: { [cid: string]: imported_module } = {}; + private modules: { [path: string]: imported_module } = {}; private packages: { [dir: string]: Package } = {}; constructor(patterns: { [k: string]: AST }, globals: string[], libs: string[]) { @@ -370,18 +370,16 @@ export class LibraryManager { this.global_context = global_context; } - private async get_or_create_module(path: string) { - const existing = this.modules_by_path[path]; + private get_or_create_module(path: string) { + const existing = this.modules[path]; if (existing) return existing; const mod = new RtsModule(path, this, this.libs, this.global_unit, this.global_context); - this.modules_by_path[path] = mod; - await mod.initialize(); - this.modules_by_cid[mod.get_cid()] = mod; + this.modules[path] = mod; return mod; } async ensureUpToDate(path: string) { - const mod = await this.get_or_create_module(path); + const mod = this.get_or_create_module(path); await mod.ensureUpToDate(); return mod; } From edbc03951d7816edac17a281589e094e8c16dba4 Mon Sep 17 00:00:00 2001 From: Abdulaziz Ghuloum Date: Sun, 22 Dec 2024 13:21:39 +0300 Subject: [PATCH 3/4] detecting and recompiling on file change --- rtsc/watch.ts | 11 +++--- src/library-manager.ts | 83 +++++++++++++++++++++++++++++++++------- src/preexpand-helpers.ts | 1 + 3 files changed, 76 insertions(+), 19 deletions(-) diff --git a/rtsc/watch.ts b/rtsc/watch.ts index d3c3e28..815afd9 100755 --- a/rtsc/watch.ts +++ b/rtsc/watch.ts @@ -7,10 +7,6 @@ import { get_globals } from "../src/global-module.ts"; import { core_patterns } from "../src/syntax-core-patterns.ts"; import { parse } from "../src/parse.ts"; -const globals = get_globals("es2024.full"); -const patterns = core_patterns(parse); -const library_manager = new LibraryManager(patterns, globals, ["es2024.full"]); - const watch_reporter: TS.WatchStatusReporter = (diagnostics, _newline, _options, error_count) => { console.log(`rtsc: ${diagnostics.messageText}`); if (error_count) { @@ -68,7 +64,8 @@ function check_path(path: string) { if (path.endsWith(suffix)) { const module_dir = dirname(path); const module_name = basename(path, suffix) + ".rts"; - const rts_file = join(module_dir, module_name); + if (host.realpath === undefined) throw new Error("host has no real path"); + const rts_file = host.realpath(join(module_dir, module_name)); if (fileExists(rts_file)) { library_manager.ensureUpToDate(rts_file); } @@ -96,4 +93,8 @@ host.watchDirectory = (path, callback, recursive, options) => { return watchDirectory(path, callback, recursive, options); }; +const globals = get_globals("es2024.full"); +const patterns = core_patterns(parse); +const library_manager = new LibraryManager(patterns, globals, ["es2024.full"], host); + const prog = TS.createWatchProgram(host); diff --git a/src/library-manager.ts b/src/library-manager.ts index 2849a9a..37e3e74 100644 --- a/src/library-manager.ts +++ b/src/library-manager.ts @@ -25,6 +25,13 @@ type module_state = | { type: "initial" } | { type: "initializing"; promise: Promise } | { type: "stale"; cid: string; pkg: Package; pkg_relative_path: string } + | { + type: "compiling"; + cid: string; + pkg: Package; + pkg_relative_path: string; + promise: Promise; + } | { type: "fresh"; cid: string; @@ -68,6 +75,8 @@ class RtsModule implements imported_module { return this.state.promise.then(() => this.ensureUpToDate()); case "stale": return this.recompile().then(() => this.ensureUpToDate()); + case "compiling": + return this.state.promise.then(() => this.ensureUpToDate()); case "fresh": case "error": return; @@ -95,7 +104,7 @@ class RtsModule implements imported_module { } private async do_initialize() { - assert(this.state.type === "initial"); + assert(this.state.type === "initial", `invalid state ${this.state.type}`); const [pkg, pkg_relative_path] = await this.library_manager.findPackage(this.path); const cid = `${pkg_relative_path} ${pkg.name} ${pkg.version}`; console.log(`initializing ${cid}`); @@ -161,17 +170,18 @@ class RtsModule implements imported_module { } } - async recompile() { - assert(this.state.type === "stale"); - console.log(`recompiling ${this.state.cid} ...`); + async do_recompile() { + const state = this.state; + assert(state.type === "stale"); + console.log(`recompiling ${state.cid} ...`); const code = await fs.readFile(this.path, { encoding: "utf-8" }); - const my_pkg = this.state.pkg; - const my_path = this.state.pkg_relative_path; + const my_pkg = state.pkg; + const my_path = state.pkg_relative_path; const source_file: source_file = { package: { name: my_pkg.name, version: my_pkg.version }, path: my_path, }; - const [_loc0, expand] = initial_step(parse(code, source_file), this.state.cid, this.libs); + const [_loc0, expand] = initial_step(parse(code, source_file), state.cid, this.libs); try { const helpers: preexpand_helpers = { manager: { @@ -220,11 +230,11 @@ class RtsModule implements imported_module { ); const exported_identifiers = get_exported_identifiers_from_rib( modular.explicit, - this.state.cid, + state.cid, context, ); const json_content = { - cid: this.state.cid, + cid: state.cid, cookie, imports: this.imported_modules.map((x) => { const [pkg, path] = x.get_pkg_and_path(); @@ -240,14 +250,45 @@ class RtsModule implements imported_module { await fs.writeFile(this.get_proxy_path(), proxy_code); const mtime = Date.now(); await fs.writeFile(this.get_json_path(), stringify(json_content)); - this.state = { ...this.state, type: "fresh", ...json_content, mtime }; + this.state = { ...state, type: "fresh", ...json_content, mtime }; + console.log(`up to date ${state.cid}`); } catch (error) { + this.state = { type: "error", reason: String(error) }; if (error instanceof StxError) { await print_stx_error(error, this.library_manager); } else { console.error(error); } - this.state = { type: "error", reason: String(error) }; + } + } + + async recompile() { + const state = this.state; + switch (state.type) { + case "stale": { + const promise = this.do_recompile(); + this.state = { ...state, type: "compiling", promise }; + return promise; + } + case "compiling": + return state.promise; + } + } + + async file_changed(): Promise { + const t = await mtime(this.path); + await this.ensureUpToDate(); + const state = this.state; + switch (state.type) { + case "fresh": { + if (t && t > state.mtime) { + this.state = { ...state, type: "stale" }; + this.ensureUpToDate(); + } + return; + } + default: + throw new Error(`invalid state? '${state.type}'`); } } @@ -264,13 +305,13 @@ class RtsModule implements imported_module { } get_cid(): string { - //await this.ensureUpToDate(); switch (this.state.type) { case "fresh": case "stale": + case "compiling": return this.state.cid; default: - throw new Error(`invalid state`); + throw new Error(`invalid state ${this.state.type}`); } } @@ -356,15 +397,25 @@ class Package { } } +type watcher = { + close: () => void; +}; + +type host = { + watchFile: (path: string, callback: (path: string) => void) => watcher; +}; + export class LibraryManager { private libs: string[]; private global_unit: CompilationUnit; private global_context: Context; private modules: { [path: string]: imported_module } = {}; private packages: { [dir: string]: Package } = {}; + private host: host; - constructor(patterns: { [k: string]: AST }, globals: string[], libs: string[]) { + constructor(patterns: { [k: string]: AST }, globals: string[], libs: string[], host: host) { this.libs = libs; + this.host = host; const [global_unit, global_context] = init_global_context(patterns, globals); this.global_unit = global_unit; this.global_context = global_context; @@ -375,6 +426,10 @@ export class LibraryManager { if (existing) return existing; const mod = new RtsModule(path, this, this.libs, this.global_unit, this.global_context); this.modules[path] = mod; + const watcher = this.host.watchFile(path, (p) => { + if (p !== path) return; + mod.file_changed(); + }); return mod; } diff --git a/src/preexpand-helpers.ts b/src/preexpand-helpers.ts index 994a984..06f7ee2 100644 --- a/src/preexpand-helpers.ts +++ b/src/preexpand-helpers.ts @@ -18,6 +18,7 @@ export type imported_module = { get_pkg_and_path(): [{ name: string; version: string }, string]; resolve_rib: (rib_id: string) => Rib; get_mtime(): number; + file_changed(): Promise; }; export type manager = { From 9fbdd522d2a2fe2e73bd21a91f74c143ad3cc095 Mon Sep 17 00:00:00 2001 From: Abdulaziz Ghuloum Date: Sun, 22 Dec 2024 14:04:52 +0300 Subject: [PATCH 4/4] reverse dependencies for recompiling modules when their imports change --- src/library-manager.ts | 24 ++++++++++++++++++++++-- src/preexpand-helpers.ts | 2 ++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/src/library-manager.ts b/src/library-manager.ts index 37e3e74..10b4da5 100644 --- a/src/library-manager.ts +++ b/src/library-manager.ts @@ -52,6 +52,7 @@ class RtsModule implements imported_module { private global_unit: CompilationUnit; private global_context: Context; public imported_modules: imported_module[] = []; + public dependant_modules: imported_module[] = []; constructor( path: string, @@ -145,6 +146,7 @@ class RtsModule implements imported_module { return; } this.imported_modules = imported_modules; + imported_modules.forEach((x) => x.dependant_modules.push(this)); console.log(`up to date ${cid}`); this.state = { type: "fresh", @@ -275,6 +277,24 @@ class RtsModule implements imported_module { } } + async force_recompile() { + const state = this.state; + await this.ensureUpToDate(); + assert(state.type === "fresh"); + this.state = { ...state, type: "stale" }; + const dependant_modules = this.dependant_modules; + dependant_modules.forEach( + (x) => (x.imported_modules = x.imported_modules.filter((x) => x !== this)), + ); + this.dependant_modules = []; + this.imported_modules.forEach( + (x) => (x.dependant_modules = x.dependant_modules.filter((x) => x !== this)), + ); + this.imported_modules = []; + dependant_modules.forEach((x) => x.force_recompile()); + this.ensureUpToDate(); + } + async file_changed(): Promise { const t = await mtime(this.path); await this.ensureUpToDate(); @@ -282,8 +302,7 @@ class RtsModule implements imported_module { switch (state.type) { case "fresh": { if (t && t > state.mtime) { - this.state = { ...state, type: "stale" }; - this.ensureUpToDate(); + this.force_recompile(); } return; } @@ -373,6 +392,7 @@ class RtsModule implements imported_module { } check(mod); this.imported_modules.push(mod); + mod.dependant_modules.push(self); return mod; } diff --git a/src/preexpand-helpers.ts b/src/preexpand-helpers.ts index 06f7ee2..c1d5add 100644 --- a/src/preexpand-helpers.ts +++ b/src/preexpand-helpers.ts @@ -10,6 +10,7 @@ export type import_resolution = { export type imported_module = { imported_modules: imported_module[]; + dependant_modules: imported_module[]; resolve_exported_identifier: (name: string, loc: Loc) => Promise; ensureUpToDate(): Promise; get_cid(): string; @@ -19,6 +20,7 @@ export type imported_module = { resolve_rib: (rib_id: string) => Rib; get_mtime(): number; file_changed(): Promise; + force_recompile(): Promise; }; export type manager = {