saving and loading import dependencies

rtsc/watch.ts

@@ -7,10 +7,6 @@ import { get_globals } from "../src/global-module.ts";
 import { core_patterns } from "../src/syntax-core-patterns.ts";
 import { parse } from "../src/parse.ts";
 
-const globals = get_globals("es2024.full");
-const patterns = core_patterns(parse);
-const library_manager = new LibraryManager(patterns, globals, ["es2024.full"]);
-
 const watch_reporter: TS.WatchStatusReporter = (diagnostics, _newline, _options, error_count) => {
   console.log(`rtsc: ${diagnostics.messageText}`);
   if (error_count) {
@@ -68,7 +64,8 @@ function check_path(path: string) {
   if (path.endsWith(suffix)) {
     const module_dir = dirname(path);
     const module_name = basename(path, suffix) + ".rts";
-    const rts_file = join(module_dir, module_name);
+    if (host.realpath === undefined) throw new Error("host has no real path");
+    const rts_file = host.realpath(join(module_dir, module_name));
     if (fileExists(rts_file)) {
       library_manager.ensureUpToDate(rts_file);
     }
@@ -96,4 +93,8 @@ host.watchDirectory = (path, callback, recursive, options) => {
   return watchDirectory(path, callback, recursive, options);
 };
 
+const globals = get_globals("es2024.full");
+const patterns = core_patterns(parse);
+const library_manager = new LibraryManager(patterns, globals, ["es2024.full"], host);
+
 const prog = TS.createWatchProgram(host);

src/library-manager.ts

@@ -19,11 +19,19 @@ import { Binding, CompilationUnit, Context, Loc, Rib } from "./syntax-structures
 import stringify from "json-stringify-pretty-compact";
 import { init_global_context } from "./global-module";
 
-const cookie = "rewrite-ts-013";
+const cookie = "rewrite-ts-016";
 
 type module_state =
   | { type: "initial" }
+  | { type: "initializing"; promise: Promise<void> }
   | { type: "stale"; cid: string; pkg: Package; pkg_relative_path: string }
+  | {
+      type: "compiling";
+      cid: string;
+      pkg: Package;
+      pkg_relative_path: string;
+      promise: Promise<void>;
+    }
   | {
       type: "fresh";
       cid: string;
@@ -32,6 +40,7 @@ type module_state =
       exported_identifiers: { [name: string]: import_resolution[] };
       context: Context;
       unit: CompilationUnit;
+      mtime: number;
     }
   | { type: "error"; reason: string };
 
@@ -43,6 +52,7 @@ class RtsModule implements imported_module {
   private global_unit: CompilationUnit;
   private global_context: Context;
   public imported_modules: imported_module[] = [];
+  public dependant_modules: imported_module[] = [];
 
   constructor(
     path: string,
@@ -62,8 +72,12 @@ class RtsModule implements imported_module {
     switch (this.state.type) {
       case "initial":
         return this.initialize().then(() => this.ensureUpToDate());
+      case "initializing":
+        return this.state.promise.then(() => this.ensureUpToDate());
       case "stale":
         return this.recompile().then(() => this.ensureUpToDate());
+      case "compiling":
+        return this.state.promise.then(() => this.ensureUpToDate());
       case "fresh":
       case "error":
         return;
@@ -90,16 +104,16 @@ class RtsModule implements imported_module {
     return this.path + ".ts";
   }
 
-  async initialize() {
-    assert(this.state.type === "initial");
-    console.log(`initializing ${this.path}`);
+  private async do_initialize() {
+    assert(this.state.type === "initial", `invalid state ${this.state.type}`);
     const [pkg, pkg_relative_path] = await this.library_manager.findPackage(this.path);
     const cid = `${pkg_relative_path} ${pkg.name} ${pkg.version}`;
+    console.log(`initializing ${cid}`);
     const json_path = this.get_json_path();
     const json_mtime = await mtime(json_path);
     const my_mtime = await mtime(this.path);
     assert(my_mtime !== undefined);
-    if (my_mtime >= (json_mtime ?? 0)) {
+    if (json_mtime === undefined || my_mtime >= json_mtime) {
       this.state = { type: "stale", cid, pkg, pkg_relative_path };
       return;
     }
@@ -111,7 +125,29 @@ class RtsModule implements imported_module {
     assert(json.cid === cid);
     assert(json.exported_identifiers !== undefined, `no exported_identifiers in ${json_path}`);
     assert(json.context !== undefined, `no exported_identifiers in ${json_path}`);
-    console.error("TODO: check dependencies");
+    assert(json.imports);
+    const imported_modules = await Promise.all(
+      (json.imports as { pkg: { name: string; version: string }; pkg_relative_path: string }[]).map(
+        (x) => {
+          const {
+            pkg: { name, version },
+            pkg_relative_path,
+          } = x;
+          const pkg = this.library_manager.get_package(name, version);
+          assert(pkg !== undefined);
+          const path = join(pkg.dir, pkg_relative_path);
+          const mod = this.library_manager.ensureUpToDate(path);
+          return mod;
+        },
+      ),
+    );
+    if (imported_modules.some((x) => x.get_mtime() > json_mtime)) {
+      this.state = { type: "stale", cid, pkg, pkg_relative_path };
+      return;
+    }
+    this.imported_modules = imported_modules;
+    imported_modules.forEach((x) => x.dependant_modules.push(this));
+    console.log(`up to date ${cid}`);
     this.state = {
       type: "fresh",
       cid,
@@ -120,20 +156,34 @@ class RtsModule implements imported_module {
       exported_identifiers: json.exported_identifiers,
       context: json.context,
       unit: json.unit,
+      mtime: json_mtime,
     };
   }
 
-  async recompile() {
-    assert(this.state.type === "stale");
-    console.log(`expanding ${this.state.cid}`);
+  async initialize() {
+    switch (this.state.type) {
+      case "initial": {
+        const promise = this.do_initialize();
+        this.state = { type: "initializing", promise };
+        return promise;
+      }
+      case "initializing":
+        return this.state.promise;
+    }
+  }
+
+  async do_recompile() {
+    const state = this.state;
+    assert(state.type === "stale");
+    console.log(`recompiling ${state.cid} ...`);
     const code = await fs.readFile(this.path, { encoding: "utf-8" });
-    const my_pkg = this.state.pkg;
-    const my_path = this.state.pkg_relative_path;
+    const my_pkg = state.pkg;
+    const my_path = state.pkg_relative_path;
     const source_file: source_file = {
       package: { name: my_pkg.name, version: my_pkg.version },
       path: my_path,
     };
-    const [_loc0, expand] = initial_step(parse(code, source_file), this.state.cid, this.libs);
+    const [_loc0, expand] = initial_step(parse(code, source_file), state.cid, this.libs);
     try {
       const helpers: preexpand_helpers = {
         manager: {
@@ -182,12 +232,16 @@ class RtsModule implements imported_module {
       );
       const exported_identifiers = get_exported_identifiers_from_rib(
         modular.explicit,
-        this.state.cid,
+        state.cid,
         context,
       );
       const json_content = {
-        cid: this.state.cid,
+        cid: state.cid,
         cookie,
+        imports: this.imported_modules.map((x) => {
+          const [pkg, path] = x.get_pkg_and_path();
+          return { pkg: { name: pkg.name, version: pkg.version }, pkg_relative_path: path };
+        }),
         exported_identifiers,
         context,
         unit,
@@ -196,15 +250,64 @@ class RtsModule implements imported_module {
       await fs.mkdir(dirname(code_path), { recursive: true });
       await fs.writeFile(code_path, await pprint(loc));
       await fs.writeFile(this.get_proxy_path(), proxy_code);
+      const mtime = Date.now();
       await fs.writeFile(this.get_json_path(), stringify(json_content));
-      this.state = { ...this.state, type: "fresh", ...json_content };
+      this.state = { ...state, type: "fresh", ...json_content, mtime };
+      console.log(`up to date ${state.cid}`);
     } catch (error) {
+      this.state = { type: "error", reason: String(error) };
       if (error instanceof StxError) {
         await print_stx_error(error, this.library_manager);
       } else {
         console.error(error);
       }
-      this.state = { type: "error", reason: String(error) };
+    }
+  }
+
+  async recompile() {
+    const state = this.state;
+    switch (state.type) {
+      case "stale": {
+        const promise = this.do_recompile();
+        this.state = { ...state, type: "compiling", promise };
+        return promise;
+      }
+      case "compiling":
+        return state.promise;
+    }
+  }
+
+  async force_recompile() {
+    const state = this.state;
+    await this.ensureUpToDate();
+    assert(state.type === "fresh");
+    this.state = { ...state, type: "stale" };
+    const dependant_modules = this.dependant_modules;
+    dependant_modules.forEach(
+      (x) => (x.imported_modules = x.imported_modules.filter((x) => x !== this)),
+    );
+    this.dependant_modules = [];
+    this.imported_modules.forEach(
+      (x) => (x.dependant_modules = x.dependant_modules.filter((x) => x !== this)),
+    );
+    this.imported_modules = [];
+    dependant_modules.forEach((x) => x.force_recompile());
+    this.ensureUpToDate();
+  }
+
+  async file_changed(): Promise<void> {
+    const t = await mtime(this.path);
+    await this.ensureUpToDate();
+    const state = this.state;
+    switch (state.type) {
+      case "fresh": {
+        if (t && t > state.mtime) {
+          this.force_recompile();
+        }
+        return;
+      }
+      default:
+        throw new Error(`invalid state? '${state.type}'`);
     }
   }
 
@@ -221,11 +324,20 @@ class RtsModule implements imported_module {
   }
 
   get_cid(): string {
-    //await this.ensureUpToDate();
     switch (this.state.type) {
       case "fresh":
       case "stale":
+      case "compiling":
         return this.state.cid;
+      default:
+        throw new Error(`invalid state ${this.state.type}`);
+    }
+  }
+
+  get_mtime(): number {
+    switch (this.state.type) {
+      case "fresh":
+        return this.state.mtime;
       default:
         throw new Error(`invalid state`);
     }
@@ -265,7 +377,10 @@ class RtsModule implements imported_module {
     return [this.state.pkg, this.state.pkg_relative_path];
   }
 
-  private get_imported_modules_for_path(import_path: string, loc: Loc): imported_module {
+  private async get_imported_modules_for_path(
+    import_path: string,
+    loc: Loc,
+  ): Promise<imported_module> {
     const mod = this.library_manager.do_import(import_path, this.path);
     if (this.imported_modules.includes(mod)) return mod;
     const self = this;
@@ -277,6 +392,7 @@ class RtsModule implements imported_module {
     }
     check(mod);
     this.imported_modules.push(mod);
+    mod.dependant_modules.push(self);
     return mod;
   }
 
@@ -301,28 +417,39 @@ class Package {
   }
 }
 
+type watcher = {
+  close: () => void;
+};
+
+type host = {
+  watchFile: (path: string, callback: (path: string) => void) => watcher;
+};
+
 export class LibraryManager {
   private libs: string[];
   private global_unit: CompilationUnit;
   private global_context: Context;
   private modules: { [path: string]: imported_module } = {};
   private packages: { [dir: string]: Package } = {};
+  private host: host;
 
-  constructor(patterns: { [k: string]: AST }, globals: string[], libs: string[]) {
+  constructor(patterns: { [k: string]: AST }, globals: string[], libs: string[], host: host) {
     this.libs = libs;
+    this.host = host;
     const [global_unit, global_context] = init_global_context(patterns, globals);
     this.global_unit = global_unit;
     this.global_context = global_context;
   }
 
   private get_or_create_module(path: string) {
-    const mod = (this.modules[path] ??= new RtsModule(
-      path,
-      this,
-      this.libs,
-      this.global_unit,
-      this.global_context,
-    ));
+    const existing = this.modules[path];
+    if (existing) return existing;
+    const mod = new RtsModule(path, this, this.libs, this.global_unit, this.global_context);
+    this.modules[path] = mod;
+    const watcher = this.host.watchFile(path, (p) => {
+      if (p !== path) return;
+      mod.file_changed();
+    });
     return mod;
   }
 

src/preexpand-helpers.ts

@@ -10,13 +10,17 @@ export type import_resolution = {
 
 export type imported_module = {
   imported_modules: imported_module[];
+  dependant_modules: imported_module[];
   resolve_exported_identifier: (name: string, loc: Loc) => Promise<import_resolution[]>;
   ensureUpToDate(): Promise<void>;
   get_cid(): string;
   find_module_by_cid(cid: string): imported_module | undefined;
   resolve_label(name: string): Promise<Binding>;
   get_pkg_and_path(): [{ name: string; version: string }, string];
   resolve_rib: (rib_id: string) => Rib;
+  get_mtime(): number;
+  file_changed(): Promise<void>;
+  force_recompile(): Promise<void>;
 };
 
 export type manager = {

test-project/.rts/main.rts.json

@@ -1,6 +1,12 @@
 {
   "cid": "test-project/main.rts rewrite-ts-visualized 0.0.0",
-  "cookie": "rewrite-ts-013",
+  "cookie": "rewrite-ts-016",
+  "imports": [
+    {
+      "pkg": {"name": "rewrite-ts-visualized", "version": "0.0.0"},
+      "pkg_relative_path": "test-project/mod.rts"
+    }
+  ],
   "exported_identifiers": {},
   "context": {"l1": {"type": "lexical", "name": "y_1"}},
   "unit": {

test-project/.rts/mod.rts.json

@@ -1,6 +1,7 @@
 {
   "cid": "test-project/mod.rts rewrite-ts-visualized 0.0.0",
-  "cookie": "rewrite-ts-013",
+  "cookie": "rewrite-ts-016",
+  "imports": [],
   "exported_identifiers": {
     "x": [
       {