feat(swingset): defaultManagerType option in makeSwingsetController (#…

…2266)

* build(xsnap): disable experimental mxMetering

* chore(moddable): update to Jan 19

* fix(xsnap): report exceptions in timer callbacks

* fix(xsnap): protect timer callbacks from GC

ref attachment from PH Tue Feb 2 07:28:28 2021 -0800

 - don't hold arraybuffer pointer across macros
 - replace C stack report / result with xsVar(1) and xsVar(2)

* style(xsnap): test-xsnap.js lint

* test(xsnap): reject regex with non-standard hyphen

 - accept std regex range

* test(xsnap): uncaught rejections should not be silent FAILing

* fix: replay deliveries in xs-worker

* fix(xs-worker): transformTildot was missing from vat powers

  - route call to manager
  - some VatPowers type annotations, esp. transformTildot

* feat(xs-worker): vatstoreGet, Set, Delete

we get an empty JSON response for at least vatstoreSet

These are missing from SwingSet/docs/vat-worker.md

* feat(swingset): defaultManagerType option in makeSwingsetController

... and to buildVatController

Usage: WORKER_TYPE=xs-worker yarn test
   or: XSNAP_DEBUG=1 WORKER_TYPE=xs-worker yarn test
   or: XSNAP_DEBUG=1 WORKER_TYPE=xs-worker yarn test --watch

ref #2260

  - fix: comms vat must be local to use setUp
  - chore(swingset): log fallback-to-local workerType
  - chore: refine xs-worker diagnostics
  - chore(xs-worker): include vatID in log messages
  - fix(xs-worker): catch exceptions from exception handler
  - chore(swingset): use ambient process.env unless caller overrides
  - fix(swingset): proplems in types.js shown by @ts-check

* chore(swingset): note setup implies local for comms vat

packages/SwingSet/src/controller.js

@@ -44,12 +44,16 @@ export async function makeSwingsetController(
 ) {
   insistStorageAPI(hostStorage);
 
+  // Use ambient process.env only if caller did not specify.
+  const { env = process.env } = runtimeOptions;
+
   // build console early so we can add console.log to diagnose early problems
   const {
     verbose,
     debugPrefix = '',
     slogFile,
     testTrackDecref,
+    defaultManagerType = env.WORKER_TYPE || 'local',
   } = runtimeOptions;
   if (typeof Compartment === 'undefined') {
     throw Error('SES must be installed before calling makeSwingsetController');
@@ -166,7 +170,7 @@ export async function makeSwingsetController(
       name,
       stdout: 'inherit',
       stderr: 'inherit',
-      // debug: true,
+      debug: !!env.XSNAP_DEBUG,
     });
 
     const bundles = {
@@ -205,7 +209,7 @@ export async function makeSwingsetController(
     FinalizationRegistry,
   };
 
-  const kernelOptions = { verbose, testTrackDecref };
+  const kernelOptions = { verbose, testTrackDecref, defaultManagerType };
   const kernel = buildKernel(kernelEndowments, deviceEndowments, kernelOptions);
 
   if (runtimeOptions.verbose) {
@@ -298,8 +302,14 @@ export async function buildVatController(
     kernelBundles,
     debugPrefix,
     testTrackDecref,
+    defaultManagerType,
   } = runtimeOptions;
-  const actualRuntimeOptions = { verbose, debugPrefix, testTrackDecref };
+  const actualRuntimeOptions = {
+    verbose,
+    debugPrefix,
+    testTrackDecref,
+    defaultManagerType,
+  };
   const initializationOptions = { verbose, kernelBundles };
   let bootstrapResult;
   if (!swingsetIsInitialized(hostStorage)) {

packages/SwingSet/src/initializeSwingset.js

@@ -279,7 +279,11 @@ export async function initializeSwingset(
     bundle: kernelBundles.comms,
     creationOptions: {
       enablePipelining: true,
+      // The use of setup rather than buildRootObject requires
+      // a local worker. We have no plans to support setup on
+      // non-local workers any time soon.
       enableSetup: true,
+      managerType: 'local',
     },
   };
 

packages/SwingSet/src/kernel/kernel.js

@@ -104,7 +104,11 @@ export default function buildKernel(
     FinalizationRegistry,
   } = kernelEndowments;
   deviceEndowments = { ...deviceEndowments }; // copy so we can modify
-  const { verbose, testTrackDecref = false } = kernelOptions;
+  const {
+    verbose,
+    testTrackDecref = false,
+    defaultManagerType = 'local',
+  } = kernelOptions;
   const logStartup = verbose ? console.debug : () => 0;
 
   insistStorageAPI(hostStorage);
@@ -562,6 +566,7 @@ export default function buildKernel(
     startSubprocessWorkerNode,
     startXSnap,
     gcTools,
+    defaultManagerType,
   });
 
   function buildVatSyscallHandler(vatID, translators) {

packages/SwingSet/src/kernel/vatManager/factory.js

@@ -16,6 +16,7 @@ export function makeVatManagerFactory({
   startSubprocessWorkerNode,
   startXSnap,
   gcTools,
+  defaultManagerType,
 }) {
   const localFactory = makeLocalVatManagerFactory({
     allVatPowers,
@@ -44,6 +45,7 @@ export function makeVatManagerFactory({
   const xsWorkerFactory = makeXsSubprocessFactory({
     startXSnap,
     kernelKeeper,
+    allVatPowers,
     testLog: allVatPowers.testLog,
     decref: gcTools.decref,
   });
@@ -74,17 +76,30 @@ export function makeVatManagerFactory({
   // returns promise for new vatManager
   function vatManagerFactory(vatID, managerOptions) {
     validateManagerOptions(managerOptions);
-    const { managerType = 'local', setup, bundle } = managerOptions;
+    const {
+      managerType = defaultManagerType,
+      setup,
+      bundle,
+      metered,
+      enableSetup,
+    } = managerOptions;
 
-    if (managerType === 'local') {
+    if (metered && managerType !== 'local') {
+      console.warn(
+        `TODO: support metered with ${managerType}; using local as work-around`,
+      );
+    }
+    if (setup && managerType !== 'local') {
+      console.warn(
+        `TODO: stop using setup() with ${managerType}; using local as work-around`,
+      );
+    }
+    if (managerType === 'local' || metered || enableSetup) {
       if (setup) {
         return localFactory.createFromSetup(vatID, setup, managerOptions);
       }
       return localFactory.createFromBundle(vatID, bundle, managerOptions);
     }
-    // 'setup' based vats must be local. TODO: stop using 'setup' in vats,
-    // but tests and comms-vat still need it
-    assert(!setup, `setup()-based vats must use a local Manager`);
 
     if (managerType === 'nodeWorker') {
       return nodeWorkerFactory.createFromBundle(vatID, bundle, managerOptions);

packages/SwingSet/src/kernel/vatManager/manager-subprocess-xsnap.js

@@ -13,8 +13,9 @@ const decoder = new TextDecoder();
 
 /**
  * @param {{
- *   startXSnap: (name: string, handleCommand: SyncHandler) => { worker: XSnap, bundles: Record<string, ExportBundle> },
+ *   allVatPowers: VatPowers,
  *   kernelKeeper: KernelKeeper,
+ *   startXSnap: (name: string, handleCommand: SyncHandler) => { worker: XSnap, bundles: Record<string, ExportBundle> },
  *   testLog: (...args: unknown[]) => void,
  *   decref: (vatID: unknown, vref: unknown, count: number) => void,
  * }} tools
@@ -28,8 +29,9 @@ const decoder = new TextDecoder();
  * @typedef { [unknown, ...unknown[]] } Tagged
  */
 export function makeXsSubprocessFactory({
-  startXSnap,
+  allVatPowers: { transformTildot },
   kernelKeeper,
+  startXSnap,
   testLog,
   decref,
 }) {
@@ -39,10 +41,13 @@ export function makeXsSubprocessFactory({
    * @param { ManagerOptions } managerOptions
    */
   async function createFromBundle(vatID, bundle, managerOptions) {
-    parentLog('createFromBundle', { vatID });
+    parentLog(vatID, 'createFromBundle', { vatID });
     const { vatParameters, virtualObjectCacheSize } = managerOptions;
-    assert(!managerOptions.metered, 'not supported yet');
-    assert(!managerOptions.enableSetup, 'not supported at all');
+    assert(!managerOptions.metered, 'xs-worker: metered not supported yet');
+    assert(
+      !managerOptions.enableSetup,
+      'xs-worker: enableSetup not supported at all',
+    );
     if (managerOptions.enableInternalMetering) {
       // TODO: warn+ignore, rather than throw, because the kernel enables it
       // for all vats, because the Spawner still needs it. When the kernel
@@ -72,10 +77,10 @@ export function makeXsSubprocessFactory({
 
     /** @type { (item: Tagged) => unknown } */
     function handleUpstream([type, ...args]) {
-      parentLog(`handleUpstream`, type, args.length);
+      parentLog(vatID, `handleUpstream`, type, args.length);
       switch (type) {
         case 'syscall': {
-          parentLog(`syscall`, args);
+          parentLog(vatID, `syscall`, args[0], args.length);
           const [scTag, ...vatSyscallArgs] = args;
           return handleSyscall([scTag, ...vatSyscallArgs]);
         }
@@ -97,22 +102,32 @@ export function makeXsSubprocessFactory({
           vatDecref(vref, count);
           return ['OK'];
         }
+        case 'transformTildot': {
+          const [extjs] = args;
+          try {
+            const stdjs = transformTildot(extjs);
+            // console.log({ extjs, stdjs });
+            return ['ok', stdjs];
+          } catch (err) {
+            return ['err', err.message];
+          }
+        }
         default:
           throw new Error(`unrecognized uplink message ${type}`);
       }
     }
 
     /** @type { (msg: Uint8Array) => Uint8Array } */
     function handleCommand(msg) {
-      parentLog('handleCommand', { length: msg.byteLength });
+      // parentLog('handleCommand', { length: msg.byteLength });
       const tagged = handleUpstream(JSON.parse(decoder.decode(msg)));
       return encoder.encode(JSON.stringify(tagged));
     }
 
     // start the worker and establish a connection
     const { worker, bundles } = startXSnap(`${vatID}`, handleCommand);
     for await (const [it, superCode] of Object.entries(bundles)) {
-      parentLog('bundle', it);
+      parentLog(vatID, 'eval bundle', it);
       assert(
         superCode.moduleFormat === 'getExport',
         details`${it} unexpected: ${superCode.moduleFormat}`,
@@ -122,15 +137,15 @@ export function makeXsSubprocessFactory({
 
     /** @type { (item: Tagged) => Promise<Tagged> } */
     async function issueTagged(item) {
-      parentLog('issueTagged', item[0]);
+      parentLog(item[0], '...', item.length - 1);
       const txt = await worker.issueStringCommand(JSON.stringify(item));
       const reply = JSON.parse(txt);
       assert(Array.isArray(reply));
       const [tag, ...rest] = reply;
       return [tag, ...rest];
     }
 
-    parentLog(`instructing worker to load bundle..`);
+    parentLog(vatID, `instructing worker to load bundle..`);
     const bundleReply = await issueTagged([
       'setBundle',
       vatID,
@@ -139,16 +154,18 @@ export function makeXsSubprocessFactory({
       virtualObjectCacheSize,
     ]);
     if (bundleReply[0] === 'dispatchReady') {
-      parentLog(`bundle loaded. dispatch ready.`);
+      parentLog(vatID, `bundle loaded. dispatch ready.`);
     } else {
       throw new Error(`failed to setBundle: ${bundleReply}`);
     }
 
     /** @type { (item: Tagged) => Promise<Tagged> } */
     async function deliver(delivery) {
-      parentLog(`sending delivery`, delivery);
+      parentLog(vatID, `sending delivery`, delivery);
+      transcriptManager.startDispatch(delivery);
       const result = await issueTagged(['deliver', ...delivery]);
-      parentLog(`deliverDone`, result[0], result.length);
+      parentLog(vatID, `deliverDone`, result[0], result.length);
+      transcriptManager.finishDispatch();
       return result;
     }
 
@@ -175,7 +192,7 @@ export function makeXsSubprocessFactory({
       shutdown,
     });
 
-    parentLog('manager', Object.keys(manager));
+    parentLog(vatID, 'manager', Object.keys(manager));
     return manager;
   }
 

packages/SwingSet/src/kernel/vatManager/supervisor-subprocess-xsnap.js

@@ -14,7 +14,7 @@ const decoder = new TextDecoder();
 function workerLog(first, ...args) {
   // @ts-ignore
   // eslint-disable-next-line
-  // print(`---worker: ${first}`, ...args);
+  // console.log(`---worker: ${first}`, ...args);
 }
 
 workerLog(`supervisor started`);
@@ -30,7 +30,7 @@ function managerPort(issueCommand) {
   const encode = item => encoder.encode(JSON.stringify(item)).buffer;
 
   /** @type { (msg: ArrayBuffer) => any } */
-  const decodeData = msg => JSON.parse(decoder.decode(msg));
+  const decodeData = msg => JSON.parse(decoder.decode(msg) || 'null');
 
   /** @type { (msg: ArrayBuffer) => Tagged } */
   function decode(msg) {
@@ -59,6 +59,7 @@ function managerPort(issueCommand) {
      * @template T
      */
     handlerFrom(f) {
+      const lastResort = encoder.encode(`exception from ${f.name}`).buffer;
       return msg => {
         const report = {};
         f(decode(msg))
@@ -67,7 +68,10 @@ function managerPort(issueCommand) {
             report.result = encode(item);
           })
           .catch(err => {
-            report.result = encode(['err', err.message]);
+            report.result = encode(['err', f.name, err.message]);
+          })
+          .catch(_err => {
+            report.result = lastResort;
           });
         return report;
       };
@@ -82,12 +86,14 @@ function managerPort(issueCommand) {
 function runAndWait(f, errmsg) {
   Promise.resolve()
     .then(f)
-    .then(undefined, err => workerLog(`doProcess: ${errmsg}:`, err));
+    .then(undefined, err => {
+      workerLog(`doProcess: ${errmsg}:`, err.message);
+    });
   return waitUntilQuiescent();
 }
 
 /**
- * @param { ReturnType<managerPort> } port
+ * @param { ReturnType<typeof managerPort> } port
  */
 function makeWorker(port) {
   /** @type { Record<string, (...args: unknown[]) => void> | null } */
@@ -141,6 +147,14 @@ function makeWorker(port) {
     return harden(cons);
   }
 
+  /** @type { (extSrc: string) => string } */
+  function transformTildot(extSrc) {
+    const reply = port.call(['transformTildot', extSrc]);
+    assert(Array.isArray(reply));
+    if (reply[0] === 'err') throw new Error(reply[1]);
+    return reply[1];
+  }
+
   /**
    * @param {unknown} vatID
    * @param {unknown} bundle
@@ -167,12 +181,16 @@ function makeWorker(port) {
       callNow: (...args) => doSyscall(['callNow', ...args]),
       subscribe: (...args) => doSyscall(['subscribe', ...args]),
       resolve: (...args) => doSyscall(['resolve', ...args]),
+      vatstoreGet: (...args) => doSyscall(['vatstoreGet', ...args]),
+      vatstoreSet: (...args) => doSyscall(['vatstoreSet', ...args]),
+      vatstoreDelete: (...args) => doSyscall(['vatstoreDelete', ...args]),
     });
 
     const vatPowers = {
       Remotable,
       getInterfaceOf,
       makeMarshal,
+      transformTildot,
       testLog: (...args) => port.send(['testLog', ...args]),
     };