Add replay ability for commands

Replaying previously executed runs is a must have feature for property based testing frameworks.
Because of the specificities of commands, commands were not eligible to replay.

This commit adds the replay capabilities to commands by specifying an extra parameter when defining them (replayPath).
Please note that commands arbitraries should not be shared accross multiple runs.

Related to #251

src/check/model/ReplayPath.ts

@@ -0,0 +1,9 @@
+/** @hidden */
+export class ReplayPath {
+  static parse(replayPathStr: string): boolean[] {
+    return [...replayPathStr].map(v => v === '1');
+  }
+  static stringify(replayPath: boolean[]): string {
+    return replayPath.map(s => (s ? '1' : '0')).join('');
+  }
+}

src/check/model/commands/CommandsArbitrary.ts

@@ -8,7 +8,9 @@ import { oneof } from '../../arbitrary/OneOfArbitrary';
 import { AsyncCommand } from '../command/AsyncCommand';
 import { Command } from '../command/Command';
 import { ICommand } from '../command/ICommand';
+import { ReplayPath } from '../ReplayPath';
 import { CommandsIterable } from './CommandsIterable';
+import { CommandsSettings } from './CommandsSettings';
 import { CommandWrapper } from './CommandWrapper';
 
 /** @hidden */
@@ -17,17 +19,29 @@ class CommandsArbitrary<Model extends object, Real, RunResult, CheckAsync extend
 > {
   readonly oneCommandArb: Arbitrary<CommandWrapper<Model, Real, RunResult, CheckAsync>>;
   readonly lengthArb: ArbitraryWithShrink<number>;
-  constructor(commandArbs: Arbitrary<ICommand<Model, Real, RunResult, CheckAsync>>[], maxCommands: number) {
+  private replayPath: boolean[];
+  private replayPathPosition: number;
+  constructor(
+    commandArbs: Arbitrary<ICommand<Model, Real, RunResult, CheckAsync>>[],
+    maxCommands: number,
+    replayPathStr: string | null,
+    readonly disableReplayLog: boolean
+  ) {
     super();
     this.oneCommandArb = oneof(...commandArbs).map(c => new CommandWrapper(c));
     this.lengthArb = nat(maxCommands);
+    this.replayPath = replayPathStr !== null ? ReplayPath.parse(replayPathStr) : [];
+    this.replayPathPosition = 0;
+  }
+  private metadataForReplay() {
+    return this.disableReplayLog ? '' : `replayPath=${JSON.stringify(ReplayPath.stringify(this.replayPath))}`;
   }
   private wrapper(
     items: Shrinkable<CommandWrapper<Model, Real, RunResult, CheckAsync>>[],
     shrunkOnce: boolean,
     alwaysKeepOneCommand: boolean
   ): Shrinkable<CommandsIterable<Model, Real, RunResult, CheckAsync>> {
-    return new Shrinkable(new CommandsIterable(items.map(s => s.value_)), () =>
+    return new Shrinkable(new CommandsIterable(items.map(s => s.value_), () => this.metadataForReplay()), () =>
       this.shrinkImpl(items, shrunkOnce, alwaysKeepOneCommand).map(v => this.wrapper(v, true, true))
     );
   }
@@ -40,12 +54,32 @@ class CommandsArbitrary<Model extends object, Real, RunResult, CheckAsync extend
     }
     return this.wrapper(items, false, false);
   }
+  private filterNonExecuted(itemsRaw: Shrinkable<CommandWrapper<Model, Real, RunResult, CheckAsync>>[]) {
+    const items: typeof itemsRaw = [];
+    for (let idx = 0; idx !== itemsRaw.length; ++idx) {
+      const c = itemsRaw[idx];
+      if (this.replayPathPosition < this.replayPath.length) {
+        // we still have replay data for this execution, we apply it
+        if (this.replayPath[this.replayPathPosition]) items.push(c);
+        // checking for mismatches to stop the run in case the replay data is wrong
+        else if (c.value_.hasRan) throw new Error(`Mismatch between replayPath and real execution`);
+      } else {
+        // we do not any replay data, we check the real status
+        if (c.value_.hasRan) {
+          this.replayPath.push(true);
+          items.push(c);
+        } else this.replayPath.push(false);
+      }
+      ++this.replayPathPosition;
+    }
+    return items;
+  }
   private shrinkImpl(
     itemsRaw: Shrinkable<CommandWrapper<Model, Real, RunResult, CheckAsync>>[],
     shrunkOnce: boolean,
     alwaysKeepOneCommand: boolean
   ): Stream<Shrinkable<CommandWrapper<Model, Real, RunResult, CheckAsync>>[]> {
-    const items = itemsRaw.filter(c => c.value_.hasRan); // filter out commands that have not been executed
+    const items = this.filterNonExecuted(itemsRaw); // filter out commands that have not been executed
     if (items.length === 0) {
       return Stream.nil<Shrinkable<CommandWrapper<Model, Real, RunResult, CheckAsync>>[]>();
     }
@@ -95,11 +129,46 @@ function commands<Model extends object, Real>(
   commandArbs: Arbitrary<Command<Model, Real>>[],
   maxCommands?: number
 ): Arbitrary<Iterable<Command<Model, Real>>>;
+/**
+ * For arrays of {@link AsyncCommand} to be executed by {@link asyncModelRun}
+ *
+ * This implementation comes with a shrinker adapted for commands.
+ * It should shrink more efficiently than {@link array} for {@link AsyncCommand} arrays.
+ *
+ * @param commandArbs Arbitraries responsible to build commands
+ * @param maxCommands Maximal number of commands to build
+ */
+function commands<Model extends object, Real, CheckAsync extends boolean>(
+  commandArbs: Arbitrary<AsyncCommand<Model, Real, CheckAsync>>[],
+  settings?: CommandsSettings
+): Arbitrary<Iterable<AsyncCommand<Model, Real, CheckAsync>>>;
+/**
+ * For arrays of {@link Command} to be executed by {@link modelRun}
+ *
+ * This implementation comes with a shrinker adapted for commands.
+ * It should shrink more efficiently than {@link array} for {@link Command} arrays.
+ *
+ * @param commandArbs Arbitraries responsible to build commands
+ * @param maxCommands Maximal number of commands to build
+ */
+function commands<Model extends object, Real>(
+  commandArbs: Arbitrary<Command<Model, Real>>[],
+  settings?: CommandsSettings
+): Arbitrary<Iterable<Command<Model, Real>>>;
 function commands<Model extends object, Real, RunResult, CheckAsync extends boolean>(
   commandArbs: Arbitrary<ICommand<Model, Real, RunResult, CheckAsync>>[],
-  maxCommands?: number
+  settings?: number | CommandsSettings
 ): Arbitrary<Iterable<ICommand<Model, Real, RunResult, CheckAsync>>> {
-  return new CommandsArbitrary(commandArbs, maxCommands != null ? maxCommands : 10);
+  const maxCommands: number =
+    settings != null && typeof settings === 'number'
+      ? settings
+      : settings != null && settings.maxCommands != null
+      ? settings.maxCommands
+      : 10;
+  const replayPath: string | null =
+    settings != null && typeof settings !== 'number' ? settings.replayPath || null : null;
+  const disableReplayLog = settings != null && typeof settings !== 'number' && !!settings.disableReplayLog;
+  return new CommandsArbitrary(commandArbs, maxCommands != null ? maxCommands : 10, replayPath, disableReplayLog);
 }
 
 export { commands };

src/check/model/commands/CommandsIterable.ts

@@ -4,17 +4,22 @@ import { CommandWrapper } from './CommandWrapper';
 /** @hidden */
 export class CommandsIterable<Model extends object, Real, RunResult, CheckAsync extends boolean = false>
   implements Iterable<CommandWrapper<Model, Real, RunResult, CheckAsync>> {
-  constructor(readonly commands: CommandWrapper<Model, Real, RunResult, CheckAsync>[]) {}
+  constructor(
+    readonly commands: CommandWrapper<Model, Real, RunResult, CheckAsync>[],
+    readonly metadataForReplay: () => string
+  ) {}
   [Symbol.iterator](): Iterator<CommandWrapper<Model, Real, RunResult, CheckAsync>> {
     return this.commands[Symbol.iterator]();
   }
   [cloneMethod]() {
-    return new CommandsIterable(this.commands.map(c => c.clone()));
+    return new CommandsIterable(this.commands.map(c => c.clone()), this.metadataForReplay);
   }
   toString(): string {
-    return this.commands
+    const serializedCommands = this.commands
       .filter(c => c.hasRan)
       .map(c => c.toString())
       .join(',');
+    const metadata = this.metadataForReplay();
+    return metadata.length !== 0 ? `${serializedCommands} /*${metadata}*/` : serializedCommands;
   }
 }

src/check/model/commands/CommandsSettings.ts

@@ -0,0 +1,5 @@
+export interface CommandsSettings {
+  maxCommands?: number;
+  disableReplayLog?: boolean;
+  replayPath?: string;
+}

test/e2e/ReplayCommands.spec.ts

@@ -0,0 +1,50 @@
+import * as fc from '../../src/fast-check';
+
+// Fake commands
+type Model = { counter: number };
+type Real = {};
+class IncBy implements fc.Command<Model, Real> {
+  constructor(readonly v: number) {}
+  check = (m: Readonly<Model>) => true;
+  run = (m: Model, r: Real) => (m.counter += this.v);
+  toString = () => `IncBy(${this.v})`;
+}
+class DecPosBy implements fc.Command<Model, Real> {
+  constructor(readonly v: number) {}
+  check = (m: Readonly<Model>) => m.counter > 0;
+  run = (m: Model, r: Real) => (m.counter -= this.v);
+  toString = () => `DecPosBy(${this.v})`;
+}
+class AlwaysPos implements fc.Command<Model, Real> {
+  check = (m: Readonly<Model>) => true;
+  run = (m: Model, r: Real) => {
+    if (m.counter < 0) throw new Error('counter is supposed to be always greater or equal to zero');
+  };
+  toString = () => `AlwaysPos()`;
+}
+
+const seed = Date.now();
+describe(`ReplayCommands (seed: ${seed})`, () => {
+  it('Should be able to replay commands by specifying replayPath in fc.commands', () => {
+    const buildProp = (replayPath?: string) => {
+      return fc.property(
+        fc.commands(
+          [fc.nat().map(v => new IncBy(v)), fc.nat().map(v => new DecPosBy(v)), fc.constant(new AlwaysPos())],
+          { replayPath }
+        ),
+        cmds => fc.modelRun(() => ({ model: { counter: 0 }, real: {} }), cmds)
+      );
+    };
+
+    const out = fc.check(buildProp(), { seed: seed });
+    expect(out.failed).toBe(true);
+
+    const path = out.counterexamplePath!;
+    const replayPath = /\/\*replayPath=['"](.*)['"]\*\//.exec(out.counterexample![0].toString())![1];
+
+    const outReplayed = fc.check(buildProp(replayPath), { seed, path });
+    expect(outReplayed.counterexamplePath).toEqual(out.counterexamplePath);
+    expect(outReplayed.counterexample![0].toString()).toEqual(out.counterexample![0].toString());
+    expect(outReplayed.numRuns).toEqual(1);
+  });
+});

test/e2e/model/CommandsArbitrary.spec.ts

@@ -74,7 +74,7 @@ describe(`CommandsArbitrary (seed: ${seed})`, () => {
               fc.constant(new OddCommand()),
               fc.integer(1, 10).map(v => new CheckLessThanCommand(v))
             ],
-            1000
+            { disableReplayLog: true, maxCommands: 1000 }
           ),
           cmds => {
             const setup = () => ({
@@ -120,7 +120,9 @@ describe(`CommandsArbitrary (seed: ${seed})`, () => {
       const out = fc.check(
         fc.property(
           fc.array(fc.nat(9), 0, 3),
-          fc.commands([fc.constant(new FailureCommand()), fc.constant(new SuccessCommand())]),
+          fc.commands([fc.constant(new FailureCommand()), fc.constant(new SuccessCommand())], {
+            disableReplayLog: true
+          }),
           fc.array(fc.nat(9), 0, 3),
           (validSteps1, cmds, validSteps2) => {
             const setup = () => ({
@@ -143,7 +145,9 @@ describe(`CommandsArbitrary (seed: ${seed})`, () => {
       const out = fc.check(
         fc.property(
           fc.array(fc.nat(9), 0, 3),
-          fc.commands([fc.constant(new FailureCommand()), fc.constant(new SuccessCommand())]),
+          fc.commands([fc.constant(new FailureCommand()), fc.constant(new SuccessCommand())], {
+            disableReplayLog: true
+          }),
           fc.array(fc.nat(9), 0, 3),
           (validSteps1, cmds, validSteps2) => {
             if (String(cmds) !== '') {

test/unit/check/model/ReplayPath.spec.ts

@@ -0,0 +1,12 @@
+import * as fc from '../../../../lib/fast-check';
+
+import { ReplayPath } from '../../../../src/check/model/ReplayPath';
+
+describe('ReplayPath', () => {
+  it('Should be able to read back itself', () =>
+    fc.assert(
+      fc.property(fc.array(fc.boolean(), 0, 1000), (replayPath: boolean[]) => {
+        expect(ReplayPath.parse(ReplayPath.stringify(replayPath))).toEqual(replayPath);
+      })
+    ));
+});

test/unit/check/model/commands/CommandsArbitrary.spec.ts

@@ -149,7 +149,7 @@ describe('CommandWrapper', () => {
         })
       ));
     it('Should provide commands which have not run yet', () => {
-      const commandsArb = commands([constant(new SuccessCommand({ data: [] }))]);
+      const commandsArb = commands([constant(new SuccessCommand({ data: [] }))], { disableReplayLog: true });
       const arbs = genericTuple([nat(16), commandsArb, nat(16)] as Arbitrary<any>[]);
       const assertCommandsNotStarted = (shrinkable: Shrinkable<[number, Iterable<Cmd>, number]>) => {
         expect(String(shrinkable.value_[1])).toEqual('');
@@ -216,5 +216,56 @@ describe('CommandWrapper', () => {
         })
       );
     });
+    it.only('Should shrink the same way when based on replay data', () => {
+      fc.assert(
+        fc.property(fc.integer().noShrink(), fc.nat(100), (seed, numValues) => {
+          // create unused logOnCheck
+          const logOnCheck: { data: string[] } = { data: [] };
+
+          // generate scenario and simulate execution
+          const rng = prand.xorshift128plus(seed);
+          const refArbitrary = commands([
+            constant(new SuccessCommand(logOnCheck)),
+            constant(new SkippedCommand(logOnCheck)),
+            constant(new FailureCommand(logOnCheck)),
+            nat().map(v => new SuccessIdCommand(v))
+          ]);
+          const refShrinkable: Shrinkable<Iterable<Cmd>> = refArbitrary.generate(new Random(rng));
+          simulateCommands(refShrinkable.value_);
+
+          // trigger computation of replayPath
+          // and extract shrinks for ref
+          const refShrinks = [
+            ...refShrinkable
+              .shrink()
+              .take(numValues)
+              .map(s => [...s.value_].map(c => c.toString()))
+          ];
+
+          // extract replayPath
+          const replayPath = /\/\*replayPath=['"](.*)['"]\*\//.exec(refShrinkable.value_.toString())![1];
+
+          // generate scenario but do not simulate execution
+          const noExecShrinkable: Shrinkable<Iterable<Cmd>> = commands(
+            [
+              constant(new SuccessCommand(logOnCheck)),
+              constant(new SkippedCommand(logOnCheck)),
+              constant(new FailureCommand(logOnCheck)),
+              nat().map(v => new SuccessIdCommand(v))
+            ],
+            { replayPath }
+          ).generate(new Random(rng));
+
+          // check shrink values are identical
+          const noExecShrinks = [
+            ...noExecShrinkable
+              .shrink()
+              .take(numValues)
+              .map(s => [...s.value_].map(c => c.toString()))
+          ];
+          expect(noExecShrinks).toEqual(refShrinks);
+        })
+      );
+    });
   });
 });

test/unit/check/model/commands/CommandsIterable.spec.ts

@@ -27,7 +27,7 @@ describe('CommandsIterable', () => {
   it('Should not reset hasRun flag on iteration', () =>
     fc.assert(
       fc.property(fc.array(fc.boolean()), runFlags => {
-        const commands = [...new CommandsIterable(buildAlreadyRanCommands(runFlags))];
+        const commands = [...new CommandsIterable(buildAlreadyRanCommands(runFlags), () => '')];
         for (let idx = 0; idx !== runFlags.length; ++idx) {
           expect(commands[idx].hasRan).toEqual(runFlags[idx]);
         }
@@ -36,7 +36,7 @@ describe('CommandsIterable', () => {
   it('Should not reset hasRun flag on the original iterable on clone', () =>
     fc.assert(
       fc.property(fc.array(fc.boolean()), runFlags => {
-        const originalIterable = new CommandsIterable(buildAlreadyRanCommands(runFlags));
+        const originalIterable = new CommandsIterable(buildAlreadyRanCommands(runFlags), () => '');
         originalIterable[cloneMethod]();
         const commands = [...originalIterable];
         for (let idx = 0; idx !== runFlags.length; ++idx) {
@@ -47,20 +47,21 @@ describe('CommandsIterable', () => {
   it('Should reset hasRun flag for the clone on clone', () =>
     fc.assert(
       fc.property(fc.array(fc.boolean()), runFlags => {
-        const commands = [...new CommandsIterable(buildAlreadyRanCommands(runFlags))[cloneMethod]()];
+        const commands = [...new CommandsIterable(buildAlreadyRanCommands(runFlags), () => '')[cloneMethod]()];
         for (let idx = 0; idx !== runFlags.length; ++idx) {
           expect(commands[idx].hasRan).toBe(false);
         }
       })
     ));
-  it('Should only print ran commands', () =>
+  it('Should only print ran commands and metadata if any', () =>
     fc.assert(
-      fc.property(fc.array(fc.boolean()), runFlags => {
-        const commandsIterable = new CommandsIterable(buildAlreadyRanCommands(runFlags));
-        const expectedToString = runFlags
+      fc.property(fc.array(fc.boolean()), fc.fullUnicodeString(), (runFlags, metadata) => {
+        const commandsIterable = new CommandsIterable(buildAlreadyRanCommands(runFlags), () => metadata);
+        const expectedCommands = runFlags
           .map((hasRan, idx) => (hasRan ? String(idx) : ''))
           .filter(s => s !== '')
           .join(',');
+        const expectedToString = metadata.length !== 0 ? `${expectedCommands} /*${metadata}*/` : expectedCommands;
         expect(commandsIterable.toString()).toEqual(expectedToString);
       })
     ));