Implement test.todo (#2961)

* Implement `test.todo`

* remove skip condition

* Allow callbacks in .todo

* Add descriptive comment

* Log todos

* Include tests in title

* edit test.todo tests

---------

Co-authored-by: dave caruso <me@paperdave.net>

packages/bun-types/bun-test.d.ts

@@ -182,6 +182,23 @@ declare module "bun:test" {
         | ((done: (err?: unknown) => void) => void),
       timeoutMs?: number,
     ): void;
+    /**
+     * Indicate a test is yet to be written or implemented correctly.
+     *
+     * When a test function is passed, it will be marked as `todo` in the test results
+     * as long the test does not pass. When the test passes, the test will be marked as
+     * `fail` in the results; you will have to remove the `.todo` or check that your test
+     * is implemented correctly.
+     *
+     * @param label the label for the test
+     * @param fn the test function
+     */
+    todo(
+      label: string,
+      fn?:
+        | (() => void | Promise<unknown>)
+        | ((done: (err?: unknown) => void) => void),
+    ): void;
   };
   /**
    * Runs a test.

src/bun.js/test/jest.zig

@@ -469,6 +469,7 @@ pub const TestRunner = struct {
         onTestPass: OnTestUpdate,
         onTestFail: OnTestUpdate,
         onTestSkip: OnTestUpdate,
+        onTestTodo: OnTestUpdate,
     };
 
     pub fn reportPass(this: *TestRunner, test_id: Test.ID, file: string, label: string, expectations: u32, elapsed_ns: u64, parent: ?*DescribeScope) void {
@@ -485,6 +486,11 @@ pub const TestRunner = struct {
         this.callback.onTestSkip(this.callback, test_id, file, label, 0, 0, parent);
     }
 
+    pub fn reportTodo(this: *TestRunner, test_id: Test.ID, file: string, label: string, parent: ?*DescribeScope) void {
+        this.tests.items(.status)[test_id] = .todo;
+        this.callback.onTestTodo(this.callback, test_id, file, label, 0, 0, parent);
+    }
+
     pub fn addTestCount(this: *TestRunner, count: u32) u32 {
         this.tests.ensureUnusedCapacity(this.allocator, count) catch unreachable;
         const start = @truncate(Test.ID, this.tests.len);
@@ -532,6 +538,7 @@ pub const TestRunner = struct {
             pass,
             fail,
             skip,
+            todo,
         };
     };
 };
@@ -3159,6 +3166,7 @@ pub const TestScope = struct {
     ran: bool = false,
     task: ?*TestRunnerTask = null,
     skipped: bool = false,
+    is_todo: bool = false,
     snapshot_count: usize = 0,
     timeout_millis: u32 = default_timeout,
 
@@ -3169,6 +3177,7 @@ pub const TestScope = struct {
             .call = call,
             .only = only,
             .skip = skip,
+            .todo = todo,
         },
         .{},
     );
@@ -3214,6 +3223,17 @@ pub const TestScope = struct {
         return prepare(this, ctx, arguments, exception, .call);
     }
 
+    pub fn todo(
+        _: void,
+        ctx: js.JSContextRef,
+        this: js.JSObjectRef,
+        _: js.JSObjectRef,
+        arguments: []const js.JSValueRef,
+        exception: js.ExceptionRef,
+    ) js.JSObjectRef {
+        return prepare(this, ctx, arguments, exception, .todo);
+    }
+
     fn prepare(
         this: js.JSObjectRef,
         ctx: js.JSContextRef,
@@ -3240,16 +3260,36 @@ pub const TestScope = struct {
             label = (label_value.toSlice(ctx, allocator).cloneIfNeeded(allocator) catch unreachable).slice();
         }
 
+        if (tag == .todo and label_value == .zero) {
+            JSError(getAllocator(ctx), "test.todo() requires a description", .{}, ctx, exception);
+            return this;
+        }
+
         const function = function_value;
         if (function.isEmptyOrUndefinedOrNull() or !function.isCell() or !function.isCallable(ctx.vm())) {
-            JSError(getAllocator(ctx), "test() expects a function", .{}, ctx, exception);
-            return this;
+            // a callback is not required for .todo
+            if (tag != .todo) {
+                JSError(getAllocator(ctx), "test() expects a function", .{}, ctx, exception);
+                return this;
+            }
         }
 
         if (tag == .only) {
             Jest.runner.?.setOnly();
         }
 
+        if (tag == .todo) {
+            DescribeScope.active.todo_counter += 1;
+            DescribeScope.active.tests.append(getAllocator(ctx), TestScope{
+                .label = label,
+                .parent = DescribeScope.active,
+                .is_todo = true,
+                .callback = if (function == .zero) null else function.asObjectRef(),
+            }) catch unreachable;
+
+            return this;
+        }
+
         if (tag == .skip or (tag != .only and Jest.runner.?.only)) {
             DescribeScope.active.skipped_counter += 1;
             DescribeScope.active.tests.append(getAllocator(ctx), TestScope{
@@ -3369,10 +3409,15 @@ pub const TestScope = struct {
 
         if (initial_value.isAnyError()) {
             if (!Jest.runner.?.did_pending_test_fail) {
-                Jest.runner.?.did_pending_test_fail = true;
+                // test failed unless it's a todo
+                Jest.runner.?.did_pending_test_fail = !this.is_todo;
                 vm.runErrorHandler(initial_value, null);
             }
 
+            if (this.is_todo) {
+                return .{ .todo = {} };
+            }
+
             return .{ .fail = active_test_expectation_counter.actual };
         }
 
@@ -3391,10 +3436,15 @@ pub const TestScope = struct {
             switch (promise.status(vm.global.vm())) {
                 .Rejected => {
                     if (!Jest.runner.?.did_pending_test_fail) {
-                        Jest.runner.?.did_pending_test_fail = true;
+                        // test failed unless it's a todo
+                        Jest.runner.?.did_pending_test_fail = !this.is_todo;
                         vm.runErrorHandler(promise.result(vm.global.vm()), null);
                     }
 
+                    if (this.is_todo) {
+                        return .{ .todo = {} };
+                    }
+
                     return .{ .fail = active_test_expectation_counter.actual };
                 },
                 .Pending => {
@@ -3427,6 +3477,11 @@ pub const TestScope = struct {
             return .{ .fail = active_test_expectation_counter.actual };
         }
 
+        if (this.is_todo) {
+            Output.prettyErrorln("  <d>^<r> <red>this test is marked as todo but passes.<r> <d>Remove `.todo` or check that test is correct.<r>", .{});
+            return .{ .fail = active_test_expectation_counter.actual };
+        }
+
         return .{ .pass = active_test_expectation_counter.actual };
     }
 
@@ -3465,6 +3520,7 @@ pub const DescribeScope = struct {
     done: bool = false,
     skipped: bool = false,
     skipped_counter: u32 = 0,
+    todo_counter: u32 = 0,
 
     pub fn isAllSkipped(this: *const DescribeScope) bool {
         return this.skipped or @as(usize, this.skipped_counter) >= this.tests.items.len;
@@ -3940,6 +3996,13 @@ pub const TestRunnerTask = struct {
         var test_: TestScope = this.describe.tests.items[test_id];
         describe.current_test_id = test_id;
         var globalThis = this.globalThis;
+
+        if (!describe.skipped and test_.is_todo and test_.callback == null) {
+            this.processTestResult(globalThis, .{ .todo = {} }, test_, test_id, describe);
+            this.deinit();
+            return false;
+        }
+
         if (test_.skipped or describe.skipped) {
             this.processTestResult(globalThis, .{ .skip = {} }, test_, test_id, describe);
             this.deinit();
@@ -4067,6 +4130,7 @@ pub const TestRunnerTask = struct {
                 describe,
             ),
             .skip => Jest.runner.?.reportSkip(test_id, this.source_file_path, test_.label, describe),
+            .todo => Jest.runner.?.reportTodo(test_id, this.source_file_path, test_.label, describe),
             .pending => @panic("Unexpected pending test"),
         }
         describe.onTestComplete(globalThis, test_id, result == .skip);
@@ -4101,4 +4165,5 @@ pub const Result = union(TestRunner.Test.Status) {
     pass: u32, // assertion count
     pending: void,
     skip: void,
+    todo: void,
 };

src/cli/test_command.zig

@@ -52,6 +52,7 @@ fn fmtStatusTextLine(comptime status: @Type(.EnumLiteral), comptime emoji: bool)
             .pass => Output.prettyFmt("<r><green>✓<r>", emoji),
             .fail => Output.prettyFmt("<r><red>✗<r>", emoji),
             .skip => Output.prettyFmt("<r><yellow>-<d>", emoji),
+            .todo => Output.prettyFmt("<r><magenta>✎<r>", emoji),
             else => @compileError("Invalid status " ++ @tagName(status)),
         };
     }
@@ -74,11 +75,13 @@ pub const CommandLineReporter = struct {
 
     failures_to_repeat_buf: std.ArrayListUnmanaged(u8) = .{},
     skips_to_repeat_buf: std.ArrayListUnmanaged(u8) = .{},
+    todos_to_repeat_buf: std.ArrayListUnmanaged(u8) = .{},
 
     pub const Summary = struct {
         pass: u32 = 0,
         expectations: u32 = 0,
         skip: u32 = 0,
+        todo: u32 = 0,
         fail: u32 = 0,
     };
 
@@ -217,6 +220,27 @@ pub const CommandLineReporter = struct {
         this.summary.expectations += expectations;
         this.jest.tests.items(.status)[id] = TestRunner.Test.Status.skip;
     }
+
+    pub fn handleTestTodo(cb: *TestRunner.Callback, id: Test.ID, _: string, label: string, expectations: u32, elapsed_ns: u64, parent: ?*jest.DescribeScope) void {
+        var writer_: std.fs.File.Writer = Output.errorWriter();
+        var this: *CommandLineReporter = @fieldParentPtr(CommandLineReporter, "callback", cb);
+
+        // when the tests skip, we want to repeat the failures at the end
+        // so that you can see them better when there are lots of tests that ran
+        const initial_length = this.todos_to_repeat_buf.items.len;
+        var writer = this.todos_to_repeat_buf.writer(bun.default_allocator);
+
+        writeTestStatusLine(.todo, &writer);
+        printTestLine(label, elapsed_ns, parent, true, writer);
+
+        writer_.writeAll(this.todos_to_repeat_buf.items[initial_length..]) catch unreachable;
+        Output.flush();
+
+        // this.updateDots();
+        this.summary.todo += 1;
+        this.summary.expectations += expectations;
+        this.jest.tests.items(.status)[id] = TestRunner.Test.Status.todo;
+    }
 };
 
 const Scanner = struct {
@@ -405,6 +429,7 @@ pub const TestCommand = struct {
             .onTestPass = CommandLineReporter.handleTestPass,
             .onTestFail = CommandLineReporter.handleTestFail,
             .onTestSkip = CommandLineReporter.handleTestSkip,
+            .onTestTodo = CommandLineReporter.handleTestTodo,
         };
         reporter.repeat_count = @max(ctx.test_options.repeat_count, 1);
         reporter.jest.callback = &reporter.callback;
@@ -473,11 +498,23 @@ pub const TestCommand = struct {
                 error_writer.writeAll(reporter.skips_to_repeat_buf.items) catch unreachable;
             }
 
-            if (reporter.summary.fail > 0) {
+            if (reporter.summary.todo > 0) {
                 if (reporter.summary.skip > 0) {
                     Output.prettyError("\n", .{});
                 }
 
+                Output.prettyError("\n<r><d>{d} tests todo:<r>\n", .{reporter.summary.todo});
+                Output.flush();
+
+                var error_writer = Output.errorWriter();
+                error_writer.writeAll(reporter.todos_to_repeat_buf.items) catch unreachable;
+            }
+
+            if (reporter.summary.fail > 0) {
+                if (reporter.summary.skip > 0 or reporter.summary.todo > 0) {
+                    Output.prettyError("\n", .{});
+                }
+
                 Output.prettyError("\n<r><d>{d} tests failed:<r>\n", .{reporter.summary.fail});
                 Output.flush();
 
@@ -516,6 +553,10 @@ pub const TestCommand = struct {
                 Output.prettyError(" <r><yellow>{d:5>} skip<r>\n", .{reporter.summary.skip});
             }
 
+            if (reporter.summary.todo > 0) {
+                Output.prettyError(" <r><magenta>{d:5>} todo<r>\n", .{reporter.summary.todo});
+            }
+
             if (reporter.summary.fail > 0) {
                 Output.prettyError("<r><red>", .{});
             } else {
@@ -567,10 +608,8 @@ pub const TestCommand = struct {
                 Output.prettyError(" {d:5>} expect() calls\n", .{reporter.summary.expectations});
             }
 
-            Output.prettyError("Ran {d} tests across {d} files ", .{
-                reporter.summary.fail + reporter.summary.pass,
-                test_files.len,
-            });
+            const total_tests = reporter.summary.fail + reporter.summary.pass + reporter.summary.skip + reporter.summary.todo;
+            Output.prettyError("Ran {d} tests across {d} files. <d>{d} total<r> ", .{ reporter.summary.fail + reporter.summary.pass, test_files.len, total_tests });
             Output.printStartEnd(ctx.start_time, std.time.nanoTimestamp());
         }
 

test/js/bun/test/test-test.test.ts

@@ -2760,6 +2760,38 @@ describe(() => {
   });
 });
 
+it("test.todo", () => {
+  const path = join(realpathSync(tmpdir()), "todo-test.test.js");
+  copyFileSync(join(import.meta.dir, "todo-test-fixture.js"), path);
+  const { stdout, stderr, exitCode } = spawnSync({
+    cmd: [bunExe(), "test", path],
+    stdout: "pipe",
+    stderr: "pipe",
+    env: bunEnv,
+    cwd: realpathSync(dirname(path)),
+  });
+
+  const err = stderr!.toString();
+  expect(err).toContain("this test is marked as todo but passes");
+  expect(err).toContain("2 todo");
+  expect(err).toContain("1 fail");
+});
+
+it("test.todo doesnt cause exit code 1", () => {
+  const path = join(realpathSync(tmpdir()), "todo-test.test.js");
+  copyFileSync(join(import.meta.dir, "todo-test-fixture-2.js"), path);
+  const { stdout, stderr, exitCode } = spawnSync({
+    cmd: [bunExe(), "test", path],
+    stdout: "pipe",
+    stderr: "pipe",
+    env: bunEnv,
+    cwd: realpathSync(dirname(path)),
+  });
+
+  const err = stderr!.toString();
+  expect(exitCode).toBe(0);
+});
+
 it("test timeouts when expected", () => {
   const path = join(realpathSync(tmpdir()), "test-timeout.test.js");
   copyFileSync(join(import.meta.dir, "timeout-test-fixture.js"), path);

test/js/bun/test/todo-test-fixture-2.js

@@ -0,0 +1,6 @@
+import { test } from "bun:test";
+
+test.todo("todo 1")
+test.todo("todo 2", () => {
+  throw new Error("this error is shown");
+})

test/js/bun/test/todo-test-fixture.js

@@ -0,0 +1,9 @@
+import { test } from "bun:test";
+
+test.todo("todo 1")
+test.todo("todo 2", () => {
+  throw new Error("this error is shown");
+})
+test.todo("todo 3", () => {
+  // passes
+});