Start discovering all available tests lazily (#3120)

### Motivation

This is the first step in addressing several test explorer related issues that we currently have open.

This PR implements lazy test discovery up until the file level.

### Implementation

The VS Code API for the explorer supports lazily discovering tests by implementing the `resolveHandler`. I essentially only implement two resolutions:

1. If there's only one workspace folder, we eager discover all test files in the workspace
2. If there are many workspace folders, then the first level in the hierarchy are the workspaces themselves. If you open any of them, then it triggers resolution and we discover all available tests in that workspace

The tests are discovered using the VS Code `findFiles` API, which is very efficient and will avoid having us implement an indexing-like mechanism in the server. Since the test explorer is VS Code only anyway, this should be fine.

I also considered two levels of hierarchy for tests (in addition to the workspace when there are many). The firsts level is the `test/spec/features` directory. Since there may be many of them in an application split by components, we need to have that level of grouping.

The second level is the immediate subdirectory of the `test/spec/features` directory, which in a Rails app normally accounts for `models`, `controllers` and so on. This hierarchy is useful because it will allow users to take actions like running all tests in a workspace, test directory or all tests of a kind (e.g.: models).

### Notes

This PR doesn't yet implement the resolution of the test files themselves. The approach we will take for that is asking the server for which tests are available in a file, which gives us the opportunity to include add-on listeners in the discovery.

Since the implementation is still partial, I put a new under development feature flag.

### Automated Tests

Added tests.

### Manual tests

Enable this under development flag
```json
{
  "rubyLsp.featureFlags": {
    "newTestExperience": true
  }
}
```
Then start the extension on this branch and navigate tests

https://github.com/user-attachments/assets/1aaf145a-7b8f-48bf-b30b-61f0eef3e039

vscode/package.json

@@ -502,6 +502,10 @@
               "description": "Opt-in/out of the Tapioca add-on",
               "type": "boolean"
             },
+            "fullTestDiscovery": {
+              "description": "UNDER DEVEOPMENT. Opt-in/out of the full test discovery experience",
+              "type": "boolean"
+            },
             "launcher": {
               "description": "Opt-in/out of the new launcher mode",
               "type": "boolean"

vscode/src/common.ts

@@ -86,6 +86,7 @@ export const SUPPORTED_LANGUAGE_IDS = ["ruby", "erb"];
 export const FEATURE_FLAGS = {
   tapiocaAddon: 1.0,
   launcher: 0.1,
+  fullTestDiscovery: -1,
 };
 
 type FeatureFlagConfigurationKey = keyof typeof FEATURE_FLAGS | "all";

vscode/src/test/suite/testController.test.ts

@@ -1,14 +1,28 @@
 import * as assert from "assert";
+import path from "path";
+import fs from "fs";
+import os from "os";
 
 import * as vscode from "vscode";
 import { CodeLens } from "vscode-languageclient/node";
+import { afterEach } from "mocha";
+import sinon from "sinon";
 
 import { TestController } from "../../testController";
-import { Command } from "../../common";
+import * as common from "../../common";
 
 import { FAKE_TELEMETRY } from "./fakeTelemetry";
 
 suite("TestController", () => {
+  const workspacePath = path.dirname(
+    path.dirname(path.dirname(path.dirname(__dirname))),
+  );
+  const workspaceUri = vscode.Uri.file(workspacePath);
+  const workspaceFolder: vscode.WorkspaceFolder = {
+    uri: workspaceUri,
+    name: path.basename(workspaceUri.fsPath),
+    index: 0,
+  };
   const context = {
     extensionMode: vscode.ExtensionMode.Test,
     subscriptions: [],
@@ -18,6 +32,10 @@ suite("TestController", () => {
     },
   } as unknown as vscode.ExtensionContext;
 
+  afterEach(() => {
+    context.subscriptions.forEach((subscription) => subscription.dispose());
+  });
+
   test("createTestItems doesn't break when there's a missing group", () => {
     const controller = new TestController(
       context,
@@ -30,7 +48,7 @@ suite("TestController", () => {
         range: new vscode.Range(0, 0, 10, 10),
         command: {
           title: "Run",
-          command: Command.RunTest,
+          command: common.Command.RunTest,
           arguments: [
             "test/fake_test.rb",
             "test_do_something",
@@ -58,4 +76,133 @@ suite("TestController", () => {
       controller.createTestItems(codeLensItems);
     });
   });
+
+  test("populates test structure directly if there's only one workspace", async () => {
+    const stub = sinon.stub(common, "featureEnabled").returns(true);
+    const controller = new TestController(
+      context,
+      FAKE_TELEMETRY,
+      () => undefined,
+    );
+    stub.restore();
+
+    const workspacesStub = sinon
+      .stub(vscode.workspace, "workspaceFolders")
+      .get(() => [workspaceFolder]);
+
+    const relativePathStub = sinon
+      .stub(vscode.workspace, "asRelativePath")
+      .callsFake((uri) =>
+        path.relative(workspacePath, (uri as vscode.Uri).fsPath),
+      );
+
+    await controller.testController.resolveHandler!(undefined);
+    workspacesStub.restore();
+    relativePathStub.restore();
+
+    const collection = controller.testController.items;
+
+    const testDir = collection.get(
+      vscode.Uri.joinPath(workspaceUri, "test").toString(),
+    );
+    assert.ok(testDir);
+
+    const serverTest = testDir!.children.get(
+      vscode.Uri.joinPath(workspaceUri, "test", "server_test.rb").toString(),
+    );
+    assert.ok(serverTest);
+  });
+
+  test("makes the workspaces the top level when there's more than one", async () => {
+    const stub = sinon.stub(common, "featureEnabled").returns(true);
+    const controller = new TestController(
+      context,
+      FAKE_TELEMETRY,
+      () => undefined,
+    );
+    stub.restore();
+
+    const secondWorkspacePath = fs.mkdtempSync(
+      path.join(os.tmpdir(), "ruby-lsp-test-controller-"),
+    );
+    const secondWorkspaceUri = vscode.Uri.file(secondWorkspacePath);
+
+    fs.mkdirSync(path.join(secondWorkspaceUri.fsPath, "test"));
+    fs.writeFileSync(
+      path.join(secondWorkspaceUri.fsPath, "test", "other_test.rb"),
+      "require 'test_helper'\n\nclass OtherTest < Minitest::Test; end",
+    );
+
+    const secondWorkspaceFolder: vscode.WorkspaceFolder = {
+      uri: secondWorkspaceUri,
+      name: "second_workspace",
+      index: 1,
+    };
+    const workspacesStub = sinon
+      .stub(vscode.workspace, "workspaceFolders")
+      .get(() => [workspaceFolder, secondWorkspaceFolder]);
+
+    const relativePathStub = sinon
+      .stub(vscode.workspace, "asRelativePath")
+      .callsFake((uri) => {
+        const filePath = (uri as vscode.Uri).fsPath;
+
+        if (path.basename(filePath) === "other_test.rb") {
+          return path.relative(secondWorkspacePath, filePath);
+        } else {
+          return path.relative(workspacePath, filePath);
+        }
+      });
+
+    const getWorkspaceStub = sinon
+      .stub(vscode.workspace, "getWorkspaceFolder")
+      .callsFake((uri) => {
+        if (uri === secondWorkspaceUri) {
+          return secondWorkspaceFolder;
+        } else {
+          return workspaceFolder;
+        }
+      });
+
+    await controller.testController.resolveHandler!(undefined);
+
+    const collection = controller.testController.items;
+
+    // First workspace
+    const workspaceItem = collection.get(workspaceUri.toString());
+    assert.ok(workspaceItem);
+    await controller.testController.resolveHandler!(workspaceItem);
+
+    const testDir = workspaceItem!.children.get(
+      vscode.Uri.joinPath(workspaceUri, "test").toString(),
+    );
+    assert.ok(testDir);
+
+    const serverTest = testDir!.children.get(
+      vscode.Uri.joinPath(workspaceUri, "test", "server_test.rb").toString(),
+    );
+    assert.ok(serverTest);
+
+    // Second workspace
+    const secondWorkspaceItem = collection.get(secondWorkspaceUri.toString());
+    assert.ok(secondWorkspaceItem);
+    await controller.testController.resolveHandler!(secondWorkspaceItem);
+
+    const secondTestDir = secondWorkspaceItem!.children.get(
+      vscode.Uri.joinPath(secondWorkspaceUri, "test").toString(),
+    );
+    assert.ok(secondTestDir);
+
+    const otherTest = secondTestDir!.children.get(
+      vscode.Uri.joinPath(
+        secondWorkspaceUri,
+        "test",
+        "other_test.rb",
+      ).toString(),
+    );
+    assert.ok(otherTest);
+    workspacesStub.restore();
+    relativePathStub.restore();
+    getWorkspaceStub.restore();
+  });
 });