From 3ba58f657238a9163bce5016b5d7a136b592eb8f Mon Sep 17 00:00:00 2001
From: Kartik Raj <karraj@microsoft.com>
Date: Wed, 15 Nov 2023 16:54:15 -0800
Subject: [PATCH 1/5] Add utilities to exec and spawn inside worker threads

---
 src/client/common/process/worker/main.ts      |  27 +++
 .../common/process/worker/plainExecWorker.ts  |  16 ++
 .../process/worker/rawProcessApiWrapper.ts    |  43 ++++
 .../common/process/worker/shellExecWorker.ts  |  16 ++
 src/client/common/process/worker/types.ts     |  38 ++++
 .../process/worker/workerRawProcessApis.ts    | 213 ++++++++++++++++++
 6 files changed, 353 insertions(+)
 create mode 100644 src/client/common/process/worker/main.ts
 create mode 100644 src/client/common/process/worker/plainExecWorker.ts
 create mode 100644 src/client/common/process/worker/rawProcessApiWrapper.ts
 create mode 100644 src/client/common/process/worker/shellExecWorker.ts
 create mode 100644 src/client/common/process/worker/types.ts
 create mode 100644 src/client/common/process/worker/workerRawProcessApis.ts

diff --git a/src/client/common/process/worker/main.ts b/src/client/common/process/worker/main.ts
new file mode 100644
index 000000000000..d8f158a73c70
--- /dev/null
+++ b/src/client/common/process/worker/main.ts
@@ -0,0 +1,27 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+import { Worker } from 'worker_threads';
+import { traceError } from '../../../logging';
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/explicit-module-boundary-types
+export async function executeWorkerFile(workerFileName: string, workerData: any): Promise<any> {
+    return new Promise((resolve, reject) => {
+        const worker = new Worker(workerFileName, { workerData });
+        worker.on('message', (res: { err: Error; res: unknown }) => {
+            if (res.err) {
+                reject(res.err);
+            }
+            resolve(res.res);
+        });
+        worker.on('error', (ex: Error) => {
+            traceError(`Error in worker ${workerFileName}`, ex);
+            reject(ex);
+        });
+        worker.on('exit', (code) => {
+            if (code !== 0) {
+                reject(new Error(`Worker ${workerFileName} stopped with exit code ${code}`));
+            }
+        });
+    });
+}
diff --git a/src/client/common/process/worker/plainExecWorker.ts b/src/client/common/process/worker/plainExecWorker.ts
new file mode 100644
index 000000000000..69374eba53d7
--- /dev/null
+++ b/src/client/common/process/worker/plainExecWorker.ts
@@ -0,0 +1,16 @@
+import { parentPort, workerData } from 'worker_threads';
+import { workerPlainExec } from './workerRawProcessApis';
+
+workerPlainExec(workerData.file, workerData.args, workerData.options, workerData.defaultEnv, workerData.disposables)
+    .then((res) => {
+        if (!parentPort) {
+            throw new Error('Not in a worker thread');
+        }
+        parentPort.postMessage({ res });
+    })
+    .catch((err) => {
+        if (!parentPort) {
+            throw new Error('Not in a worker thread');
+        }
+        parentPort.postMessage({ err });
+    });
diff --git a/src/client/common/process/worker/rawProcessApiWrapper.ts b/src/client/common/process/worker/rawProcessApiWrapper.ts
new file mode 100644
index 000000000000..dc984a1439d8
--- /dev/null
+++ b/src/client/common/process/worker/rawProcessApiWrapper.ts
@@ -0,0 +1,43 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+import { SpawnOptions } from 'child_process';
+import * as path from 'path';
+import { WorkspaceService } from '../../application/workspace';
+import { ProcessLogger } from '../logger';
+import { executeWorkerFile } from './main';
+import { EnvironmentVariables, ExecutionResult, IDisposable, ShellOptions } from './types';
+
+export function shellExec(
+    command: string,
+    options: ShellOptions,
+    defaultEnv?: EnvironmentVariables,
+    disposables?: Set<IDisposable>,
+): Promise<ExecutionResult<string>> {
+    const processLogger = new ProcessLogger(new WorkspaceService());
+    processLogger.logProcess(command, undefined, options);
+    return executeWorkerFile(path.join(__dirname, 'shellExecWorker.js'), {
+        command,
+        options,
+        defaultEnv,
+        disposables,
+    });
+}
+
+export function plainExec(
+    file: string,
+    args: string[],
+    options: SpawnOptions & { doNotLog?: boolean } = {},
+    defaultEnv?: EnvironmentVariables,
+    disposables?: Set<IDisposable>,
+): Promise<ExecutionResult<string>> {
+    const processLogger = new ProcessLogger(new WorkspaceService());
+    processLogger.logProcess(file, args, options);
+    return executeWorkerFile(path.join(__dirname, 'plainExecWorker.js'), {
+        file,
+        args,
+        options,
+        defaultEnv,
+        disposables,
+    });
+}
diff --git a/src/client/common/process/worker/shellExecWorker.ts b/src/client/common/process/worker/shellExecWorker.ts
new file mode 100644
index 000000000000..739d0bc1b6d5
--- /dev/null
+++ b/src/client/common/process/worker/shellExecWorker.ts
@@ -0,0 +1,16 @@
+import { parentPort, workerData } from 'worker_threads';
+import { workerShellExec } from './workerRawProcessApis';
+
+workerShellExec(workerData.command, workerData.options, workerData.defaultEnv, workerData.disposables)
+    .then((res) => {
+        if (!parentPort) {
+            throw new Error('Not in a worker thread');
+        }
+        parentPort.postMessage(res);
+    })
+    .catch((ex) => {
+        if (!parentPort) {
+            throw new Error('Not in a worker thread');
+        }
+        parentPort.postMessage(ex);
+    });
diff --git a/src/client/common/process/worker/types.ts b/src/client/common/process/worker/types.ts
new file mode 100644
index 000000000000..5c58aec10214
--- /dev/null
+++ b/src/client/common/process/worker/types.ts
@@ -0,0 +1,38 @@
+/* eslint-disable @typescript-eslint/no-empty-function */
+/* eslint-disable @typescript-eslint/explicit-module-boundary-types */
+import { ExecOptions, SpawnOptions as ChildProcessSpawnOptions } from 'child_process';
+
+export function noop() {}
+export interface IDisposable {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    dispose(): void | undefined | Promise<void>;
+}
+export type EnvironmentVariables = Record<string, string | undefined>;
+export class StdErrError extends Error {
+    // eslint-disable-next-line @typescript-eslint/no-useless-constructor
+    constructor(message: string) {
+        super(message);
+    }
+}
+
+export type SpawnOptions = ChildProcessSpawnOptions & {
+    encoding?: string;
+    // /**
+    //  * Can't use `CancellationToken` here as it comes from vscode which is not available in worker threads.
+    //  */
+    // token?: CancellationToken;
+    mergeStdOutErr?: boolean;
+    throwOnStdErr?: boolean;
+    extraVariables?: NodeJS.ProcessEnv;
+    // /**
+    //  * Can't use `OutputChannel` here as it comes from vscode which is not available in worker threads.
+    //  */
+    // outputChannel?: OutputChannel;
+    stdinStr?: string;
+};
+export type ShellOptions = ExecOptions & { throwOnStdErr?: boolean };
+
+export type ExecutionResult<T extends string | Buffer> = {
+    stdout: T;
+    stderr?: T;
+};
diff --git a/src/client/common/process/worker/workerRawProcessApis.ts b/src/client/common/process/worker/workerRawProcessApis.ts
new file mode 100644
index 000000000000..1c1c6ca5b46b
--- /dev/null
+++ b/src/client/common/process/worker/workerRawProcessApis.ts
@@ -0,0 +1,213 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+// !!!! IMPORTANT: DO NOT IMPORT FROM VSCODE MODULE AS IT IS NOT AVAILABLE INSIDE WORKER THREADS !!!!
+
+import { exec, execSync, spawn } from 'child_process';
+import { Readable } from 'stream';
+import { createDeferred } from '../../utils/async';
+import { DEFAULT_ENCODING } from '../constants';
+import { decodeBuffer } from '../decoder';
+import {
+    ShellOptions,
+    SpawnOptions,
+    EnvironmentVariables,
+    IDisposable,
+    noop,
+    StdErrError,
+    ExecutionResult,
+} from './types';
+
+const PS_ERROR_SCREEN_BOGUS = /your [0-9]+x[0-9]+ screen size is bogus\. expect trouble/;
+
+function getDefaultOptions<T extends ShellOptions | SpawnOptions>(options: T, defaultEnv?: EnvironmentVariables): T {
+    const defaultOptions = { ...options };
+    const execOptions = defaultOptions as SpawnOptions;
+    if (execOptions) {
+        execOptions.encoding =
+            typeof execOptions.encoding === 'string' && execOptions.encoding.length > 0
+                ? execOptions.encoding
+                : DEFAULT_ENCODING;
+        const { encoding } = execOptions;
+        delete execOptions.encoding;
+        execOptions.encoding = encoding;
+    }
+    if (!defaultOptions.env || Object.keys(defaultOptions.env).length === 0) {
+        const env = defaultEnv || process.env;
+        defaultOptions.env = { ...env };
+    } else {
+        defaultOptions.env = { ...defaultOptions.env };
+    }
+
+    if (execOptions && execOptions.extraVariables) {
+        defaultOptions.env = { ...defaultOptions.env, ...execOptions.extraVariables };
+    }
+
+    // Always ensure we have unbuffered output.
+    defaultOptions.env.PYTHONUNBUFFERED = '1';
+    if (!defaultOptions.env.PYTHONIOENCODING) {
+        defaultOptions.env.PYTHONIOENCODING = 'utf-8';
+    }
+
+    return defaultOptions;
+}
+
+export function workerShellExec(
+    command: string,
+    options: ShellOptions,
+    defaultEnv?: EnvironmentVariables,
+    disposables?: Set<IDisposable>,
+): Promise<ExecutionResult<string>> {
+    const shellOptions = getDefaultOptions(options, defaultEnv);
+    return new Promise((resolve, reject) => {
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        const callback = (e: any, stdout: any, stderr: any) => {
+            if (e && e !== null) {
+                reject(e);
+            } else if (shellOptions.throwOnStdErr && stderr && stderr.length) {
+                reject(new Error(stderr));
+            } else {
+                stdout = filterOutputUsingCondaRunMarkers(stdout);
+                // Make sure stderr is undefined if we actually had none. This is checked
+                // elsewhere because that's how exec behaves.
+                resolve({ stderr: stderr && stderr.length > 0 ? stderr : undefined, stdout });
+            }
+        };
+        let procExited = false;
+        const proc = exec(command, shellOptions, callback); // NOSONAR
+        proc.once('close', () => {
+            procExited = true;
+        });
+        proc.once('exit', () => {
+            procExited = true;
+        });
+        proc.once('error', () => {
+            procExited = true;
+        });
+        const disposable: IDisposable = {
+            dispose: () => {
+                // If process has not exited nor killed, force kill it.
+                if (!procExited && !proc.killed) {
+                    if (proc.pid) {
+                        killPid(proc.pid);
+                    } else {
+                        proc.kill();
+                    }
+                }
+            },
+        };
+        if (disposables) {
+            disposables.add(disposable);
+        }
+    });
+}
+
+export function workerPlainExec(
+    file: string,
+    args: string[],
+    options: SpawnOptions & { doNotLog?: boolean } = {},
+    defaultEnv?: EnvironmentVariables,
+    disposables?: Set<IDisposable>,
+): Promise<ExecutionResult<string>> {
+    const spawnOptions = getDefaultOptions(options, defaultEnv);
+    const encoding = spawnOptions.encoding ? spawnOptions.encoding : 'utf8';
+    const proc = spawn(file, args, spawnOptions);
+    // Listen to these errors (unhandled errors in streams tears down the process).
+    // Errors will be bubbled up to the `error` event in `proc`, hence no need to log.
+    proc.stdout?.on('error', noop);
+    proc.stderr?.on('error', noop);
+    const deferred = createDeferred<ExecutionResult<string>>();
+    const disposable: IDisposable = {
+        dispose: () => {
+            // If process has not exited nor killed, force kill it.
+            if (!proc.killed && !deferred.completed) {
+                if (proc.pid) {
+                    killPid(proc.pid);
+                } else {
+                    proc.kill();
+                }
+            }
+        },
+    };
+    disposables?.add(disposable);
+    const internalDisposables: IDisposable[] = [];
+
+    // eslint-disable-next-line @typescript-eslint/ban-types
+    const on = (ee: Readable | null, name: string, fn: Function) => {
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        ee?.on(name, fn as any);
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        internalDisposables.push({ dispose: () => ee?.removeListener(name, fn as any) as any });
+    };
+
+    // Tokens not supported yet as they come from vscode module which is not available.
+    // if (options.token) {
+    //     internalDisposables.push(options.token.onCancellationRequested(disposable.dispose));
+    // }
+
+    const stdoutBuffers: Buffer[] = [];
+    on(proc.stdout, 'data', (data: Buffer) => {
+        stdoutBuffers.push(data);
+    });
+    const stderrBuffers: Buffer[] = [];
+    on(proc.stderr, 'data', (data: Buffer) => {
+        if (options.mergeStdOutErr) {
+            stdoutBuffers.push(data);
+            stderrBuffers.push(data);
+        } else {
+            stderrBuffers.push(data);
+        }
+    });
+
+    proc.once('close', () => {
+        if (deferred.completed) {
+            return;
+        }
+        const stderr: string | undefined =
+            stderrBuffers.length === 0 ? undefined : decodeBuffer(stderrBuffers, encoding);
+        if (
+            stderr &&
+            stderr.length > 0 &&
+            options.throwOnStdErr &&
+            // ignore this specific error silently; see this issue for context: https://github.com/microsoft/vscode/issues/75932
+            !(PS_ERROR_SCREEN_BOGUS.test(stderr) && stderr.replace(PS_ERROR_SCREEN_BOGUS, '').trim().length === 0)
+        ) {
+            deferred.reject(new StdErrError(stderr));
+        } else {
+            let stdout = decodeBuffer(stdoutBuffers, encoding);
+            stdout = filterOutputUsingCondaRunMarkers(stdout);
+            deferred.resolve({ stdout, stderr });
+        }
+        internalDisposables.forEach((d) => d.dispose());
+        disposable.dispose();
+    });
+    proc.once('error', (ex) => {
+        deferred.reject(ex);
+        internalDisposables.forEach((d) => d.dispose());
+        disposable.dispose();
+    });
+
+    return deferred.promise;
+}
+
+function filterOutputUsingCondaRunMarkers(stdout: string) {
+    // These markers are added if conda run is used or `interpreterInfo.py` is
+    // run, see `get_output_via_markers.py`.
+    const regex = />>>PYTHON-EXEC-OUTPUT([\s\S]*)<<<PYTHON-EXEC-OUTPUT/;
+    const match = stdout.match(regex);
+    const filteredOut = match !== null && match.length >= 2 ? match[1].trim() : undefined;
+    return filteredOut !== undefined ? filteredOut : stdout;
+}
+
+function killPid(pid: number): void {
+    try {
+        if (process.platform === 'win32') {
+            // Windows doesn't support SIGTERM, so execute taskkill to kill the process
+            execSync(`taskkill /pid ${pid} /T /F`); // NOSONAR
+        } else {
+            process.kill(pid);
+        }
+    } catch {
+        console.warn('Unable to kill process with pid', pid);
+    }
+}

From 384e56af37269d5f1446d18293138c368af38103 Mon Sep 17 00:00:00 2001
From: Kartik Raj <karraj@microsoft.com>
Date: Wed, 15 Nov 2023 17:31:54 -0800
Subject: [PATCH 2/5] Shell exec and exec using worker threads in discovery

---
 .../common/externalDependencies.ts            | 46 ++++++++-----------
 .../common/windowsRegistry.ts                 |  2 +-
 2 files changed, 20 insertions(+), 28 deletions(-)

diff --git a/src/client/pythonEnvironments/common/externalDependencies.ts b/src/client/pythonEnvironments/common/externalDependencies.ts
index 357967236c9e..70ca21d4a8a5 100644
--- a/src/client/pythonEnvironments/common/externalDependencies.ts
+++ b/src/client/pythonEnvironments/common/externalDependencies.ts
@@ -3,7 +3,6 @@
 
 import * as fsapi from 'fs-extra';
 import * as path from 'path';
-import { Worker } from 'worker_threads';
 import * as vscode from 'vscode';
 import { IWorkspaceService } from '../../common/application/types';
 import { ExecutionResult, IProcessServiceFactory, ShellOptions, SpawnOptions } from '../../common/process/types';
@@ -12,6 +11,9 @@ import { chain, iterable } from '../../common/utils/async';
 import { getOSType, OSType } from '../../common/utils/platform';
 import { IServiceContainer } from '../../ioc/types';
 import { traceError, traceVerbose } from '../../logging';
+import { DiscoveryUsingWorkers } from '../../common/experiments/groups';
+import { plainExec, shellExec } from '../../common/process/worker/rawProcessApiWrapper';
+import { IEnvironmentVariablesProvider } from '../../common/variables/types';
 
 let internalServiceContainer: IServiceContainer;
 export function initializeExternalDependencies(serviceContainer: IServiceContainer): void {
@@ -21,13 +23,25 @@ export function initializeExternalDependencies(serviceContainer: IServiceContain
 // processes
 
 export async function shellExecute(command: string, options: ShellOptions = {}): Promise<ExecutionResult<string>> {
-    const service = await internalServiceContainer.get<IProcessServiceFactory>(IProcessServiceFactory).create();
-    return service.shellExec(command, options);
+    if (inExperiment(DiscoveryUsingWorkers.experiment)) {
+        const service = await internalServiceContainer.get<IProcessServiceFactory>(IProcessServiceFactory).create();
+        return service.shellExec(command, options);
+    }
+    const envVarsService = internalServiceContainer.get<IEnvironmentVariablesProvider>(IEnvironmentVariablesProvider);
+    const envs = await envVarsService.getEnvironmentVariables();
+    options.env = { ...options.env, ...envs };
+    return shellExec(command, options);
 }
 
 export async function exec(file: string, args: string[], options: SpawnOptions = {}): Promise<ExecutionResult<string>> {
-    const service = await internalServiceContainer.get<IProcessServiceFactory>(IProcessServiceFactory).create();
-    return service.exec(file, args, options);
+    if (inExperiment(DiscoveryUsingWorkers.experiment)) {
+        const service = await internalServiceContainer.get<IProcessServiceFactory>(IProcessServiceFactory).create();
+        return service.exec(file, args, options);
+    }
+    const envVarsService = internalServiceContainer.get<IEnvironmentVariablesProvider>(IEnvironmentVariablesProvider);
+    const envs = await envVarsService.getEnvironmentVariables();
+    options.env = { ...options.env, ...envs };
+    return plainExec(file, args, options);
 }
 
 export function inExperiment(experimentName: string): boolean {
@@ -201,25 +215,3 @@ export function onDidChangePythonSetting(name: string, callback: () => void, roo
         }
     });
 }
-
-// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/explicit-module-boundary-types
-export async function executeWorkerFile(workerFileName: string, workerData: any): Promise<any> {
-    return new Promise((resolve, reject) => {
-        const worker = new Worker(workerFileName, { workerData });
-        worker.on('message', (res: { err: Error; res: unknown }) => {
-            if (res.err) {
-                reject(res.err);
-            }
-            resolve(res.res);
-        });
-        worker.on('error', (ex: Error) => {
-            traceError(`Error in worker ${workerFileName}`, ex);
-            reject(ex);
-        });
-        worker.on('exit', (code) => {
-            if (code !== 0) {
-                reject(new Error(`Worker ${workerFileName} stopped with exit code ${code}`));
-            }
-        });
-    });
-}
diff --git a/src/client/pythonEnvironments/common/windowsRegistry.ts b/src/client/pythonEnvironments/common/windowsRegistry.ts
index efac5bb3209f..ac58963a6c14 100644
--- a/src/client/pythonEnvironments/common/windowsRegistry.ts
+++ b/src/client/pythonEnvironments/common/windowsRegistry.ts
@@ -5,7 +5,7 @@
 import { HKCU, HKLM, Options, REG_SZ, Registry, RegistryItem } from 'winreg';
 import * as path from 'path';
 import { createDeferred } from '../../common/utils/async';
-import { executeWorkerFile } from './externalDependencies';
+import { executeWorkerFile } from '../../common/process/worker/main';
 
 export { HKCU, HKLM, REG_SZ, Options };
 

From 487afe8ef12a848f33e2338d9f3043c7421da3fa Mon Sep 17 00:00:00 2001
From: Kartik Raj <karraj@microsoft.com>
Date: Wed, 15 Nov 2023 17:51:20 -0800
Subject: [PATCH 3/5] Use worker threads for fetching conda environments

---
 src/client/common/process/worker/plainExecWorker.ts    | 10 ++++++++--
 .../common/process/worker/rawProcessApiWrapper.ts      | 10 +++-------
 src/client/common/process/worker/shellExecWorker.ts    |  8 ++++----
 .../common/process/worker/workerRawProcessApis.ts      |  4 ++--
 .../pythonEnvironments/common/externalDependencies.ts  | 10 +++++-----
 5 files changed, 22 insertions(+), 20 deletions(-)

diff --git a/src/client/common/process/worker/plainExecWorker.ts b/src/client/common/process/worker/plainExecWorker.ts
index 69374eba53d7..9893142ed51c 100644
--- a/src/client/common/process/worker/plainExecWorker.ts
+++ b/src/client/common/process/worker/plainExecWorker.ts
@@ -1,7 +1,13 @@
 import { parentPort, workerData } from 'worker_threads';
-import { workerPlainExec } from './workerRawProcessApis';
+import { _workerPlainExecImpl } from './workerRawProcessApis';
 
-workerPlainExec(workerData.file, workerData.args, workerData.options, workerData.defaultEnv, workerData.disposables)
+_workerPlainExecImpl(
+    workerData.file,
+    workerData.args,
+    workerData.options,
+    workerData.defaultEnv,
+    workerData.disposables,
+)
     .then((res) => {
         if (!parentPort) {
             throw new Error('Not in a worker thread');
diff --git a/src/client/common/process/worker/rawProcessApiWrapper.ts b/src/client/common/process/worker/rawProcessApiWrapper.ts
index dc984a1439d8..ab478fc1aa77 100644
--- a/src/client/common/process/worker/rawProcessApiWrapper.ts
+++ b/src/client/common/process/worker/rawProcessApiWrapper.ts
@@ -6,13 +6,12 @@ import * as path from 'path';
 import { WorkspaceService } from '../../application/workspace';
 import { ProcessLogger } from '../logger';
 import { executeWorkerFile } from './main';
-import { EnvironmentVariables, ExecutionResult, IDisposable, ShellOptions } from './types';
+import { EnvironmentVariables, ExecutionResult, ShellOptions } from './types';
 
-export function shellExec(
+export function workerShellExec(
     command: string,
     options: ShellOptions,
     defaultEnv?: EnvironmentVariables,
-    disposables?: Set<IDisposable>,
 ): Promise<ExecutionResult<string>> {
     const processLogger = new ProcessLogger(new WorkspaceService());
     processLogger.logProcess(command, undefined, options);
@@ -20,16 +19,14 @@ export function shellExec(
         command,
         options,
         defaultEnv,
-        disposables,
     });
 }
 
-export function plainExec(
+export function workerPlainExec(
     file: string,
     args: string[],
     options: SpawnOptions & { doNotLog?: boolean } = {},
     defaultEnv?: EnvironmentVariables,
-    disposables?: Set<IDisposable>,
 ): Promise<ExecutionResult<string>> {
     const processLogger = new ProcessLogger(new WorkspaceService());
     processLogger.logProcess(file, args, options);
@@ -38,6 +35,5 @@ export function plainExec(
         args,
         options,
         defaultEnv,
-        disposables,
     });
 }
diff --git a/src/client/common/process/worker/shellExecWorker.ts b/src/client/common/process/worker/shellExecWorker.ts
index 739d0bc1b6d5..ffaa5f435484 100644
--- a/src/client/common/process/worker/shellExecWorker.ts
+++ b/src/client/common/process/worker/shellExecWorker.ts
@@ -1,16 +1,16 @@
 import { parentPort, workerData } from 'worker_threads';
-import { workerShellExec } from './workerRawProcessApis';
+import { _workerShellExecImpl } from './workerRawProcessApis';
 
-workerShellExec(workerData.command, workerData.options, workerData.defaultEnv, workerData.disposables)
+_workerShellExecImpl(workerData.command, workerData.options, workerData.defaultEnv, workerData.disposables)
     .then((res) => {
         if (!parentPort) {
             throw new Error('Not in a worker thread');
         }
-        parentPort.postMessage(res);
+        parentPort.postMessage({ res });
     })
     .catch((ex) => {
         if (!parentPort) {
             throw new Error('Not in a worker thread');
         }
-        parentPort.postMessage(ex);
+        parentPort.postMessage({ ex });
     });
diff --git a/src/client/common/process/worker/workerRawProcessApis.ts b/src/client/common/process/worker/workerRawProcessApis.ts
index 1c1c6ca5b46b..5b04aaa40b0a 100644
--- a/src/client/common/process/worker/workerRawProcessApis.ts
+++ b/src/client/common/process/worker/workerRawProcessApis.ts
@@ -52,7 +52,7 @@ function getDefaultOptions<T extends ShellOptions | SpawnOptions>(options: T, de
     return defaultOptions;
 }
 
-export function workerShellExec(
+export function _workerShellExecImpl(
     command: string,
     options: ShellOptions,
     defaultEnv?: EnvironmentVariables,
@@ -102,7 +102,7 @@ export function workerShellExec(
     });
 }
 
-export function workerPlainExec(
+export function _workerPlainExecImpl(
     file: string,
     args: string[],
     options: SpawnOptions & { doNotLog?: boolean } = {},
diff --git a/src/client/pythonEnvironments/common/externalDependencies.ts b/src/client/pythonEnvironments/common/externalDependencies.ts
index 70ca21d4a8a5..8392ac5c0c73 100644
--- a/src/client/pythonEnvironments/common/externalDependencies.ts
+++ b/src/client/pythonEnvironments/common/externalDependencies.ts
@@ -12,7 +12,7 @@ import { getOSType, OSType } from '../../common/utils/platform';
 import { IServiceContainer } from '../../ioc/types';
 import { traceError, traceVerbose } from '../../logging';
 import { DiscoveryUsingWorkers } from '../../common/experiments/groups';
-import { plainExec, shellExec } from '../../common/process/worker/rawProcessApiWrapper';
+import { workerPlainExec, workerShellExec } from '../../common/process/worker/rawProcessApiWrapper';
 import { IEnvironmentVariablesProvider } from '../../common/variables/types';
 
 let internalServiceContainer: IServiceContainer;
@@ -23,25 +23,25 @@ export function initializeExternalDependencies(serviceContainer: IServiceContain
 // processes
 
 export async function shellExecute(command: string, options: ShellOptions = {}): Promise<ExecutionResult<string>> {
-    if (inExperiment(DiscoveryUsingWorkers.experiment)) {
+    if (!inExperiment(DiscoveryUsingWorkers.experiment)) {
         const service = await internalServiceContainer.get<IProcessServiceFactory>(IProcessServiceFactory).create();
         return service.shellExec(command, options);
     }
     const envVarsService = internalServiceContainer.get<IEnvironmentVariablesProvider>(IEnvironmentVariablesProvider);
     const envs = await envVarsService.getEnvironmentVariables();
     options.env = { ...options.env, ...envs };
-    return shellExec(command, options);
+    return workerShellExec(command, options);
 }
 
 export async function exec(file: string, args: string[], options: SpawnOptions = {}): Promise<ExecutionResult<string>> {
-    if (inExperiment(DiscoveryUsingWorkers.experiment)) {
+    if (!inExperiment(DiscoveryUsingWorkers.experiment)) {
         const service = await internalServiceContainer.get<IProcessServiceFactory>(IProcessServiceFactory).create();
         return service.exec(file, args, options);
     }
     const envVarsService = internalServiceContainer.get<IEnvironmentVariablesProvider>(IEnvironmentVariablesProvider);
     const envs = await envVarsService.getEnvironmentVariables();
     options.env = { ...options.env, ...envs };
-    return plainExec(file, args, options);
+    return workerPlainExec(file, args, options);
 }
 
 export function inExperiment(experimentName: string): boolean {

From 3315913bed10d73c7bb67bec8631a1c41a046470 Mon Sep 17 00:00:00 2001
From: Kartik Raj <karraj@microsoft.com>
Date: Fri, 17 Nov 2023 06:41:37 +0000
Subject: [PATCH 4/5] s

---
 .github/workflows/codeql-analysis.yml |  68 -------
 .github/workflows/pr-check.yml        | 273 --------------------------
 .github/workflows/pr-file-check.yml   |  44 -----
 3 files changed, 385 deletions(-)
 delete mode 100644 .github/workflows/codeql-analysis.yml
 delete mode 100644 .github/workflows/pr-file-check.yml

diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
deleted file mode 100644
index 5b037d5a1d0b..000000000000
--- a/.github/workflows/codeql-analysis.yml
+++ /dev/null
@@ -1,68 +0,0 @@
-# For most projects, this workflow file will not need changing; you simply need
-# to commit it to your repository.
-#
-# You may wish to alter this file to override the set of languages analyzed,
-# or to provide custom queries or build logic.
-name: 'CodeQL'
-
-on:
-  push:
-    branches:
-      - main
-      - release-*
-      - release/*
-  pull_request:
-    # The branches below must be a subset of the branches above
-    branches: [main]
-  schedule:
-    - cron: '0 3 * * 0'
-
-permissions:
-  security-events: write
-
-jobs:
-  analyze:
-    name: Analyze
-    runs-on: ubuntu-latest
-
-    strategy:
-      fail-fast: false
-      matrix:
-        # Override automatic language detection by changing the below list
-        # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python']
-        language: ['javascript', 'python']
-        # Learn more...
-        # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      # Initializes the CodeQL tools for scanning.
-      - name: Initialize CodeQL
-        uses: github/codeql-action/init@v2
-        with:
-          languages: ${{ matrix.language }}
-          # If you wish to specify custom queries, you can do so here or in a config file.
-          # By default, queries listed here will override any specified in a config file.
-          # Prefix the list here with "+" to use these queries and those in the config file.
-          # queries: ./path/to/local/query, your-org/your-repo/queries@main
-
-      # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
-      # If this step fails, then you should remove it and run the build manually (see below)
-      #- name: Autobuild
-      #  uses: github/codeql-action/autobuild@v1
-
-      # ℹ️ Command-line programs to run using the OS shell.
-      # 📚 https://git.io/JvXDl
-
-      # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
-      #    and modify them (or add more) to build your code if your project
-      #    uses a compiled language
-
-      #- run: |
-      #   make bootstrap
-      #   make release
-
-      - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v2
diff --git a/.github/workflows/pr-check.yml b/.github/workflows/pr-check.yml
index 3283d3281e5b..7889694c3e72 100644
--- a/.github/workflows/pr-check.yml
+++ b/.github/workflows/pr-check.yml
@@ -34,91 +34,6 @@ jobs:
           vsix_name: ${{ env.VSIX_NAME }}
           artifact_name: ${{ env.ARTIFACT_NAME_VSIX }}
 
-  lint:
-    name: Lint
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Lint
-        uses: ./.github/actions/lint
-        with:
-          node_version: ${{ env.NODE_VERSION }}
-
-  check-types:
-    name: Check Python types
-    runs-on: ubuntu-latest
-    steps:
-      - name: Use Python ${{ env.PYTHON_VERSION }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ env.PYTHON_VERSION }}
-
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Install base Python requirements
-        uses: brettcannon/pip-secure-install@v1
-        with:
-          options: '-t ./pythonFiles/lib/python --no-cache-dir --implementation py'
-
-      - name: Install Jedi requirements
-        uses: brettcannon/pip-secure-install@v1
-        with:
-          requirements-file: './pythonFiles/jedilsp_requirements/requirements.txt'
-          options: '-t ./pythonFiles/lib/jedilsp --no-cache-dir --implementation py'
-
-      - name: Install other Python requirements
-        run: |
-          python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python --no-cache-dir --implementation py --no-deps --upgrade --pre debugpy
-          python -m pip install --upgrade -r build/test-requirements.txt
-
-      - name: Run Pyright
-        uses: jakebailey/pyright-action@v1
-        with:
-          version: 1.1.308
-          working-directory: 'pythonFiles'
-
-  python-tests:
-    name: Python Tests
-    # The value of runs-on is the OS of the current job (specified in the strategy matrix below) instead of being hardcoded.
-    runs-on: ${{ matrix.os }}
-    defaults:
-      run:
-        working-directory: ${{ env.special-working-directory }}
-    strategy:
-      fail-fast: false
-      matrix:
-        # We're not running CI on macOS for now because it's one less matrix entry to lower the number of runners used,
-        # macOS runners are expensive, and we assume that Ubuntu is enough to cover the Unix case.
-        os: [ubuntu-latest, windows-latest]
-        # Run the tests on the oldest and most recent versions of Python.
-        python: ['3.8', '3.x', '3.12-dev']
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          path: ${{ env.special-working-directory-relative }}
-
-      - name: Use Python ${{ matrix.python }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python }}
-
-      - name: Install base Python requirements
-        uses: brettcannon/pip-secure-install@v1
-        with:
-          requirements-file: '"${{ env.special-working-directory-relative }}/requirements.txt"'
-          options: '-t "${{ env.special-working-directory-relative }}/pythonFiles/lib/python" --no-cache-dir --implementation py'
-
-      - name: Install test requirements
-        run: python -m pip install --upgrade -r build/test-requirements.txt
-
-      - name: Run Python unit tests
-        run: python pythonFiles/tests/run_all.py
-
   tests:
     name: Tests
     # The value of runs-on is the OS of the current job (specified in the strategy matrix below) instead of being hardcoded.
@@ -339,191 +254,3 @@ jobs:
         with:
           node_version: ${{ env.NODE_VERSION }}
           artifact_name: ${{ env.ARTIFACT_NAME_VSIX }}
-
-  ### Coverage run
-  coverage:
-    name: Coverage
-    # The value of runs-on is the OS of the current job (specified in the strategy matrix below) instead of being hardcoded.
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      matrix:
-        # Only run coverage on linux for PRs
-        os: [ubuntu-latest]
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Install Node
-        uses: actions/setup-node@v4
-        with:
-          node-version: ${{ env.NODE_VERSION }}
-          cache: 'npm'
-
-      - name: Install dependencies (npm ci)
-        run: npm ci
-
-      - name: Compile
-        run: npx gulp prePublishNonBundle
-
-      - name: Use Python ${{ env.PYTHON_VERSION }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ env.PYTHON_VERSION }}
-          cache: 'pip'
-          cache-dependency-path: |
-            requirements.txt
-            pythonFiles/jedilsp_requirements/requirements.txt
-            build/test-requirements.txt
-            build/functional-test-requirements.txt
-
-      - name: Install base Python requirements
-        uses: brettcannon/pip-secure-install@v1
-        with:
-          options: '-t ./pythonFiles/lib/python --implementation py'
-
-      - name: Install Jedi requirements
-        uses: brettcannon/pip-secure-install@v1
-        with:
-          requirements-file: './pythonFiles/jedilsp_requirements/requirements.txt'
-          options: '-t ./pythonFiles/lib/jedilsp --implementation py'
-
-      - name: Install debugpy
-        run: |
-          # We need to have debugpy so that tests relying on it keep passing, but we don't need install_debugpy's logic in the test phase.
-          python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python --implementation py --no-deps --upgrade --pre debugpy
-
-      - name: Install test requirements
-        run: python -m pip install --upgrade -r build/test-requirements.txt
-
-      - name: Install functional test requirements
-        run: python -m pip install --upgrade -r ./build/functional-test-requirements.txt
-
-      - name: Prepare pipenv for venv tests
-        env:
-          TEST_FILES_SUFFIX: testvirtualenvs
-          PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json'
-        shell: pwsh
-        run: |
-          python -m pip install pipenv
-          python -m pipenv run python ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} pipenvPath
-
-      - name: Prepare poetry for venv tests
-        env:
-          TEST_FILES_SUFFIX: testvirtualenvs
-        shell: pwsh
-        run: |
-          python -m pip install poetry
-          Move-Item -Path ".\build\ci\pyproject.toml" -Destination .
-          poetry env use python
-
-      - name: Prepare virtualenv for venv tests
-        env:
-          TEST_FILES_SUFFIX: testvirtualenvs
-          PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json'
-        shell: pwsh
-        run: |
-          python -m pip install virtualenv
-          python -m virtualenv .virtualenv/
-          if ('${{ matrix.os }}' -match 'windows-latest') {
-            & ".virtualenv/Scripts/python.exe" ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} virtualEnvPath
-          } else {
-            & ".virtualenv/bin/python" ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} virtualEnvPath
-          }
-
-      - name: Prepare venv for venv tests
-        env:
-          TEST_FILES_SUFFIX: testvirtualenvs
-          PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json'
-        shell: pwsh
-        run: |
-          python -m venv .venv
-          if ('${{ matrix.os }}' -match 'windows-latest') {
-            & ".venv/Scripts/python.exe" ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} venvPath
-          } else {
-            & ".venv/bin/python" ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} venvPath
-          }
-
-      - name: Prepare conda for venv tests
-        env:
-          TEST_FILES_SUFFIX: testvirtualenvs
-          PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json'
-        shell: pwsh
-        run: |
-          # 1. For `terminalActivation.testvirtualenvs.test.ts`
-          if ('${{ matrix.os }}' -match 'windows-latest') {
-            $condaPythonPath = Join-Path -Path $Env:CONDA -ChildPath python.exe
-            $condaExecPath = Join-Path -Path $Env:CONDA -ChildPath Scripts | Join-Path -ChildPath conda
-          } else{
-            $condaPythonPath = Join-Path -Path $Env:CONDA -ChildPath bin | Join-Path -ChildPath python
-            $condaExecPath = Join-Path -Path $Env:CONDA -ChildPath bin | Join-Path -ChildPath conda
-          }
-          & $condaPythonPath ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} condaExecPath $condaExecPath
-          & $condaPythonPath ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} condaPath
-          & $condaExecPath init --all
-
-      - name: Run TypeScript unit tests
-        run: npm run test:unittests:cover
-
-      - name: Run Python unit tests
-        run: |
-          python pythonFiles/tests/run_all.py
-
-      # The virtual environment based tests use the `testSingleWorkspace` set of tests
-      # with the environment variable `TEST_FILES_SUFFIX` set to `testvirtualenvs`,
-      # which is set in the "Prepare environment for venv tests" step.
-      # We also use a third-party GitHub Action to install xvfb on Linux,
-      # run tests and then clean up the process once the tests ran.
-      # See https://github.com/GabrielBB/xvfb-action
-      - name: Run venv tests
-        env:
-          TEST_FILES_SUFFIX: testvirtualenvs
-          CI_PYTHON_VERSION: ${{ env.PYTHON_VERSION }}
-          CI_DISABLE_AUTO_SELECTION: 1
-        uses: GabrielBB/xvfb-action@v1.6
-        with:
-          run: npm run testSingleWorkspace:cover
-
-      - name: Run single-workspace tests
-        env:
-          CI_PYTHON_VERSION: ${{ env.PYTHON_VERSION }}
-          CI_DISABLE_AUTO_SELECTION: 1
-        uses: GabrielBB/xvfb-action@v1.6
-        with:
-          run: npm run testSingleWorkspace:cover
-
-      # Enable these tests when coverage is setup for multiroot workspace tests
-      # - name: Run multi-workspace tests
-      #   env:
-      #     CI_PYTHON_VERSION: ${{ env.PYTHON_VERSION }}
-      #     CI_DISABLE_AUTO_SELECTION: 1
-      #   uses: GabrielBB/xvfb-action@v1.6
-      #   with:
-      #     run: npm run testMultiWorkspace:cover
-
-      # Enable these tests when coverage is setup for debugger tests
-      # - name: Run debugger tests
-      #   env:
-      #     CI_PYTHON_VERSION: ${{ env.PYTHON_VERSION }}
-      #     CI_DISABLE_AUTO_SELECTION: 1
-      #   uses: GabrielBB/xvfb-action@v1.6
-      #   with:
-      #     run: npm run testDebugger:cover
-
-      # Run TypeScript functional tests
-      - name: Run TypeScript functional tests
-        env:
-          CI_PYTHON_VERSION: ${{ env.PYTHON_VERSION }}
-          CI_DISABLE_AUTO_SELECTION: 1
-        run: npm run test:functional:cover
-
-      - name: Generate coverage reports
-        run: npm run test:cover:report
-
-      - name: Upload HTML report
-        uses: actions/upload-artifact@v3
-        with:
-          name: ${{ runner.os }}-coverage-report-html
-          path: ./coverage
-          retention-days: 1
diff --git a/.github/workflows/pr-file-check.yml b/.github/workflows/pr-file-check.yml
deleted file mode 100644
index ba019c790e99..000000000000
--- a/.github/workflows/pr-file-check.yml
+++ /dev/null
@@ -1,44 +0,0 @@
-name: PR files
-
-on:
-  pull_request:
-    types:
-      # On by default if you specify no types.
-      - 'opened'
-      - 'reopened'
-      - 'synchronize'
-      # For `skip-label` only.
-      - 'labeled'
-      - 'unlabeled'
-
-jobs:
-  changed-files-in-pr:
-    name: 'Check for changed files'
-    runs-on: ubuntu-latest
-    steps:
-      - name: 'package-lock.json matches package.json'
-        uses: brettcannon/check-for-changed-files@v1.2.0
-        with:
-          prereq-pattern: 'package.json'
-          file-pattern: 'package-lock.json'
-          skip-label: 'skip package*.json'
-          failure-message: '${prereq-pattern} was edited but ${file-pattern} was not (the ${skip-label} label can be used to pass this check)'
-
-      - name: 'package.json matches package-lock.json'
-        uses: brettcannon/check-for-changed-files@v1.2.0
-        with:
-          prereq-pattern: 'package-lock.json'
-          file-pattern: 'package.json'
-          skip-label: 'skip package*.json'
-          failure-message: '${prereq-pattern} was edited but ${file-pattern} was not (the ${skip-label} label can be used to pass this check)'
-
-      - name: 'Tests'
-        uses: brettcannon/check-for-changed-files@v1.2.0
-        with:
-          prereq-pattern: src/**/*.ts
-          file-pattern: |
-            src/**/*.test.ts
-            src/**/*.testvirtualenvs.ts
-            .github/test_plan.md
-          skip-label: 'skip tests'
-          failure-message: 'TypeScript code was edited without also editing a ${file-pattern} file; see the Testing page in our wiki on testing guidelines (the ${skip-label} label can be used to pass this check)'

From 300e22429269ec528109ad5947c1c3f63de1d6b8 Mon Sep 17 00:00:00 2001
From: Kartik Raj <karraj@microsoft.com>
Date: Fri, 17 Nov 2023 06:43:58 +0000
Subject: [PATCH 5/5] asd

---
 .github/workflows/pr-check.yml | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/.github/workflows/pr-check.yml b/.github/workflows/pr-check.yml
index 7889694c3e72..f6692af3bc5f 100644
--- a/.github/workflows/pr-check.yml
+++ b/.github/workflows/pr-check.yml
@@ -2,10 +2,6 @@ name: PR/CI Check
 
 on:
   pull_request:
-  push:
-    branches-ignore:
-      - main
-      - release*
 
 env:
   NODE_VERSION: 18.17.1
@@ -243,7 +239,7 @@ jobs:
       matrix:
         # We're not running CI on macOS for now because it's one less matrix entry to lower the number of runners used,
         # macOS runners are expensive, and we assume that Ubuntu is enough to cover the UNIX case.
-        os: [ubuntu-latest, windows-latest]
+        os: [ubuntu-latest]
     steps:
       # Need the source to have the tests available.
       - name: Checkout