Refactor to use the windows installer when creating a base environment

Python_Engine/Compute/BasePythonEnvironment.cs

@@ -33,11 +33,14 @@ namespace BH.Engine.Python
 {
     public static partial class Compute
     {
+        [PreviousVersion("8.0", "BH.Engine.Python.Compute.BasePythonEnvironment(System.Boolean, System.Boolean)")]
         [Description("Retrieve or reinstall the base Python Environment for BHoM workflows.")]
+        [Input("version", "The target version of python to be installed or retrieved.")]
         [Input("reload", "Reload the base Python environment rather than recreating it, if it already exists.")]
         [Input("run", "Start the installation/retrieval of the BHoM Base Python Environment.")]
         [Output("env", "The base Python Environment for all BHoM workflows.")]
         public static PythonEnvironment BasePythonEnvironment(
+            PythonVersion version = PythonVersion.v3_10,
             bool reload = true,
             bool run = false
         )
@@ -52,10 +55,10 @@ public static PythonEnvironment BasePythonEnvironment(
                 // create PythonEnvironments directory if it doesnt already exist
                 Directory.CreateDirectory(Query.DirectoryEnvironments());
             }
-            
+
             // determine whether the base environment already exists
-            string targetExecutable = Path.Combine(Query.DirectoryBaseEnvironment(), "python.exe");
-            bool exists = Directory.Exists(Query.DirectoryBaseEnvironment()) && File.Exists(targetExecutable);
+            string targetExecutable = Path.Combine(Query.DirectoryBaseEnvironment(version), "python.exe");
+            bool exists = File.Exists(targetExecutable);
 
             if (exists && reload)
                 return new PythonEnvironment() { Name = Query.ToolkitName(), Executable = targetExecutable };
@@ -64,50 +67,14 @@ public static PythonEnvironment BasePythonEnvironment(
                 // remove all existing environments and kernels
                 RemoveEverything();
 
-            // download the target Python version and convert into a "full" python installation bypassing admin rights
-            string executable = PythonVersion.v3_10_5.DownloadPython(Query.ToolkitName());
-            string pipInstaller = DownloadGetPip(Path.GetDirectoryName(executable));
-            string baseEnvironmentDirectory = Path.GetDirectoryName(executable);
-
-            // install pip into the python installation
-            Process process = new Process()
-            {
-                StartInfo = new ProcessStartInfo()
-                {
-                    FileName = Modify.AddQuotesIfRequired(executable),
-                    Arguments = Modify.AddQuotesIfRequired(pipInstaller) + " --no-warn-script-location",
-                    RedirectStandardError=true,
-                    UseShellExecute=false,
-                }
-            };
-            using (Process p = Process.Start(process.StartInfo))
-            {
-                string standardError = p.StandardError.ReadToEnd();
-                p.WaitForExit();
-                if (p.ExitCode != 0)
-                    BH.Engine.Base.Compute.RecordError($"Error installing pip.\n{standardError}");
-                File.Delete(pipInstaller);
-            }
-
-            // delete files with the suffix ._pth from installedDirectory
-            List<string> pthFiles = Directory.GetFiles(baseEnvironmentDirectory, "*.*", SearchOption.TopDirectoryOnly).Where(s => s.EndsWith("._pth")).ToList();
-            foreach (string pthFile in pthFiles)
-            {
-                File.Delete(pthFile);
-            }
-
-            // move files with the suffix .dll and .pyd from installedDirectory into a DLLs directory
-            string libDirectory = Directory.CreateDirectory(Path.Combine(baseEnvironmentDirectory, "DLLs")).FullName;
-            List<string> libFiles = Directory.GetFiles(baseEnvironmentDirectory, "*.*", SearchOption.TopDirectoryOnly).Where(s => (s.EndsWith(".dll") || s.EndsWith(".pyd")) && !Path.GetFileName(s).Contains("python") && !Path.GetFileName(s).Contains("vcruntime")).ToList();
-            foreach (string libFile in libFiles)
-            {
-                File.Move(libFile, Path.Combine(libDirectory, Path.GetFileName(libFile)));
-            }
+            // download and run the installer for the target Python version
+            string exe = version.DownloadPythonVersion();
 
             // install essential packages into base environment
-            InstallPackages(executable, new List<string>() { "virtualenv", "jupyterlab", "black", "pylint" });
+            InstallPackages(exe, new List<string>() { "virtualenv", "jupyterlab", "black", "pylint" });
+            InstallPackageLocal(exe, Path.Combine(Query.DirectoryCode(), Query.ToolkitName()));
 
-            return new PythonEnvironment() { Name = Query.ToolkitName(), Executable = executable };
+            return new PythonEnvironment() { Name = Query.ToolkitName(), Executable = exe };
         }
     }
 }

Python_Engine/Compute/Download.cs

@@ -24,7 +24,9 @@
 using BH.oM.Python.Enums;
 using System;
 using System.ComponentModel;
+using System.Diagnostics;
 using System.IO;
+using System.Reflection;
 using System.Xml.Linq;
 
 namespace BH.Engine.Python
@@ -85,34 +87,53 @@
             return filePath;
         }
 
-        // TODO - THIS METHOD HAS CHANGED BUT IS STILL USED, SO NEEDS DEPRECATING
-        // changed from what to what ? 
-        [Description("Download the target version of Python.")]
-        [Input("version", "A Python version.")]
-        [Input("name", "Name of target exe file.")]
-        [Output("executablePath", "The path of the executable for the downloaded Python.")]
-        public static string DownloadPython(this PythonVersion version, string name = null)
+        /******************************************************/
+
+        [PreviousVersion("8.0", "BH.Engine.Python.Compute.DownloadPython(BH.oM.Python.Enums.PythonVersion, System.String)")]
+        [Description("Download and install a specified version of python, and return the executable for it.")]
+        [Input("version", "The version of python to download.")]
+        [Output("pythonExecutable", "The executable (python.exe) for the python version that was installed")]
+        public static string DownloadPythonVersion(this PythonVersion version)
         {
             string url = version.EmbeddableURL();
-            if (string.IsNullOrEmpty(name))
-                name = Path.GetFileNameWithoutExtension(url);
-            string targetExecutable = Path.Combine(Query.DirectoryEnvironments(), name, "python.exe");
-
-            if (File.Exists(targetExecutable))
-                return targetExecutable;
-
-            string zipfile = DownloadFile(url, Query.DirectoryEnvironments());
-            UnzipFile(zipfile, Query.DirectoryEnvironments(), name, true);
-
-            return targetExecutable;
-        }
 
-        [Description("Download the pip installer")]
-        [Input("targetDirectory", "The directory into which get-pip.py will be downloaded.")]
-        [Output("getpipPath", "The path of the file used to install pip into an embedded Python environment.")]
-        public static string DownloadGetPip(string targetDirectory)
-        {
-            return DownloadFile("https://bootstrap.pypa.io/get-pip.py", targetDirectory);
+            string basePath = Path.Combine(Query.DirectoryBaseEnvironment(version));
+
+            if (File.Exists(Path.Combine(basePath, "python.exe")))
+                return Path.Combine(basePath, "python.exe");
+
+            if (!Directory.Exists(basePath))
+                Directory.CreateDirectory(basePath);
+            else
+            {
+                Directory.Delete(basePath, true); //if there are any files here already for some reason, remove them.
+                Directory.CreateDirectory(basePath);
+            }
+
+            string installerFile = DownloadFile(url, basePath, "installer.exe");
+
+            using (Process install = new Process()
+            {
+                StartInfo = new ProcessStartInfo()
+                {
+                    FileName = installerFile,
+                    Arguments = $"/passive InstallAllUsers=0 InstallLauncherAllUsers=0 Include_launcher=0 Shortcuts=0 AssociateFiles=0 Include_tools=0 Include_test=0 TargetDir={Modify.AddQuotesIfRequired(basePath)}",
+                    RedirectStandardError = true,
+                    UseShellExecute = false,
+                }
+            })
+            {
+                install.Start();
+                string stderr = install.StandardError.ReadToEnd();
+                install.WaitForExit();
+                if (install.ExitCode != 0)
+                {
+                    BH.Engine.Base.Compute.RecordError($"Error installing python: {stderr}");
+                    return null;
+                }
+            }
+
+            return Path.Combine(basePath, "python.exe");
         }
     }
 }

Python_Engine/Compute/Remove.cs

@@ -21,7 +21,11 @@
  */
 
 using BH.oM.Base.Attributes;
+using BH.oM.Python.Enums;
+using System;
+using System.Collections.Generic;
 using System.ComponentModel;
+using System.Diagnostics;
 using System.IO;
 
 namespace BH.Engine.Python
@@ -77,19 +81,60 @@ public static void RemoveAllVirtualEnvironments()
             }
         }
 
-        [Description("Completely remove the base BHoM Python environment.")]
-        public static void RemoveBaseEnvironment()
+        [PreviousVersion("8.0", "BH.Engine.Python.Compute.RemoveBaseEnvironment()")]
+        [Description("Remove the base install for the python version specified.")]
+        [Input("version", "The base python version to remove.")]
+        public static void RemoveBaseVersion(PythonVersion version = PythonVersion.v3_10)
         {
-            string basePath = Path.Combine(Query.DirectoryEnvironments(), Query.ToolkitName());
-            if (Directory.Exists(basePath))
+            string basePath = Path.Combine(Query.DirectoryEnvironments(), Query.ToolkitName(), version.ToString());
+            string baseInstaller = Path.Combine(basePath, "installer.exe");
+
+            // If the installer does not exist and the folder does - assume a bad/invalid install and just delete the entire folder.
+            if (!File.Exists(baseInstaller) && Directory.Exists(basePath))
+            {
                 Directory.Delete(basePath, true);
+                return;
+            }
+            else if (!Directory.Exists(basePath))
+                // If the base path doesn't exist there is nothing to remove.
+                return;
+
+            using (Process uninstall = new Process()
+            {
+                StartInfo = new ProcessStartInfo()
+                {
+                    FileName = baseInstaller,
+                    Arguments = "/uninstall /passive",
+                    RedirectStandardError = true,
+                    UseShellExecute = false,
+                }
+            })
+            {
+                uninstall.Start();
+                string stderr = uninstall.StandardError.ReadToEnd();
+                uninstall.WaitForExit();
+                if (uninstall.ExitCode != 0)
+                {
+                    BH.Engine.Base.Compute.RecordError($"Error uninstalling python: {stderr}");
+                    return;
+                }
+            }
+
+            // Finally remove base folder as the installer may have missed something.
+            Directory.Delete(basePath, true);
         }
 
         [Description("Completely remove all BHoM-related Python environments and kernels.")]
         public static void RemoveEverything()
         {
             RemoveAllVirtualEnvironments();
-            RemoveBaseEnvironment();
+
+            //remove all python versions installed in base directory.
+            foreach (PythonVersion e in Enum.GetValues(typeof(PythonVersion)))
+            {
+                if (e != PythonVersion.Undefined)
+                    RemoveBaseVersion(e);
+            }
         }
     }
 }

Python_Engine/Compute/UnzipFile.cs

@@ -29,6 +29,7 @@ namespace BH.Engine.Python
 {
     public static partial class Compute
     {
+        //This method is no longer used by python toolkit, and perhaps should be removed or moved to the file toolkit instead.
         [Description("Extract the contents of an archive.")]
         [Input("archivePath", "The archive to extract.")]
         [Input("targetDirectory", "The destination directory to extract into.")]

Python_Engine/Compute/VirtualEnvironment.cs

@@ -20,6 +20,7 @@
  * along with this code. If not, see <https://www.gnu.org/licenses/lgpl-3.0.html>.      
  */
 
+using BH.Engine.Base;
 using BH.oM.Base.Attributes;
 using BH.oM.Python;
 using BH.oM.Python.Enums;
@@ -45,22 +46,17 @@ public static PythonEnvironment VirtualEnvironment(this PythonVersion version, s
                 BH.Engine.Base.Compute.RecordError("A BHoM Python virtual environment cannot cannot contain invalid filepath characters.");
                 return null;
             }
-            if (version == PythonVersion.Undefined)
-            {
-                BH.Engine.Base.Compute.RecordError("Please provide a version of Python.");
-                return null;
-            }
 
-            // check that base environment is installed and return null and raise error if it isn't
-            string baseEnvironmentExecutable = Path.Combine(Query.DirectoryBaseEnvironment(), "python.exe");
-            if (!File.Exists(baseEnvironmentExecutable))
+            if (version == PythonVersion.Undefined)
             {
-                BH.Engine.Base.Compute.RecordWarning("The base Python environment doesnt seem to be installed. Install it first in order to run this method.");
+                BH.Engine.Base.Compute.RecordError("Please provide a valid Python version.");
                 return null;
             }
 
             string targetExecutable = Query.VirtualEnvironmentExecutable(name);
             string targetDirectory = Query.VirtualEnvironmentDirectory(name);
+            string versionExecutable = Path.Combine(Query.DirectoryBaseEnvironment(version), "python.exe");
+
             bool exists = Query.VirtualEnvironmentExists(name);
 
             if (exists && reload)
@@ -73,31 +69,28 @@ public static PythonEnvironment VirtualEnvironment(this PythonVersion version, s
                 RemoveKernel(name);
             }
 
-            // download the target version of Python
-            string referencedExecutable = version.DownloadPython();
-
-            // move the directory containing referencedExecutable into Query.DirectoryBaseEnvironment() using the same name
-            string sourceDirectory = Path.GetDirectoryName(referencedExecutable);
-            string destinationDirectory = Path.Combine(Query.DirectoryBaseEnvironment(), new DirectoryInfo(Path.GetDirectoryName(referencedExecutable)).Name);
-            if (!Directory.Exists(destinationDirectory))
+            if (!File.Exists(versionExecutable))
             {
-                Directory.Move(sourceDirectory, destinationDirectory);
+                // The output here should be the same, but to be sure replace the value.
+                BH.Engine.Base.Compute.RecordNote($"The base environment for the requested version {version} has been installed as it was not present.");
+                versionExecutable = BasePythonEnvironment(version, run: true)?.Executable;
+
+                if (versionExecutable == null) // If the executable is null, then python wasn't installed correctly, so return null - the error message for downloading the version should suffice.
+                    return null;
             }
-            if (Directory.Exists(sourceDirectory))
-                Directory.Delete(sourceDirectory, true);
-            referencedExecutable = Path.Combine(destinationDirectory, "python.exe");
 
             // create the venv from the base environment
             Process process = new Process()
             {
                 StartInfo = new ProcessStartInfo()
                 {
-                    FileName = Modify.AddQuotesIfRequired(baseEnvironmentExecutable),
-                    Arguments = $"-m virtualenv --python={Modify.AddQuotesIfRequired(referencedExecutable)} {Modify.AddQuotesIfRequired(targetDirectory)}",
+                    FileName = Modify.AddQuotesIfRequired(versionExecutable),
+                    Arguments = $"-m virtualenv --python={Modify.AddQuotesIfRequired(versionExecutable)} {Modify.AddQuotesIfRequired(targetDirectory)}",
                     RedirectStandardError = true,
                     UseShellExecute = false,
                 }
             };
+
             using (Process p = Process.Start(process.StartInfo))
             {
                 string standardError = p.StandardError.ReadToEnd();
@@ -120,15 +113,14 @@ public static PythonEnvironment VirtualEnvironment(this PythonVersion version, s
                     UseShellExecute = false,
                 }
             };
+
             using (Process p = Process.Start(process2.StartInfo))
             {
                 string standardError = p.StandardError.ReadToEnd();
                 p.WaitForExit();
                 if (p.ExitCode != 0)
                     BH.Engine.Base.Compute.RecordError($"Error registering the \"{name}\" virtual environment.\n{standardError}");
             }
-            // replace text in a file
-
 
             return new PythonEnvironment() { Executable = targetExecutable, Name = name };
         }

Python_Engine/Convert/ToPython.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * This file is part of the Buildings and Habitats object Model (BHoM)
  * Copyright (c) 2015 - 2024, the respective contributors. All rights reserved.
  *
@@ -38,4 +38,4 @@ public static bool ToPython<T>(this T[,] input)
             return false;
         }
     }
-}
+}

Python_Engine/Python/src/python_toolkit/bhom/__init__.py

@@ -0,0 +1,23 @@
+"""Root for the bhom subpackage."""
+
+from pathlib import Path  # pylint: disable=E0401
+from os import path
+
+from win32api import HIWORD, LOWORD, GetFileVersionInfo
+
+BHOM_ASSEMBLIES_DIRECTORY = Path(path.expandvars("%PROGRAMDATA%/BHoM/Assemblies"))
+BHOM_LOG_FOLDER = Path(path.expandvars("%PROGRAMDATA%/BHoM/Logs"))
+TOOLKIT_NAME = "Python_Toolkit"
+
+if not BHOM_LOG_FOLDER.exists():
+    BHOM_LOG_FOLDER = Path(path.expandvars("%TEMP%/BHoMLogs"))
+    BHOM_LOG_FOLDER.mkdir(exist_ok=True)
+
+if not BHOM_ASSEMBLIES_DIRECTORY.exists():
+    BHOM_VERSION = ""
+else:
+    _file_version_ms = GetFileVersionInfo(
+        (BHOM_ASSEMBLIES_DIRECTORY / "BHoM.dll").as_posix(), "\\"
+    )["FileVersionMS"]
+
+    BHOM_VERSION = f"{HIWORD(_file_version_ms)}.{LOWORD(_file_version_ms)}"