From 7fa170b86defaca751f58cf02de729e1ad9c7cf5 Mon Sep 17 00:00:00 2001 From: Thomas Edward Kingstone Date: Thu, 5 Sep 2024 15:28:41 +0100 Subject: [PATCH 01/24] initial solution for the refactor. python installer now used instead of embedded packages, and removed most versions from the python version enum. Utilise the installer to remove the environment, and allowed multiplt versions of python to be installed in the python toolkit folder --- .../Compute/BasePythonEnvironment.cs | 49 ++-------- Python_Engine/Compute/Download.cs | 78 ++++++++++++---- Python_Engine/Compute/Remove.cs | 54 ++++++++++- Python_Engine/Compute/VirtualEnvironment.cs | 25 ++--- Python_Engine/Query/EmbeddableURL.cs | 93 ++++++++++--------- Python_oM/Enums/PythonVersion.cs | 29 +++--- 6 files changed, 187 insertions(+), 141 deletions(-) diff --git a/Python_Engine/Compute/BasePythonEnvironment.cs b/Python_Engine/Compute/BasePythonEnvironment.cs index c06db95..7bf8210 100644 --- a/Python_Engine/Compute/BasePythonEnvironment.cs +++ b/Python_Engine/Compute/BasePythonEnvironment.cs @@ -54,8 +54,8 @@ public static PythonEnvironment BasePythonEnvironment( } // determine whether the base environment already exists - string targetExecutable = Path.Combine(Query.DirectoryBaseEnvironment(), "python.exe"); - bool exists = Directory.Exists(Query.DirectoryBaseEnvironment()) && File.Exists(targetExecutable); + string targetExecutable = Path.Combine(Query.DirectoryBaseEnvironment(), PythonVersion.v3_10_11.ToString(), "python.exe"); + bool exists = File.Exists(targetExecutable); if (exists && reload) return new PythonEnvironment() { Name = Query.ToolkitName(), Executable = targetExecutable }; @@ -64,50 +64,13 @@ public static PythonEnvironment BasePythonEnvironment( // remove all existing environments and kernels RemoveEverything(); - // download the target Python version and convert into a "full" python installation bypassing admin rights - string executable = PythonVersion.v3_10_5.DownloadPython(Query.ToolkitName()); - string pipInstaller = DownloadGetPip(Path.GetDirectoryName(executable)); - string baseEnvironmentDirectory = Path.GetDirectoryName(executable); - - // install pip into the python installation - Process process = new Process() - { - StartInfo = new ProcessStartInfo() - { - FileName = Modify.AddQuotesIfRequired(executable), - Arguments = Modify.AddQuotesIfRequired(pipInstaller) + " --no-warn-script-location", - RedirectStandardError=true, - UseShellExecute=false, - } - }; - using (Process p = Process.Start(process.StartInfo)) - { - string standardError = p.StandardError.ReadToEnd(); - p.WaitForExit(); - if (p.ExitCode != 0) - BH.Engine.Base.Compute.RecordError($"Error installing pip.\n{standardError}"); - File.Delete(pipInstaller); - } - - // delete files with the suffix ._pth from installedDirectory - List pthFiles = Directory.GetFiles(baseEnvironmentDirectory, "*.*", SearchOption.TopDirectoryOnly).Where(s => s.EndsWith("._pth")).ToList(); - foreach (string pthFile in pthFiles) - { - File.Delete(pthFile); - } - - // move files with the suffix .dll and .pyd from installedDirectory into a DLLs directory - string libDirectory = Directory.CreateDirectory(Path.Combine(baseEnvironmentDirectory, "DLLs")).FullName; - List libFiles = Directory.GetFiles(baseEnvironmentDirectory, "*.*", SearchOption.TopDirectoryOnly).Where(s => (s.EndsWith(".dll") || s.EndsWith(".pyd")) && !Path.GetFileName(s).Contains("python") && !Path.GetFileName(s).Contains("vcruntime")).ToList(); - foreach (string libFile in libFiles) - { - File.Move(libFile, Path.Combine(libDirectory, Path.GetFileName(libFile))); - } + // download the installer for the target Python version + string exe = PythonVersion.v3_10_11.DownloadPythonVersion(); // install essential packages into base environment - InstallPackages(executable, new List() { "virtualenv", "jupyterlab", "black", "pylint" }); + InstallPackages(exe, new List() { "virtualenv", "jupyterlab", "black", "pylint" }); - return new PythonEnvironment() { Name = Query.ToolkitName(), Executable = executable }; + return new PythonEnvironment() { Name = Query.ToolkitName(), Executable = exe }; } } } diff --git a/Python_Engine/Compute/Download.cs b/Python_Engine/Compute/Download.cs index 0c8c81f..2df3d87 100644 --- a/Python_Engine/Compute/Download.cs +++ b/Python_Engine/Compute/Download.cs @@ -24,7 +24,9 @@ using BH.oM.Python.Enums; using System; using System.ComponentModel; +using System.Diagnostics; using System.IO; +using System.Reflection; using System.Xml.Linq; namespace BH.Engine.Python @@ -84,35 +86,77 @@ public static string DownloadFile( return filePath; } - + /* // TODO - THIS METHOD HAS CHANGED BUT IS STILL USED, SO NEEDS DEPRECATING - // changed from what to what ? - [Description("Download the target version of Python.")] + // changed from what to what ? + [Description("Download the installer for the target version of python.")] [Input("version", "A Python version.")] - [Input("name", "Name of target exe file.")] - [Output("executablePath", "The path of the executable for the downloaded Python.")] + [Input("name", "Name of the toolkit for this installation of python.")] + [Output("executablePath", "The path of the executable for the downloaded Python installer.")] public static string DownloadPython(this PythonVersion version, string name = null) { string url = version.EmbeddableURL(); if (string.IsNullOrEmpty(name)) name = Path.GetFileNameWithoutExtension(url); - string targetExecutable = Path.Combine(Query.DirectoryEnvironments(), name, "python.exe"); - + + string targetExecutable = Path.Combine(Query.DirectoryBaseEnvironment(), version.ToString(), "installer.exe"); + if (File.Exists(targetExecutable)) return targetExecutable; - - string zipfile = DownloadFile(url, Query.DirectoryEnvironments()); - UnzipFile(zipfile, Query.DirectoryEnvironments(), name, true); - return targetExecutable; - } + if (!Directory.Exists(Path.Combine(Query.DirectoryEnvironments(), name))) + Directory.CreateDirectory(Path.Combine(Query.DirectoryEnvironments(), name)); + + string exefile = DownloadFile(url, Path.Combine(Query.DirectoryEnvironments(), name)); - [Description("Download the pip installer")] - [Input("targetDirectory", "The directory into which get-pip.py will be downloaded.")] - [Output("getpipPath", "The path of the file used to install pip into an embedded Python environment.")] - public static string DownloadGetPip(string targetDirectory) + return exefile; + }*/ + + [PreviousVersion("7.3", "BH.Engine.Python.Compute.DownloadPython(BH.oM.Python.PythonVersion, System.String)")] + [Description("Download and install a specified version of python, and return the executable for it.")] + [Input("version", "The version of python to download.")] + [Output("pythonExecutable", "The executable (python.exe) for the python version that was installed")] + public static string DownloadPythonVersion(this PythonVersion version) { - return DownloadFile("https://bootstrap.pypa.io/get-pip.py", targetDirectory); + string url = version.EmbeddableURL(); + + string basePath = Path.Combine(Query.DirectoryBaseEnvironment(), version.ToString()); + + if (File.Exists(Path.Combine(basePath, "python.exe"))) + return Path.Combine(basePath, "python.exe"); + + if (!Directory.Exists(basePath)) + Directory.CreateDirectory(basePath); + else + { + Directory.Delete(basePath, true); //if there are any files here already for some reason, remove them. + Directory.CreateDirectory(basePath); + } + + string installerFile = DownloadFile(url, basePath, "installer.exe"); + + using (Process install = new Process() + { + StartInfo = new ProcessStartInfo() + { + FileName = installerFile, + Arguments = $"/passive InstallAllUsers=0 InstallLauncherAllUsers=0 Include_launcher=0 Shortcuts=0 AssociateFiles=0 Include_tools=0 Include_test=0 TargetDir={Modify.AddQuotesIfRequired(basePath)}", + RedirectStandardError = true, + UseShellExecute = false, + } + }) + { + install.Start(); + string stderr = install.StandardError.ReadToEnd(); + install.WaitForExit(); + if (install.ExitCode != 0) + { + BH.Engine.Base.Compute.RecordError($"Error installing python: {stderr}"); + return null; + } + } + + return Path.Combine(basePath, "python.exe"); } } } diff --git a/Python_Engine/Compute/Remove.cs b/Python_Engine/Compute/Remove.cs index 4ee414f..16ed227 100644 --- a/Python_Engine/Compute/Remove.cs +++ b/Python_Engine/Compute/Remove.cs @@ -21,7 +21,11 @@ */ using BH.oM.Base.Attributes; +using BH.oM.Python.Enums; +using System; +using System.Collections.Generic; using System.ComponentModel; +using System.Diagnostics; using System.IO; namespace BH.Engine.Python @@ -77,19 +81,59 @@ public static void RemoveAllVirtualEnvironments() } } - [Description("Completely remove the base BHoM Python environment.")] - public static void RemoveBaseEnvironment() + [PreviousVersion("7.3", "BH.Engine.Python.Compute.RemoveBaseEnvironment()")] + [Input("version", "The base python version to remove.")] + public static void RemoveBaseVersion(PythonVersion version = PythonVersion.v3_10_11) { - string basePath = Path.Combine(Query.DirectoryEnvironments(), Query.ToolkitName()); - if (Directory.Exists(basePath)) + string basePath = Path.Combine(Query.DirectoryBaseEnvironment(), version.ToString()); + string baseInstaller = Path.Combine(basePath, "installer.exe"); + + // If the installer does not exist and the folder does - assume a bad/invalid install and just delete the entire folder. + if (!File.Exists(baseInstaller) && Directory.Exists(basePath)) + { Directory.Delete(basePath, true); + return; + } + else if (!Directory.Exists(basePath)) + // If the base path doesn't exist there is nothing to remove. + return; + + using (Process uninstall = new Process() + { + StartInfo = new ProcessStartInfo() + { + FileName = baseInstaller, + Arguments = "/uninstall /passive", + RedirectStandardError = true, + UseShellExecute = false, + } + }) + { + uninstall.Start(); + string stderr = uninstall.StandardError.ReadToEnd(); + uninstall.WaitForExit(); + if (uninstall.ExitCode != 0) + { + BH.Engine.Base.Compute.RecordError($"Error uninstalling python: {stderr}"); + return; + } + } + + // Finally remove base folder as the installer may have missed something. + Directory.Delete(basePath, true); } [Description("Completely remove all BHoM-related Python environments and kernels.")] public static void RemoveEverything() { RemoveAllVirtualEnvironments(); - RemoveBaseEnvironment(); + + //remove all python versions installed in base directory. + foreach (PythonVersion e in Enum.GetValues(typeof(PythonVersion))) + { + if (e != PythonVersion.Undefined) + RemoveBaseVersion(e); + } } } } diff --git a/Python_Engine/Compute/VirtualEnvironment.cs b/Python_Engine/Compute/VirtualEnvironment.cs index 0046bdc..f62b828 100644 --- a/Python_Engine/Compute/VirtualEnvironment.cs +++ b/Python_Engine/Compute/VirtualEnvironment.cs @@ -20,6 +20,7 @@ * along with this code. If not, see . */ +using BH.Engine.Base; using BH.oM.Base.Attributes; using BH.oM.Python; using BH.oM.Python.Enums; @@ -52,7 +53,7 @@ public static PythonEnvironment VirtualEnvironment(this PythonVersion version, s } // check that base environment is installed and return null and raise error if it isn't - string baseEnvironmentExecutable = Path.Combine(Query.DirectoryBaseEnvironment(), "python.exe"); + string baseEnvironmentExecutable = Path.Combine(Query.DirectoryBaseEnvironment(), PythonVersion.v3_10_11.ToString(), "python.exe"); if (!File.Exists(baseEnvironmentExecutable)) { BH.Engine.Base.Compute.RecordWarning("The base Python environment doesnt seem to be installed. Install it first in order to run this method."); @@ -61,6 +62,8 @@ public static PythonEnvironment VirtualEnvironment(this PythonVersion version, s string targetExecutable = Query.VirtualEnvironmentExecutable(name); string targetDirectory = Query.VirtualEnvironmentDirectory(name); + string versionExecutable = Path.Combine(Query.DirectoryBaseEnvironment(), version.ToString(), "python.exe"); + bool exists = Query.VirtualEnvironmentExists(name); if (exists && reload) @@ -73,19 +76,9 @@ public static PythonEnvironment VirtualEnvironment(this PythonVersion version, s RemoveKernel(name); } - // download the target version of Python - string referencedExecutable = version.DownloadPython(); - - // move the directory containing referencedExecutable into Query.DirectoryBaseEnvironment() using the same name - string sourceDirectory = Path.GetDirectoryName(referencedExecutable); - string destinationDirectory = Path.Combine(Query.DirectoryBaseEnvironment(), new DirectoryInfo(Path.GetDirectoryName(referencedExecutable)).Name); - if (!Directory.Exists(destinationDirectory)) - { - Directory.Move(sourceDirectory, destinationDirectory); - } - if (Directory.Exists(sourceDirectory)) - Directory.Delete(sourceDirectory, true); - referencedExecutable = Path.Combine(destinationDirectory, "python.exe"); + if (!File.Exists(versionExecutable)) + // The output here should be the same, but to be sure replace the value. + versionExecutable = version.DownloadPythonVersion(); // create the venv from the base environment Process process = new Process() @@ -93,11 +86,12 @@ public static PythonEnvironment VirtualEnvironment(this PythonVersion version, s StartInfo = new ProcessStartInfo() { FileName = Modify.AddQuotesIfRequired(baseEnvironmentExecutable), - Arguments = $"-m virtualenv --python={Modify.AddQuotesIfRequired(referencedExecutable)} {Modify.AddQuotesIfRequired(targetDirectory)}", + Arguments = $"-m virtualenv --python={Modify.AddQuotesIfRequired(versionExecutable)} {Modify.AddQuotesIfRequired(targetDirectory)}", RedirectStandardError = true, UseShellExecute = false, } }; + using (Process p = Process.Start(process.StartInfo)) { string standardError = p.StandardError.ReadToEnd(); @@ -129,7 +123,6 @@ public static PythonEnvironment VirtualEnvironment(this PythonVersion version, s } // replace text in a file - return new PythonEnvironment() { Executable = targetExecutable, Name = name }; } } diff --git a/Python_Engine/Query/EmbeddableURL.cs b/Python_Engine/Query/EmbeddableURL.cs index 33d2333..62dcc5e 100644 --- a/Python_Engine/Query/EmbeddableURL.cs +++ b/Python_Engine/Query/EmbeddableURL.cs @@ -34,54 +34,55 @@ public static partial class Query [Output("url", "The url for the matching Python version.")] public static string EmbeddableURL(this PythonVersion version) { + // See the PythonVersion enum for why most of these are commented out. Dictionary versions = new Dictionary() { - //{ PythonVersion.v3_7_2, "https://www.python.org/ftp/python/3.7.2/python-3.7.2rc1-embed-amd64.zip" }, - { PythonVersion.v3_7_3, "https://www.python.org/ftp/python/3.7.3/python-3.7.3rc1-embed-amd64.zip" }, - { PythonVersion.v3_7_4, "https://www.python.org/ftp/python/3.7.4/python-3.7.4rc2-embed-amd64.zip" }, - { PythonVersion.v3_7_5, "https://www.python.org/ftp/python/3.7.5/python-3.7.5rc1-embed-amd64.zip" }, - { PythonVersion.v3_7_6, "https://www.python.org/ftp/python/3.7.6/python-3.7.6rc1-embed-amd64.zip" }, - { PythonVersion.v3_7_7, "https://www.python.org/ftp/python/3.7.7/python-3.7.7rc1-embed-amd64.zip" }, - { PythonVersion.v3_7_8, "https://www.python.org/ftp/python/3.7.8/python-3.7.8rc1-embed-amd64.zip" }, - { PythonVersion.v3_7_9, "https://www.python.org/ftp/python/3.7.9/python-3.7.9-embed-amd64.zip" }, - { PythonVersion.v3_8_0, "https://www.python.org/ftp/python/3.8.0/python-3.8.0rc1-embed-amd64.zip" }, - { PythonVersion.v3_8_1, "https://www.python.org/ftp/python/3.8.1/python-3.8.1rc1-embed-amd64.zip" }, - { PythonVersion.v3_8_2, "https://www.python.org/ftp/python/3.8.2/python-3.8.2rc2-embed-amd64.zip" }, - { PythonVersion.v3_8_3, "https://www.python.org/ftp/python/3.8.3/python-3.8.3rc1-embed-amd64.zip" }, - { PythonVersion.v3_8_4, "https://www.python.org/ftp/python/3.8.4/python-3.8.4rc1-embed-amd64.zip" }, - { PythonVersion.v3_8_5, "https://www.python.org/ftp/python/3.8.5/python-3.8.5-embed-amd64.zip" }, - { PythonVersion.v3_8_6, "https://www.python.org/ftp/python/3.8.6/python-3.8.6rc1-embed-amd64.zip" }, - { PythonVersion.v3_8_7, "https://www.python.org/ftp/python/3.8.7/python-3.8.7rc1-embed-amd64.zip" }, - { PythonVersion.v3_8_8, "https://www.python.org/ftp/python/3.8.8/python-3.8.8rc1-embed-amd64.zip" }, - { PythonVersion.v3_8_9, "https://www.python.org/ftp/python/3.8.9/python-3.8.9-embed-amd64.zip" }, - { PythonVersion.v3_9_0, "https://www.python.org/ftp/python/3.9.0/python-3.9.0rc2-embed-amd64.zip" }, - { PythonVersion.v3_9_1, "https://www.python.org/ftp/python/3.9.1/python-3.9.1rc1-embed-amd64.zip" }, - { PythonVersion.v3_9_2, "https://www.python.org/ftp/python/3.9.2/python-3.9.2rc1-embed-amd64.zip" }, - { PythonVersion.v3_9_3, "https://www.python.org/ftp/python/3.9.3/python-3.9.3-embed-amd64.zip" }, - { PythonVersion.v3_9_4, "https://www.python.org/ftp/python/3.9.4/python-3.9.4-embed-amd64.zip" }, - { PythonVersion.v3_9_5, "https://www.python.org/ftp/python/3.9.5/python-3.9.5-embed-amd64.zip" }, - { PythonVersion.v3_9_6, "https://www.python.org/ftp/python/3.9.6/python-3.9.6-embed-amd64.zip" }, - { PythonVersion.v3_9_7, "https://www.python.org/ftp/python/3.9.7/python-3.9.7-embed-amd64.zip" }, - { PythonVersion.v3_9_8, "https://www.python.org/ftp/python/3.9.8/python-3.9.8-embed-amd64.zip" }, - { PythonVersion.v3_9_9, "https://www.python.org/ftp/python/3.9.9/python-3.9.9-embed-amd64.zip" }, - { PythonVersion.v3_9_10, "https://www.python.org/ftp/python/3.9.10/python-3.9.10-embed-amd64.zip" }, - { PythonVersion.v3_10_0, "https://www.python.org/ftp/python/3.10.0/python-3.10.0rc2-embed-amd64.zip" }, - { PythonVersion.v3_10_1, "https://www.python.org/ftp/python/3.10.1/python-3.10.1-embed-amd64.zip" }, - { PythonVersion.v3_10_2, "https://www.python.org/ftp/python/3.10.2/python-3.10.2-embed-amd64.zip" }, - { PythonVersion.v3_10_3, "https://www.python.org/ftp/python/3.10.3/python-3.10.3-embed-amd64.zip" }, - { PythonVersion.v3_10_4, "https://www.python.org/ftp/python/3.10.4/python-3.10.4-embed-amd64.zip" }, - { PythonVersion.v3_10_5, "https://www.python.org/ftp/python/3.10.5/python-3.10.5-embed-amd64.zip" }, - { PythonVersion.v3_10_6, "https://www.python.org/ftp/python/3.10.6/python-3.10.6-embed-amd64.zip" }, - { PythonVersion.v3_10_7, "https://www.python.org/ftp/python/3.10.7/python-3.10.7-embed-amd64.zip" }, - { PythonVersion.v3_10_8, "https://www.python.org/ftp/python/3.10.8/python-3.10.8-embed-amd64.zip" }, - { PythonVersion.v3_10_9, "https://www.python.org/ftp/python/3.10.9/python-3.10.9-embed-amd64.zip" }, - { PythonVersion.v3_10_10, "https://www.python.org/ftp/python/3.10.10/python-3.10.10-embed-amd64.zip" }, - { PythonVersion.v3_10_11, "https://www.python.org/ftp/python/3.10.11/python-3.10.11-embed-amd64.zip" }, - { PythonVersion.v3_11_0, "https://www.python.org/ftp/python/3.11.0/python-3.11.0rc2-embed-amd64.zip" }, - { PythonVersion.v3_11_1, "https://www.python.org/ftp/python/3.11.1/python-3.11.1-embed-amd64.zip" }, - { PythonVersion.v3_11_2, "https://www.python.org/ftp/python/3.11.2/python-3.11.2-embed-amd64.zip" }, - { PythonVersion.v3_11_3, "https://www.python.org/ftp/python/3.11.3/python-3.11.3-embed-amd64.zip" }, - { PythonVersion.v3_11_4, "https://www.python.org/ftp/python/3.11.4/python-3.11.4-embed-amd64.zip" }, + //{ PythonVersion.v3_7_2, "https://www.python.org/ftp/python/3.7.2/python-3.7.2rc1-amd64.exe" }, + /*{ PythonVersion.v3_7_3, "https://www.python.org/ftp/python/3.7.3/python-3.7.3rc1-amd64.exe" }, + { PythonVersion.v3_7_4, "https://www.python.org/ftp/python/3.7.4/python-3.7.4rc2-amd64.exe" }, + { PythonVersion.v3_7_5, "https://www.python.org/ftp/python/3.7.5/python-3.7.5rc1-amd64.exe" }, + { PythonVersion.v3_7_6, "https://www.python.org/ftp/python/3.7.6/python-3.7.6rc1-amd64.exe" }, + { PythonVersion.v3_7_7, "https://www.python.org/ftp/python/3.7.7/python-3.7.7rc1-amd64.exe" }, + { PythonVersion.v3_7_8, "https://www.python.org/ftp/python/3.7.8/python-3.7.8rc1-amd64.exe" },*/ + { PythonVersion.v3_7_9, "https://www.python.org/ftp/python/3.7.9/python-3.7.9-amd64.exe" }, + /*{ PythonVersion.v3_8_0, "https://www.python.org/ftp/python/3.8.0/python-3.8.0rc1-amd64.exe" }, + { PythonVersion.v3_8_1, "https://www.python.org/ftp/python/3.8.1/python-3.8.1rc1-amd64.exe" }, + { PythonVersion.v3_8_2, "https://www.python.org/ftp/python/3.8.2/python-3.8.2rc2-amd64.exe" }, + { PythonVersion.v3_8_3, "https://www.python.org/ftp/python/3.8.3/python-3.8.3rc1-amd64.exe" }, + { PythonVersion.v3_8_4, "https://www.python.org/ftp/python/3.8.4/python-3.8.4rc1-amd64.exe" }, + { PythonVersion.v3_8_5, "https://www.python.org/ftp/python/3.8.5/python-3.8.5-amd64.exe" }, + { PythonVersion.v3_8_6, "https://www.python.org/ftp/python/3.8.6/python-3.8.6rc1-amd64.exe" }, + { PythonVersion.v3_8_7, "https://www.python.org/ftp/python/3.8.7/python-3.8.7rc1-amd64.exe" }, + { PythonVersion.v3_8_8, "https://www.python.org/ftp/python/3.8.8/python-3.8.8rc1-amd64.exe" },*/ + { PythonVersion.v3_8_9, "https://www.python.org/ftp/python/3.8.9/python-3.8.9-amd64.exe" }, + /*{ PythonVersion.v3_9_0, "https://www.python.org/ftp/python/3.9.0/python-3.9.0rc2-amd64.exe" }, + { PythonVersion.v3_9_1, "https://www.python.org/ftp/python/3.9.1/python-3.9.1rc1-amd64.exe" }, + { PythonVersion.v3_9_2, "https://www.python.org/ftp/python/3.9.2/python-3.9.2rc1-amd64.exe" }, + { PythonVersion.v3_9_3, "https://www.python.org/ftp/python/3.9.3/python-3.9.3-amd64.exe" }, + { PythonVersion.v3_9_4, "https://www.python.org/ftp/python/3.9.4/python-3.9.4-amd64.exe" }, + { PythonVersion.v3_9_5, "https://www.python.org/ftp/python/3.9.5/python-3.9.5-amd64.exe" }, + { PythonVersion.v3_9_6, "https://www.python.org/ftp/python/3.9.6/python-3.9.6-amd64.exe" }, + { PythonVersion.v3_9_7, "https://www.python.org/ftp/python/3.9.7/python-3.9.7-amd64.exe" }, + { PythonVersion.v3_9_8, "https://www.python.org/ftp/python/3.9.8/python-3.9.8-amd64.exe" }, + { PythonVersion.v3_9_9, "https://www.python.org/ftp/python/3.9.9/python-3.9.9-amd64.exe" },*/ + { PythonVersion.v3_9_10, "https://www.python.org/ftp/python/3.9.10/python-3.9.10-amd64.exe" }, + /*{ PythonVersion.v3_10_0, "https://www.python.org/ftp/python/3.10.0/python-3.10.0rc2-amd64.exe" }, + { PythonVersion.v3_10_1, "https://www.python.org/ftp/python/3.10.1/python-3.10.1-amd64.exe" }, + { PythonVersion.v3_10_2, "https://www.python.org/ftp/python/3.10.2/python-3.10.2-amd64.exe" }, + { PythonVersion.v3_10_3, "https://www.python.org/ftp/python/3.10.3/python-3.10.3-amd64.exe" }, + { PythonVersion.v3_10_4, "https://www.python.org/ftp/python/3.10.4/python-3.10.4-amd64.exe" }, + { PythonVersion.v3_10_5, "https://www.python.org/ftp/python/3.10.5/python-3.10.5-amd64.exe" }, + { PythonVersion.v3_10_6, "https://www.python.org/ftp/python/3.10.6/python-3.10.6-amd64.exe" }, + { PythonVersion.v3_10_7, "https://www.python.org/ftp/python/3.10.7/python-3.10.7-amd64.exe" }, + { PythonVersion.v3_10_8, "https://www.python.org/ftp/python/3.10.8/python-3.10.8-amd64.exe" }, + { PythonVersion.v3_10_9, "https://www.python.org/ftp/python/3.10.9/python-3.10.9-amd64.exe" }, + { PythonVersion.v3_10_10, "https://www.python.org/ftp/python/3.10.10/python-3.10.10-amd64.exe" },*/ + { PythonVersion.v3_10_11, "https://www.python.org/ftp/python/3.10.11/python-3.10.11-amd64.exe" }, + /*{ PythonVersion.v3_11_0, "https://www.python.org/ftp/python/3.11.0/python-3.11.0rc2-amd64.exe" }, + { PythonVersion.v3_11_1, "https://www.python.org/ftp/python/3.11.1/python-3.11.1-amd64.exe" }, + { PythonVersion.v3_11_2, "https://www.python.org/ftp/python/3.11.2/python-3.11.2-amd64.exe" }, + { PythonVersion.v3_11_3, "https://www.python.org/ftp/python/3.11.3/python-3.11.3-amd64.exe" },*/ + { PythonVersion.v3_11_4, "https://www.python.org/ftp/python/3.11.4/python-3.11.4-amd64.exe" }, }; return versions[version]; diff --git a/Python_oM/Enums/PythonVersion.cs b/Python_oM/Enums/PythonVersion.cs index 1b078e9..6964653 100644 --- a/Python_oM/Enums/PythonVersion.cs +++ b/Python_oM/Enums/PythonVersion.cs @@ -24,16 +24,17 @@ namespace BH.oM.Python.Enums { + // There are multiple specific versions here that are commented out - When installing, only one specific version (e.g 3.10.x) per minor python version (e.g 3.x) can be installed, and changes between specific versions are usually bug fixes and not breaking. however the specific versions are kept commented for posterity. public enum PythonVersion { [Description("Undefined")] Undefined, - [Description("3.7.0")] + /*[Description("3.7.0")] v3_7_0, [Description("3.7.1")] v3_7_1, - //[Description("3.7.2")] - //v3_7_2, + [Description("3.7.2")] + v3_7_2, [Description("3.7.3")] v3_7_3, [Description("3.7.4")] @@ -45,10 +46,10 @@ public enum PythonVersion [Description("3.7.7")] v3_7_7, [Description("3.7.8")] - v3_7_8, + v3_7_8,*/ [Description("3.7.9")] v3_7_9, - [Description("3.8.0")] + /*[Description("3.8.0")] v3_8_0, [Description("3.8.1")] v3_8_1, @@ -65,12 +66,12 @@ public enum PythonVersion [Description("3.8.7")] v3_8_7, [Description("3.8.8")] - v3_8_8, + v3_8_8,*/ [Description("3.8.9")] v3_8_9, - [Description("3.8.10")] - v3_8_10, - [Description("3.9.0")] + //[Description("3.8.10")] + //v3_8_10, + /*[Description("3.9.0")] v3_9_0, [Description("3.9.1")] v3_9_1, @@ -89,10 +90,10 @@ public enum PythonVersion [Description("3.9.8")] v3_9_8, [Description("3.9.9")] - v3_9_9, + v3_9_9,*/ [Description("3.9.10")] v3_9_10, - [Description("3.10.0")] + /*[Description("3.10.0")] v3_10_0, [Description("3.10.1")] v3_10_1, @@ -113,17 +114,17 @@ public enum PythonVersion [Description("3.10.9")] v3_10_9, [Description("3.10.10")] - v3_10_10, + v3_10_10,*/ [Description("3.10.11")] v3_10_11, - [Description("3.11.0")] + /*[Description("3.11.0")] v3_11_0, [Description("3.11.1")] v3_11_1, [Description("3.11.2")] v3_11_2, [Description("3.11.3")] - v3_11_3, + v3_11_3,*/ [Description("3.11.4")] v3_11_4, } From 2d61d7954410b42da26f72d58512681037ea557b Mon Sep 17 00:00:00 2001 From: Thomas Edward Kingstone Date: Thu, 5 Sep 2024 16:19:02 +0100 Subject: [PATCH 02/24] rename the python versions to be just the major numbers to avoid confusion --- .../Compute/BasePythonEnvironment.cs | 4 ++-- Python_Engine/Compute/Remove.cs | 2 +- Python_Engine/Compute/VirtualEnvironment.cs | 2 +- Python_Engine/Query/Directory.cs | 6 +++--- Python_Engine/Query/EmbeddableURL.cs | 10 +++++----- Python_Engine/Query/Version.cs | 8 +++++++- Python_oM/Enums/PythonVersion.cs | 20 +++++++++---------- 7 files changed, 29 insertions(+), 23 deletions(-) diff --git a/Python_Engine/Compute/BasePythonEnvironment.cs b/Python_Engine/Compute/BasePythonEnvironment.cs index 7bf8210..b85ec53 100644 --- a/Python_Engine/Compute/BasePythonEnvironment.cs +++ b/Python_Engine/Compute/BasePythonEnvironment.cs @@ -54,7 +54,7 @@ public static PythonEnvironment BasePythonEnvironment( } // determine whether the base environment already exists - string targetExecutable = Path.Combine(Query.DirectoryBaseEnvironment(), PythonVersion.v3_10_11.ToString(), "python.exe"); + string targetExecutable = Path.Combine(Query.DirectoryBaseEnvironment(), PythonVersion.v3_10.ToString(), "python.exe"); bool exists = File.Exists(targetExecutable); if (exists && reload) @@ -65,7 +65,7 @@ public static PythonEnvironment BasePythonEnvironment( RemoveEverything(); // download the installer for the target Python version - string exe = PythonVersion.v3_10_11.DownloadPythonVersion(); + string exe = PythonVersion.v3_10.DownloadPythonVersion(); // install essential packages into base environment InstallPackages(exe, new List() { "virtualenv", "jupyterlab", "black", "pylint" }); diff --git a/Python_Engine/Compute/Remove.cs b/Python_Engine/Compute/Remove.cs index 16ed227..b0ca140 100644 --- a/Python_Engine/Compute/Remove.cs +++ b/Python_Engine/Compute/Remove.cs @@ -83,7 +83,7 @@ public static void RemoveAllVirtualEnvironments() [PreviousVersion("7.3", "BH.Engine.Python.Compute.RemoveBaseEnvironment()")] [Input("version", "The base python version to remove.")] - public static void RemoveBaseVersion(PythonVersion version = PythonVersion.v3_10_11) + public static void RemoveBaseVersion(PythonVersion version = PythonVersion.v3_10) { string basePath = Path.Combine(Query.DirectoryBaseEnvironment(), version.ToString()); string baseInstaller = Path.Combine(basePath, "installer.exe"); diff --git a/Python_Engine/Compute/VirtualEnvironment.cs b/Python_Engine/Compute/VirtualEnvironment.cs index f62b828..42c3de1 100644 --- a/Python_Engine/Compute/VirtualEnvironment.cs +++ b/Python_Engine/Compute/VirtualEnvironment.cs @@ -53,7 +53,7 @@ public static PythonEnvironment VirtualEnvironment(this PythonVersion version, s } // check that base environment is installed and return null and raise error if it isn't - string baseEnvironmentExecutable = Path.Combine(Query.DirectoryBaseEnvironment(), PythonVersion.v3_10_11.ToString(), "python.exe"); + string baseEnvironmentExecutable = Path.Combine(Query.DirectoryBaseEnvironment(), PythonVersion.v3_10.ToString(), "python.exe"); if (!File.Exists(baseEnvironmentExecutable)) { BH.Engine.Base.Compute.RecordWarning("The base Python environment doesnt seem to be installed. Install it first in order to run this method."); diff --git a/Python_Engine/Query/Directory.cs b/Python_Engine/Query/Directory.cs index f9d5ed9..34f16ed 100644 --- a/Python_Engine/Query/Directory.cs +++ b/Python_Engine/Query/Directory.cs @@ -21,7 +21,7 @@ */ using BH.oM.Base.Attributes; - +using BH.oM.Python.Enums; using System.ComponentModel; using System.IO; @@ -69,9 +69,9 @@ public static string DirectoryEnvironments() [Description("The location where the base Python environment exists.")] [Output("The location where the base Python environment exists.")] - public static string DirectoryBaseEnvironment() + public static string DirectoryBaseEnvironment(PythonVersion version = PythonVersion.v3_10) { - return Path.Combine(Query.DirectoryEnvironments(), Query.ToolkitName()); + return Path.Combine(Query.DirectoryEnvironments(), Query.ToolkitName(), version.ToString()); } } } diff --git a/Python_Engine/Query/EmbeddableURL.cs b/Python_Engine/Query/EmbeddableURL.cs index 62dcc5e..72dd0fe 100644 --- a/Python_Engine/Query/EmbeddableURL.cs +++ b/Python_Engine/Query/EmbeddableURL.cs @@ -44,7 +44,7 @@ public static string EmbeddableURL(this PythonVersion version) { PythonVersion.v3_7_6, "https://www.python.org/ftp/python/3.7.6/python-3.7.6rc1-amd64.exe" }, { PythonVersion.v3_7_7, "https://www.python.org/ftp/python/3.7.7/python-3.7.7rc1-amd64.exe" }, { PythonVersion.v3_7_8, "https://www.python.org/ftp/python/3.7.8/python-3.7.8rc1-amd64.exe" },*/ - { PythonVersion.v3_7_9, "https://www.python.org/ftp/python/3.7.9/python-3.7.9-amd64.exe" }, + { PythonVersion.v3_7, "https://www.python.org/ftp/python/3.7.9/python-3.7.9-amd64.exe" }, /*{ PythonVersion.v3_8_0, "https://www.python.org/ftp/python/3.8.0/python-3.8.0rc1-amd64.exe" }, { PythonVersion.v3_8_1, "https://www.python.org/ftp/python/3.8.1/python-3.8.1rc1-amd64.exe" }, { PythonVersion.v3_8_2, "https://www.python.org/ftp/python/3.8.2/python-3.8.2rc2-amd64.exe" }, @@ -54,7 +54,7 @@ public static string EmbeddableURL(this PythonVersion version) { PythonVersion.v3_8_6, "https://www.python.org/ftp/python/3.8.6/python-3.8.6rc1-amd64.exe" }, { PythonVersion.v3_8_7, "https://www.python.org/ftp/python/3.8.7/python-3.8.7rc1-amd64.exe" }, { PythonVersion.v3_8_8, "https://www.python.org/ftp/python/3.8.8/python-3.8.8rc1-amd64.exe" },*/ - { PythonVersion.v3_8_9, "https://www.python.org/ftp/python/3.8.9/python-3.8.9-amd64.exe" }, + { PythonVersion.v3_8, "https://www.python.org/ftp/python/3.8.9/python-3.8.9-amd64.exe" }, /*{ PythonVersion.v3_9_0, "https://www.python.org/ftp/python/3.9.0/python-3.9.0rc2-amd64.exe" }, { PythonVersion.v3_9_1, "https://www.python.org/ftp/python/3.9.1/python-3.9.1rc1-amd64.exe" }, { PythonVersion.v3_9_2, "https://www.python.org/ftp/python/3.9.2/python-3.9.2rc1-amd64.exe" }, @@ -65,7 +65,7 @@ public static string EmbeddableURL(this PythonVersion version) { PythonVersion.v3_9_7, "https://www.python.org/ftp/python/3.9.7/python-3.9.7-amd64.exe" }, { PythonVersion.v3_9_8, "https://www.python.org/ftp/python/3.9.8/python-3.9.8-amd64.exe" }, { PythonVersion.v3_9_9, "https://www.python.org/ftp/python/3.9.9/python-3.9.9-amd64.exe" },*/ - { PythonVersion.v3_9_10, "https://www.python.org/ftp/python/3.9.10/python-3.9.10-amd64.exe" }, + { PythonVersion.v3_9, "https://www.python.org/ftp/python/3.9.10/python-3.9.10-amd64.exe" }, /*{ PythonVersion.v3_10_0, "https://www.python.org/ftp/python/3.10.0/python-3.10.0rc2-amd64.exe" }, { PythonVersion.v3_10_1, "https://www.python.org/ftp/python/3.10.1/python-3.10.1-amd64.exe" }, { PythonVersion.v3_10_2, "https://www.python.org/ftp/python/3.10.2/python-3.10.2-amd64.exe" }, @@ -77,12 +77,12 @@ public static string EmbeddableURL(this PythonVersion version) { PythonVersion.v3_10_8, "https://www.python.org/ftp/python/3.10.8/python-3.10.8-amd64.exe" }, { PythonVersion.v3_10_9, "https://www.python.org/ftp/python/3.10.9/python-3.10.9-amd64.exe" }, { PythonVersion.v3_10_10, "https://www.python.org/ftp/python/3.10.10/python-3.10.10-amd64.exe" },*/ - { PythonVersion.v3_10_11, "https://www.python.org/ftp/python/3.10.11/python-3.10.11-amd64.exe" }, + { PythonVersion.v3_10, "https://www.python.org/ftp/python/3.10.11/python-3.10.11-amd64.exe" }, /*{ PythonVersion.v3_11_0, "https://www.python.org/ftp/python/3.11.0/python-3.11.0rc2-amd64.exe" }, { PythonVersion.v3_11_1, "https://www.python.org/ftp/python/3.11.1/python-3.11.1-amd64.exe" }, { PythonVersion.v3_11_2, "https://www.python.org/ftp/python/3.11.2/python-3.11.2-amd64.exe" }, { PythonVersion.v3_11_3, "https://www.python.org/ftp/python/3.11.3/python-3.11.3-amd64.exe" },*/ - { PythonVersion.v3_11_4, "https://www.python.org/ftp/python/3.11.4/python-3.11.4-amd64.exe" }, + { PythonVersion.v3_11, "https://www.python.org/ftp/python/3.11.4/python-3.11.4-amd64.exe" }, }; return versions[version]; diff --git a/Python_Engine/Query/Version.cs b/Python_Engine/Query/Version.cs index 1b3bb28..470a654 100644 --- a/Python_Engine/Query/Version.cs +++ b/Python_Engine/Query/Version.cs @@ -23,9 +23,11 @@ using BH.oM.Base.Attributes; using BH.oM.Python.Enums; using System; +using System.Collections.Generic; using System.ComponentModel; using System.Diagnostics; using System.IO; +using System.Linq; namespace BH.Engine.Python { @@ -68,7 +70,11 @@ public static PythonVersion Version(string pythonExecutable) } } - return (PythonVersion)Enum.Parse(typeof(PythonVersion), "v" + versionString.Replace("Python ", "").Replace(".", "_")); + List strings = versionString.Replace("Python ", "").Replace(".", "_").Split('_').ToList(); + strings.RemoveAt(strings.Count - 1); + string enumParseable = "v" + strings.Aggregate((a, b) => $"{a}_{b}"); + + return (PythonVersion)Enum.Parse(typeof(PythonVersion), enumParseable); } catch { diff --git a/Python_oM/Enums/PythonVersion.cs b/Python_oM/Enums/PythonVersion.cs index 6964653..6ee7881 100644 --- a/Python_oM/Enums/PythonVersion.cs +++ b/Python_oM/Enums/PythonVersion.cs @@ -47,8 +47,8 @@ public enum PythonVersion v3_7_7, [Description("3.7.8")] v3_7_8,*/ - [Description("3.7.9")] - v3_7_9, + [Description("3.7")] //3.7.9 + v3_7, /*[Description("3.8.0")] v3_8_0, [Description("3.8.1")] @@ -67,8 +67,8 @@ public enum PythonVersion v3_8_7, [Description("3.8.8")] v3_8_8,*/ - [Description("3.8.9")] - v3_8_9, + [Description("3.8")] //3.8.9 + v3_8, //[Description("3.8.10")] //v3_8_10, /*[Description("3.9.0")] @@ -91,8 +91,8 @@ public enum PythonVersion v3_9_8, [Description("3.9.9")] v3_9_9,*/ - [Description("3.9.10")] - v3_9_10, + [Description("3.9")] //3.9.10 + v3_9, /*[Description("3.10.0")] v3_10_0, [Description("3.10.1")] @@ -115,8 +115,8 @@ public enum PythonVersion v3_10_9, [Description("3.10.10")] v3_10_10,*/ - [Description("3.10.11")] - v3_10_11, + [Description("3.10")] //3.10.11 + v3_10, /*[Description("3.11.0")] v3_11_0, [Description("3.11.1")] @@ -125,8 +125,8 @@ public enum PythonVersion v3_11_2, [Description("3.11.3")] v3_11_3,*/ - [Description("3.11.4")] - v3_11_4, + [Description("3.11")] //3.11.4 + v3_11, } } From bbb5621024ef202399679740ad4f9a76182f14ab Mon Sep 17 00:00:00 2001 From: Thomas Edward Kingstone Date: Mon, 9 Sep 2024 10:15:34 +0100 Subject: [PATCH 03/24] fixed remove method --- Python_Engine/Compute/Remove.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python_Engine/Compute/Remove.cs b/Python_Engine/Compute/Remove.cs index b0ca140..ee1b4d6 100644 --- a/Python_Engine/Compute/Remove.cs +++ b/Python_Engine/Compute/Remove.cs @@ -85,7 +85,7 @@ public static void RemoveAllVirtualEnvironments() [Input("version", "The base python version to remove.")] public static void RemoveBaseVersion(PythonVersion version = PythonVersion.v3_10) { - string basePath = Path.Combine(Query.DirectoryBaseEnvironment(), version.ToString()); + string basePath = Path.Combine(Query.DirectoryEnvironments(), Query.ToolkitName(), version.ToString()); string baseInstaller = Path.Combine(basePath, "installer.exe"); // If the installer does not exist and the folder does - assume a bad/invalid install and just delete the entire folder. From badc7a9bff621287c985ae047e6e8dda6f966dbe Mon Sep 17 00:00:00 2001 From: Thomas Edward Kingstone Date: Mon, 9 Sep 2024 10:27:59 +0100 Subject: [PATCH 04/24] fixed base environment install location --- Python_Engine/Compute/BasePythonEnvironment.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Python_Engine/Compute/BasePythonEnvironment.cs b/Python_Engine/Compute/BasePythonEnvironment.cs index b85ec53..e7c0f13 100644 --- a/Python_Engine/Compute/BasePythonEnvironment.cs +++ b/Python_Engine/Compute/BasePythonEnvironment.cs @@ -52,9 +52,9 @@ public static PythonEnvironment BasePythonEnvironment( // create PythonEnvironments directory if it doesnt already exist Directory.CreateDirectory(Query.DirectoryEnvironments()); } - + // determine whether the base environment already exists - string targetExecutable = Path.Combine(Query.DirectoryBaseEnvironment(), PythonVersion.v3_10.ToString(), "python.exe"); + string targetExecutable = Path.Combine(Query.DirectoryBaseEnvironment(), "python.exe"); bool exists = File.Exists(targetExecutable); if (exists && reload) From 15d41fee171c66e4d88f430e2eb419bcc70946fe Mon Sep 17 00:00:00 2001 From: Thomas Edward Kingstone Date: Mon, 9 Sep 2024 10:37:18 +0100 Subject: [PATCH 05/24] use DirectoryBaseEnvironment properly everywhere --- Python_Engine/Compute/Download.cs | 2 +- Python_Engine/Compute/VirtualEnvironment.cs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Python_Engine/Compute/Download.cs b/Python_Engine/Compute/Download.cs index 2df3d87..a7e4096 100644 --- a/Python_Engine/Compute/Download.cs +++ b/Python_Engine/Compute/Download.cs @@ -120,7 +120,7 @@ public static string DownloadPythonVersion(this PythonVersion version) { string url = version.EmbeddableURL(); - string basePath = Path.Combine(Query.DirectoryBaseEnvironment(), version.ToString()); + string basePath = Path.Combine(Query.DirectoryBaseEnvironment(version)); if (File.Exists(Path.Combine(basePath, "python.exe"))) return Path.Combine(basePath, "python.exe"); diff --git a/Python_Engine/Compute/VirtualEnvironment.cs b/Python_Engine/Compute/VirtualEnvironment.cs index 42c3de1..c77a4fc 100644 --- a/Python_Engine/Compute/VirtualEnvironment.cs +++ b/Python_Engine/Compute/VirtualEnvironment.cs @@ -53,7 +53,7 @@ public static PythonEnvironment VirtualEnvironment(this PythonVersion version, s } // check that base environment is installed and return null and raise error if it isn't - string baseEnvironmentExecutable = Path.Combine(Query.DirectoryBaseEnvironment(), PythonVersion.v3_10.ToString(), "python.exe"); + string baseEnvironmentExecutable = Path.Combine(Query.DirectoryBaseEnvironment(), "python.exe"); if (!File.Exists(baseEnvironmentExecutable)) { BH.Engine.Base.Compute.RecordWarning("The base Python environment doesnt seem to be installed. Install it first in order to run this method."); @@ -62,7 +62,7 @@ public static PythonEnvironment VirtualEnvironment(this PythonVersion version, s string targetExecutable = Query.VirtualEnvironmentExecutable(name); string targetDirectory = Query.VirtualEnvironmentDirectory(name); - string versionExecutable = Path.Combine(Query.DirectoryBaseEnvironment(), version.ToString(), "python.exe"); + string versionExecutable = Path.Combine(Query.DirectoryBaseEnvironment(version), "python.exe"); bool exists = Query.VirtualEnvironmentExists(name); From 1131f9ec132ae51337ca389d536aa7197c5de508 Mon Sep 17 00:00:00 2001 From: Thomas Edward Kingstone Date: Fri, 13 Sep 2024 14:16:41 +0100 Subject: [PATCH 06/24] added bhom analytics logger, and added the interpreter path (sys.executable) to CustomData within the log message. --- .../Python/src/python_toolkit/__init__.py | 1 - .../src/python_toolkit/bhom/__init__.py | 23 +++++ .../src/python_toolkit/bhom/analytics.py | 95 +++++++++++++++++++ .../src/python_toolkit/bhom/bhom.mplstyle | 81 ++++++++++++++++ .../python_toolkit/bhom/logging/__init__.py | 2 + .../python_toolkit/bhom/logging/console.py | 19 ++++ .../src/python_toolkit/bhom/logging/file.py | 28 ++++++ .../Python/src/python_toolkit/bhom/util.py | 24 +++++ .../src/python_toolkit/bhomutil/__init__.py | 1 - 9 files changed, 272 insertions(+), 2 deletions(-) delete mode 100644 Python_Engine/Python/src/python_toolkit/__init__.py create mode 100644 Python_Engine/Python/src/python_toolkit/bhom/__init__.py create mode 100644 Python_Engine/Python/src/python_toolkit/bhom/analytics.py create mode 100644 Python_Engine/Python/src/python_toolkit/bhom/bhom.mplstyle create mode 100644 Python_Engine/Python/src/python_toolkit/bhom/logging/__init__.py create mode 100644 Python_Engine/Python/src/python_toolkit/bhom/logging/console.py create mode 100644 Python_Engine/Python/src/python_toolkit/bhom/logging/file.py create mode 100644 Python_Engine/Python/src/python_toolkit/bhom/util.py delete mode 100644 Python_Engine/Python/src/python_toolkit/bhomutil/__init__.py diff --git a/Python_Engine/Python/src/python_toolkit/__init__.py b/Python_Engine/Python/src/python_toolkit/__init__.py deleted file mode 100644 index 5f28270..0000000 --- a/Python_Engine/Python/src/python_toolkit/__init__.py +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/Python_Engine/Python/src/python_toolkit/bhom/__init__.py b/Python_Engine/Python/src/python_toolkit/bhom/__init__.py new file mode 100644 index 0000000..8a69ac5 --- /dev/null +++ b/Python_Engine/Python/src/python_toolkit/bhom/__init__.py @@ -0,0 +1,23 @@ +"""Root for the bhom subpackage.""" + +from pathlib import Path # pylint: disable=E0401 +from os import path + +from win32api import HIWORD, LOWORD, GetFileVersionInfo + +BHOM_ASSEMBLIES_DIRECTORY = Path(path.expandvars("%PROGRAMDATA%/BHoM/Assemblies")) +BHOM_LOG_FOLDER = Path(path.expandvars("%PROGRAMDATA%/BHoM/Logs")) +TOOLKIT_NAME = "Python_Toolkit" + +if not BHOM_LOG_FOLDER.exists(): + BHOM_LOG_FOLDER = Path(path.expandvars("%TEMP%/BHoMLogs")) + BHOM_LOG_FOLDER.mkdir(exist_ok=True) + +if not BHOM_ASSEMBLIES_DIRECTORY.exists(): + BHOM_VERSION = "" +else: + _file_version_ms = GetFileVersionInfo( + (BHOM_ASSEMBLIES_DIRECTORY / "BHoM.dll").as_posix(), "\\" + )["FileVersionMS"] + + BHOM_VERSION = f"{HIWORD(_file_version_ms)}.{LOWORD(_file_version_ms)}" diff --git a/Python_Engine/Python/src/python_toolkit/bhom/analytics.py b/Python_Engine/Python/src/python_toolkit/bhom/analytics.py new file mode 100644 index 0000000..318377b --- /dev/null +++ b/Python_Engine/Python/src/python_toolkit/bhom/analytics.py @@ -0,0 +1,95 @@ +"""BHoM analytics decorator.""" +# pylint: disable=E0401 +import inspect +import json +import sys +import uuid +from functools import wraps +from typing import Any, Callable + +# pylint: enable=E0401 + +from .logging import ANALYTICS_LOGGER +from .util import csharp_ticks +from . import BHOM_VERSION, TOOLKIT_NAME + + +def bhom_analytics() -> Callable: + """Decorator for capturing usage data. + + Returns + ------- + Callable + The decorated function. + """ + + def decorator(function: Callable): + """A decorator to capture usage data for called methods/functions. + + Arguments + --------- + function : Callable + The function to decorate. + + Returns + ------- + Callable + The decorated function. + """ + + @wraps(function) + def wrapper(*args, **kwargs) -> Any: + """A wrapper around the function that captures usage analytics.""" + + _id = uuid.uuid4() + + # get the data being passed to the function, expected dtype and return type + argspec = inspect.getfullargspec(function)[-1] + argspec.pop("return", None) + + _args = [f'{{"_t": "{argspec[k]}", "Name": "{k}"}}' for k in argspec.keys()] + + exec_metadata = { + "BHoMVersion": BHOM_VERSION, + "BHoM_Guid": _id, + "CallerName": function.__name__, + "ComponentId": _id, + "CustomData": {"interpreter", sys.executable}, + "Errors": [], + "FileId": "", + "FileName": "", + "Fragments": [], + "Name": "", + # TODO - get project properties from another function/logging + # method (or from BHoM DLL analytics capture ...) + "ProjectID": "", + "SelectedItem": { + "MethodName": function.__name__, + "Parameters": _args, + "TypeName": f"{function.__module__}.{function.__qualname__}", + "_bhomVersion": BHOM_VERSION, + "_t": "Python", + }, + "Time": { + "$date": csharp_ticks(short=True), + }, + "UI": "Python", + "UiVersion": TOOLKIT_NAME, + "_t": "BH.oM.UI.UsageLogEntry", + } + + try: + result = function(*args, **kwargs) + except Exception as exc: # pylint: disable=broad-except + exec_metadata["Errors"].extend(sys.exc_info()) + raise exc + finally: + ANALYTICS_LOGGER.info( + json.dumps(exec_metadata, default=str, indent=None) + ) + + return result + + return wrapper + + return decorator diff --git a/Python_Engine/Python/src/python_toolkit/bhom/bhom.mplstyle b/Python_Engine/Python/src/python_toolkit/bhom/bhom.mplstyle new file mode 100644 index 0000000..0c1d79c --- /dev/null +++ b/Python_Engine/Python/src/python_toolkit/bhom/bhom.mplstyle @@ -0,0 +1,81 @@ +# Default matplotlib settings for this toolkit. + +# Set custom colors. All colors are in web style hex format. +axes.prop_cycle: cycler('color', ['702F8A', 'E63187', '00A9E0', 'FFCF04', '6CC24E', 'EB671C', '00A499', 'D50032', '24135F', '6D104E', '006DA8', 'D06A13', '5D822D', 'F0AC1B', '1C3660', 'BC204B', '8F72B0', 'FCD16D', '8DB9CA', 'EE7837', 'AFC1A2', 'B72B77', 'A0D2C9', 'E6484D']) + +# Face settings +axes.facecolor: white +axes.edgecolor: black + +# Style spines +axes.linewidth: 0.8 +axes.spines.top: False +axes.spines.left: True +axes.spines.right: False +axes.spines.bottom: True + +# Set line styling for line plots +lines.linewidth: 1 +lines.solid_capstyle: round +lines.dash_capstyle: round + +# Grid style +axes.axisbelow: True +axes.grid: true +axes.grid.axis: both +grid.color: 958B82 +grid.linestyle: -- +grid.linewidth: 0.5 + +# Setting font sizes and spacing +axes.labelsize: medium +axes.labelweight: semibold +axes.ymargin: 0.1 +font.family: sans-serif +font.sans-serif: Segoe UI +font.size: 10 +xtick.labelsize: medium +xtick.major.pad: 3.5 +ytick.labelsize: medium +ytick.major.pad: 3.5 + +# date formatter +date.autoformatter.day: %b-%d +date.autoformatter.hour: %b-%d %H +date.autoformatter.microsecond: %M:%S.%f +date.autoformatter.minute: %d %H:%M +date.autoformatter.month: %b +date.autoformatter.second: %H:%M:%S +date.autoformatter.year: %Y + +# Title +axes.titlecolor: black +axes.titlelocation: left +axes.titlepad: 6 +axes.titlesize: large +axes.titleweight: bold + +# Remove major and minor ticks except for on the x-axis. +xtick.major.size: 3 +xtick.minor.size: 2 +ytick.major.size: 3 +ytick.minor.size: 2 + +# Set spacing for figure and also DPI. +figure.subplot.left: 0.08 +figure.subplot.right: 0.95 +figure.subplot.bottom: 0.07 +figure.figsize: 12, 5 +figure.dpi: 150 +figure.facecolor: white + +# Properties for saving the figure. Ensure a high DPI when saving so we have a good resolution. +savefig.dpi: 300 +savefig.facecolor: white +savefig.bbox: tight +savefig.pad_inches: 0.2 + +# Legend Styling +legend.framealpha: 0 +legend.frameon: False +legend.facecolor: inherit \ No newline at end of file diff --git a/Python_Engine/Python/src/python_toolkit/bhom/logging/__init__.py b/Python_Engine/Python/src/python_toolkit/bhom/logging/__init__.py new file mode 100644 index 0000000..4e9eae5 --- /dev/null +++ b/Python_Engine/Python/src/python_toolkit/bhom/logging/__init__.py @@ -0,0 +1,2 @@ +from .file import ANALYTICS_LOGGER +from .console import CONSOLE_LOGGER diff --git a/Python_Engine/Python/src/python_toolkit/bhom/logging/console.py b/Python_Engine/Python/src/python_toolkit/bhom/logging/console.py new file mode 100644 index 0000000..d36178b --- /dev/null +++ b/Python_Engine/Python/src/python_toolkit/bhom/logging/console.py @@ -0,0 +1,19 @@ +"""Logging utilities for BHoM analytics.""" + +# pylint: disable=E0401 +import logging +import sys + +# pylint: enable=E0401 + +from .. import TOOLKIT_NAME + +formatter = logging.Formatter("%(name)s - %(levelname)s - %(message)s") +handler = logging.StreamHandler(sys.stdout) +handler.setLevel(logging.DEBUG) +handler.setFormatter(formatter) + +CONSOLE_LOGGER = logging.getLogger(f"{TOOLKIT_NAME}[console]") +CONSOLE_LOGGER.propagate = False +CONSOLE_LOGGER.setLevel(logging.DEBUG) +CONSOLE_LOGGER.addHandler(handler) diff --git a/Python_Engine/Python/src/python_toolkit/bhom/logging/file.py b/Python_Engine/Python/src/python_toolkit/bhom/logging/file.py new file mode 100644 index 0000000..9c39acc --- /dev/null +++ b/Python_Engine/Python/src/python_toolkit/bhom/logging/file.py @@ -0,0 +1,28 @@ +"""Logging utilities for BHoM analytics.""" + +# pylint: disable=E0401 +import logging +from datetime import datetime +from logging.handlers import RotatingFileHandler + +# pylint: enable=E0401 + + +from .. import TOOLKIT_NAME, BHOM_LOG_FOLDER + +formatter = logging.Formatter("%(message)s") +handler = RotatingFileHandler( + BHOM_LOG_FOLDER / f"{TOOLKIT_NAME}_{datetime.now().strftime('%Y%m%d')}.log", + mode="a", + maxBytes=25 * 1024 * 1024, # 25mb max before file overwritten + backupCount=1, + encoding="utf-8", + delay=True, # wait until all logs collected before writing +) +handler.setLevel(logging.DEBUG) +handler.setFormatter(formatter) + +ANALYTICS_LOGGER = logging.getLogger(f"{TOOLKIT_NAME}") +ANALYTICS_LOGGER.propagate = False +ANALYTICS_LOGGER.setLevel(logging.DEBUG) +ANALYTICS_LOGGER.addHandler(handler) diff --git a/Python_Engine/Python/src/python_toolkit/bhom/util.py b/Python_Engine/Python/src/python_toolkit/bhom/util.py new file mode 100644 index 0000000..3200384 --- /dev/null +++ b/Python_Engine/Python/src/python_toolkit/bhom/util.py @@ -0,0 +1,24 @@ +"""General utility functions.""" +# pylint: disable=E0401 +from datetime import datetime + +# pylint: enable=E0401 + + +def csharp_ticks(date_time: datetime = datetime.utcnow(), short: bool = False) -> int: + """Python implementation of C# DateTime.UtcNow.Ticks. + + Args: + date_time (datetime, optional): The datetime to convert to ticks. Defaults to datetime.utcnow(). + short (bool, optional): Whether to return the short ticks. Defaults to False. + + Returns: + int: The ticks. + """ + + _ticks = (date_time - datetime(1, 1, 1)).total_seconds() + + if short: + return int(_ticks) + + return int(_ticks * (10**7)) diff --git a/Python_Engine/Python/src/python_toolkit/bhomutil/__init__.py b/Python_Engine/Python/src/python_toolkit/bhomutil/__init__.py deleted file mode 100644 index 5f28270..0000000 --- a/Python_Engine/Python/src/python_toolkit/bhomutil/__init__.py +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file From 09247c431d684768c70853c6ba7c053419cc372e Mon Sep 17 00:00:00 2001 From: Thomas Edward Kingstone Date: Fri, 13 Sep 2024 14:57:27 +0100 Subject: [PATCH 07/24] changed log entry type to be the same as the one in BHoM --- Python_Engine/Python/src/python_toolkit/bhom/analytics.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Python_Engine/Python/src/python_toolkit/bhom/analytics.py b/Python_Engine/Python/src/python_toolkit/bhom/analytics.py index 318377b..392aaa0 100644 --- a/Python_Engine/Python/src/python_toolkit/bhom/analytics.py +++ b/Python_Engine/Python/src/python_toolkit/bhom/analytics.py @@ -66,16 +66,14 @@ def wrapper(*args, **kwargs) -> Any: "SelectedItem": { "MethodName": function.__name__, "Parameters": _args, - "TypeName": f"{function.__module__}.{function.__qualname__}", - "_bhomVersion": BHOM_VERSION, - "_t": "Python", + "TypeName": f"{function.__module__}.{function.__qualname__}" }, "Time": { "$date": csharp_ticks(short=True), }, "UI": "Python", "UiVersion": TOOLKIT_NAME, - "_t": "BH.oM.UI.UsageLogEntry", + "_t": "BH.oM.Base.UsageLogEntry", } try: From 604532a09b4356813848b643415db8cb3c697c87 Mon Sep 17 00:00:00 2001 From: Thomas Edward Kingstone Date: Tue, 1 Oct 2024 10:08:38 +0100 Subject: [PATCH 08/24] add method separator for Download.cs --- Python_Engine/Compute/Download.cs | 25 +------------------------ 1 file changed, 1 insertion(+), 24 deletions(-) diff --git a/Python_Engine/Compute/Download.cs b/Python_Engine/Compute/Download.cs index a7e4096..06836a9 100644 --- a/Python_Engine/Compute/Download.cs +++ b/Python_Engine/Compute/Download.cs @@ -86,31 +86,8 @@ public static string DownloadFile( return filePath; } - /* - // TODO - THIS METHOD HAS CHANGED BUT IS STILL USED, SO NEEDS DEPRECATING - // changed from what to what ? - [Description("Download the installer for the target version of python.")] - [Input("version", "A Python version.")] - [Input("name", "Name of the toolkit for this installation of python.")] - [Output("executablePath", "The path of the executable for the downloaded Python installer.")] - public static string DownloadPython(this PythonVersion version, string name = null) - { - string url = version.EmbeddableURL(); - if (string.IsNullOrEmpty(name)) - name = Path.GetFileNameWithoutExtension(url); - - string targetExecutable = Path.Combine(Query.DirectoryBaseEnvironment(), version.ToString(), "installer.exe"); - - if (File.Exists(targetExecutable)) - return targetExecutable; - - if (!Directory.Exists(Path.Combine(Query.DirectoryEnvironments(), name))) - Directory.CreateDirectory(Path.Combine(Query.DirectoryEnvironments(), name)); - - string exefile = DownloadFile(url, Path.Combine(Query.DirectoryEnvironments(), name)); - return exefile; - }*/ + /******************************************************/ [PreviousVersion("7.3", "BH.Engine.Python.Compute.DownloadPython(BH.oM.Python.PythonVersion, System.String)")] [Description("Download and install a specified version of python, and return the executable for it.")] From a674209aa04246d9d6018e294a5ebc9929c53246 Mon Sep 17 00:00:00 2001 From: Thomas Edward Kingstone Date: Tue, 1 Oct 2024 12:01:01 +0100 Subject: [PATCH 09/24] update versioning for removed/updated methods. --- Python_Engine/Compute/Download.cs | 2 +- Python_Engine/Compute/Remove.cs | 2 +- Python_Engine/Query/Directory.cs | 1 + Python_Engine/Versioning80.json | 5 +++++ 4 files changed, 8 insertions(+), 2 deletions(-) create mode 100644 Python_Engine/Versioning80.json diff --git a/Python_Engine/Compute/Download.cs b/Python_Engine/Compute/Download.cs index 06836a9..ca6b908 100644 --- a/Python_Engine/Compute/Download.cs +++ b/Python_Engine/Compute/Download.cs @@ -89,7 +89,7 @@ public static string DownloadFile( /******************************************************/ - [PreviousVersion("7.3", "BH.Engine.Python.Compute.DownloadPython(BH.oM.Python.PythonVersion, System.String)")] + [PreviousVersion("8.0", "BH.Engine.Python.Compute.DownloadPython(BH.oM.Python.PythonVersion, System.String)")] [Description("Download and install a specified version of python, and return the executable for it.")] [Input("version", "The version of python to download.")] [Output("pythonExecutable", "The executable (python.exe) for the python version that was installed")] diff --git a/Python_Engine/Compute/Remove.cs b/Python_Engine/Compute/Remove.cs index ee1b4d6..41afc05 100644 --- a/Python_Engine/Compute/Remove.cs +++ b/Python_Engine/Compute/Remove.cs @@ -81,7 +81,7 @@ public static void RemoveAllVirtualEnvironments() } } - [PreviousVersion("7.3", "BH.Engine.Python.Compute.RemoveBaseEnvironment()")] + [PreviousVersion("8.0", "BH.Engine.Python.Compute.RemoveBaseEnvironment()")] [Input("version", "The base python version to remove.")] public static void RemoveBaseVersion(PythonVersion version = PythonVersion.v3_10) { diff --git a/Python_Engine/Query/Directory.cs b/Python_Engine/Query/Directory.cs index 34f16ed..bf88b18 100644 --- a/Python_Engine/Query/Directory.cs +++ b/Python_Engine/Query/Directory.cs @@ -67,6 +67,7 @@ public static string DirectoryEnvironments() return dir; } + [PreviousVersion("8.0", "BH.Engine.Python.Query.DirectoryBaseEnvironment()")] [Description("The location where the base Python environment exists.")] [Output("The location where the base Python environment exists.")] public static string DirectoryBaseEnvironment(PythonVersion version = PythonVersion.v3_10) diff --git a/Python_Engine/Versioning80.json b/Python_Engine/Versioning80.json new file mode 100644 index 0000000..5b66a73 --- /dev/null +++ b/Python_Engine/Versioning80.json @@ -0,0 +1,5 @@ +{ + "MessageForDeleted": { + "BH.Engine.Python.Compute.DownloadGetPip(System.String)": "The DownloadGetPip method was removed due to an update to how base python environments are installed, and the functionality within this method is now covered by BasePythonEnvironment." + } +} \ No newline at end of file From 01eb8945ce1a77668dd179c3cf3d8d1984c91664 Mon Sep 17 00:00:00 2001 From: Thomas Edward Kingstone Date: Tue, 1 Oct 2024 12:23:26 +0100 Subject: [PATCH 10/24] fix versioning keys --- Python_Engine/Compute/Download.cs | 2 +- Python_Engine/Versioning80.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Python_Engine/Compute/Download.cs b/Python_Engine/Compute/Download.cs index ca6b908..ac653d7 100644 --- a/Python_Engine/Compute/Download.cs +++ b/Python_Engine/Compute/Download.cs @@ -89,7 +89,7 @@ public static string DownloadFile( /******************************************************/ - [PreviousVersion("8.0", "BH.Engine.Python.Compute.DownloadPython(BH.oM.Python.PythonVersion, System.String)")] + [PreviousVersion("8.0", "BH.Engine.Python.Enums.Compute.DownloadPython(BH.oM.Python.PythonVersion, System.String)")] [Description("Download and install a specified version of python, and return the executable for it.")] [Input("version", "The version of python to download.")] [Output("pythonExecutable", "The executable (python.exe) for the python version that was installed")] diff --git a/Python_Engine/Versioning80.json b/Python_Engine/Versioning80.json index 5b66a73..a811dbc 100644 --- a/Python_Engine/Versioning80.json +++ b/Python_Engine/Versioning80.json @@ -1,5 +1,5 @@ { "MessageForDeleted": { - "BH.Engine.Python.Compute.DownloadGetPip(System.String)": "The DownloadGetPip method was removed due to an update to how base python environments are installed, and the functionality within this method is now covered by BasePythonEnvironment." + "BH.Engine.Python.Compute.DownloadGetPip": "The DownloadGetPip method was removed due to an update to how base python environments are installed, and the functionality within this method is now covered by BasePythonEnvironment." } } \ No newline at end of file From 4413c312fdd3f2720473355e692110e75126bbb4 Mon Sep 17 00:00:00 2001 From: Thomas Edward Kingstone Date: Tue, 1 Oct 2024 12:38:50 +0100 Subject: [PATCH 11/24] rename versioning file --- Python_Engine/{Versioning80.json => Versioning_80.json} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Python_Engine/{Versioning80.json => Versioning_80.json} (100%) diff --git a/Python_Engine/Versioning80.json b/Python_Engine/Versioning_80.json similarity index 100% rename from Python_Engine/Versioning80.json rename to Python_Engine/Versioning_80.json From 162cf4202a6c1062c0c9864cc1d18782603cadb5 Mon Sep 17 00:00:00 2001 From: Thomas Edward Kingstone Date: Tue, 1 Oct 2024 12:39:50 +0100 Subject: [PATCH 12/24] updated wrong versioning key :P --- Python_Engine/Compute/Download.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python_Engine/Compute/Download.cs b/Python_Engine/Compute/Download.cs index ac653d7..ae92006 100644 --- a/Python_Engine/Compute/Download.cs +++ b/Python_Engine/Compute/Download.cs @@ -89,7 +89,7 @@ public static string DownloadFile( /******************************************************/ - [PreviousVersion("8.0", "BH.Engine.Python.Enums.Compute.DownloadPython(BH.oM.Python.PythonVersion, System.String)")] + [PreviousVersion("8.0", "BH.Engine.Python.Compute.DownloadPython(BH.oM.Python.Enums.PythonVersion, System.String)")] [Description("Download and install a specified version of python, and return the executable for it.")] [Input("version", "The version of python to download.")] [Output("pythonExecutable", "The executable (python.exe) for the python version that was installed")] From 895c4e87d74952010018cf896fc4dfbaf6329d2d Mon Sep 17 00:00:00 2001 From: Thomas Edward Kingstone Date: Tue, 1 Oct 2024 12:54:39 +0100 Subject: [PATCH 13/24] fix key for DownloadGetPip --- Python_Engine/Versioning_80.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python_Engine/Versioning_80.json b/Python_Engine/Versioning_80.json index a811dbc..5b66a73 100644 --- a/Python_Engine/Versioning_80.json +++ b/Python_Engine/Versioning_80.json @@ -1,5 +1,5 @@ { "MessageForDeleted": { - "BH.Engine.Python.Compute.DownloadGetPip": "The DownloadGetPip method was removed due to an update to how base python environments are installed, and the functionality within this method is now covered by BasePythonEnvironment." + "BH.Engine.Python.Compute.DownloadGetPip(System.String)": "The DownloadGetPip method was removed due to an update to how base python environments are installed, and the functionality within this method is now covered by BasePythonEnvironment." } } \ No newline at end of file From 9fc085a9584c2c90b9ab408148be24c82ab7acfb Mon Sep 17 00:00:00 2001 From: Thomas Edward Kingstone Date: Tue, 1 Oct 2024 13:56:39 +0100 Subject: [PATCH 14/24] updated DirectoryBaseEnvironment and BasePythonEnvironment --- Python_Engine/Compute/BasePythonEnvironment.cs | 8 +++++--- Python_Engine/Compute/VirtualEnvironment.cs | 4 ++-- Python_Engine/Query/Directory.cs | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Python_Engine/Compute/BasePythonEnvironment.cs b/Python_Engine/Compute/BasePythonEnvironment.cs index e7c0f13..4f2fbca 100644 --- a/Python_Engine/Compute/BasePythonEnvironment.cs +++ b/Python_Engine/Compute/BasePythonEnvironment.cs @@ -34,10 +34,12 @@ namespace BH.Engine.Python public static partial class Compute { [Description("Retrieve or reinstall the base Python Environment for BHoM workflows.")] + [Input("version", "The target version of python to be installed or retrieved.")] [Input("reload", "Reload the base Python environment rather than recreating it, if it already exists.")] [Input("run", "Start the installation/retrieval of the BHoM Base Python Environment.")] [Output("env", "The base Python Environment for all BHoM workflows.")] public static PythonEnvironment BasePythonEnvironment( + PythonVersion version = PythonVersion.v3_10, bool reload = true, bool run = false ) @@ -54,7 +56,7 @@ public static PythonEnvironment BasePythonEnvironment( } // determine whether the base environment already exists - string targetExecutable = Path.Combine(Query.DirectoryBaseEnvironment(), "python.exe"); + string targetExecutable = Path.Combine(Query.DirectoryBaseEnvironment(version), "python.exe"); bool exists = File.Exists(targetExecutable); if (exists && reload) @@ -64,8 +66,8 @@ public static PythonEnvironment BasePythonEnvironment( // remove all existing environments and kernels RemoveEverything(); - // download the installer for the target Python version - string exe = PythonVersion.v3_10.DownloadPythonVersion(); + // download and run the installer for the target Python version + string exe = version.DownloadPythonVersion(); // install essential packages into base environment InstallPackages(exe, new List() { "virtualenv", "jupyterlab", "black", "pylint" }); diff --git a/Python_Engine/Compute/VirtualEnvironment.cs b/Python_Engine/Compute/VirtualEnvironment.cs index c77a4fc..6e2453e 100644 --- a/Python_Engine/Compute/VirtualEnvironment.cs +++ b/Python_Engine/Compute/VirtualEnvironment.cs @@ -53,10 +53,10 @@ public static PythonEnvironment VirtualEnvironment(this PythonVersion version, s } // check that base environment is installed and return null and raise error if it isn't - string baseEnvironmentExecutable = Path.Combine(Query.DirectoryBaseEnvironment(), "python.exe"); + string baseEnvironmentExecutable = Path.Combine(Query.DirectoryBaseEnvironment(version), "python.exe"); if (!File.Exists(baseEnvironmentExecutable)) { - BH.Engine.Base.Compute.RecordWarning("The base Python environment doesnt seem to be installed. Install it first in order to run this method."); + BH.Engine.Base.Compute.RecordWarning("The base Python environment for this version doesn't seem to be installed. Install it first in order to run this method."); return null; } diff --git a/Python_Engine/Query/Directory.cs b/Python_Engine/Query/Directory.cs index bf88b18..68dd52b 100644 --- a/Python_Engine/Query/Directory.cs +++ b/Python_Engine/Query/Directory.cs @@ -70,7 +70,7 @@ public static string DirectoryEnvironments() [PreviousVersion("8.0", "BH.Engine.Python.Query.DirectoryBaseEnvironment()")] [Description("The location where the base Python environment exists.")] [Output("The location where the base Python environment exists.")] - public static string DirectoryBaseEnvironment(PythonVersion version = PythonVersion.v3_10) + public static string DirectoryBaseEnvironment(this PythonVersion version) { return Path.Combine(Query.DirectoryEnvironments(), Query.ToolkitName(), version.ToString()); } From fc0513db002553fd1778c07e0525e30e405bf514 Mon Sep 17 00:00:00 2001 From: Thomas Edward Kingstone Date: Tue, 1 Oct 2024 14:56:08 +0100 Subject: [PATCH 15/24] add local package to base environment on install --- Python_Engine/Compute/BasePythonEnvironment.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Python_Engine/Compute/BasePythonEnvironment.cs b/Python_Engine/Compute/BasePythonEnvironment.cs index 4f2fbca..936559f 100644 --- a/Python_Engine/Compute/BasePythonEnvironment.cs +++ b/Python_Engine/Compute/BasePythonEnvironment.cs @@ -72,6 +72,8 @@ public static PythonEnvironment BasePythonEnvironment( // install essential packages into base environment InstallPackages(exe, new List() { "virtualenv", "jupyterlab", "black", "pylint" }); + InstallPackageLocal(Query.DirectoryCode(), Query.ToolkitName()); + return new PythonEnvironment() { Name = Query.ToolkitName(), Executable = exe }; } } From db51afc32b64e67d68cb0d46bbbe86a110bba32c Mon Sep 17 00:00:00 2001 From: Thomas Edward Kingstone Date: Tue, 1 Oct 2024 15:05:35 +0100 Subject: [PATCH 16/24] mistake in method signature --- Python_Engine/Compute/BasePythonEnvironment.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python_Engine/Compute/BasePythonEnvironment.cs b/Python_Engine/Compute/BasePythonEnvironment.cs index 936559f..4c32e9d 100644 --- a/Python_Engine/Compute/BasePythonEnvironment.cs +++ b/Python_Engine/Compute/BasePythonEnvironment.cs @@ -72,7 +72,7 @@ public static PythonEnvironment BasePythonEnvironment( // install essential packages into base environment InstallPackages(exe, new List() { "virtualenv", "jupyterlab", "black", "pylint" }); - InstallPackageLocal(Query.DirectoryCode(), Query.ToolkitName()); + InstallPackageLocal(exe, Path.Combine(Query.DirectoryCode(), Query.ToolkitName())); return new PythonEnvironment() { Name = Query.ToolkitName(), Executable = exe }; } From 71a1a7c22e3c4476fb53393cc30cfc3d58189a43 Mon Sep 17 00:00:00 2001 From: Thomas Edward Kingstone Date: Tue, 1 Oct 2024 15:15:33 +0100 Subject: [PATCH 17/24] general compliance changes --- Python_Engine/Compute/Remove.cs | 1 + Python_Engine/Compute/UnzipFile.cs | 1 + Python_Engine/Convert/ToPython.cs | 41 ------------------------------ Python_Engine/Python_Engine.csproj | 4 +++ Python_Engine/Query/Directory.cs | 11 ++++---- Python_Engine/Query/ToolkitName.cs | 2 +- 6 files changed, 13 insertions(+), 47 deletions(-) delete mode 100644 Python_Engine/Convert/ToPython.cs diff --git a/Python_Engine/Compute/Remove.cs b/Python_Engine/Compute/Remove.cs index 41afc05..8ccf112 100644 --- a/Python_Engine/Compute/Remove.cs +++ b/Python_Engine/Compute/Remove.cs @@ -82,6 +82,7 @@ public static void RemoveAllVirtualEnvironments() } [PreviousVersion("8.0", "BH.Engine.Python.Compute.RemoveBaseEnvironment()")] + [Description("Remove the base install for the python version specified.")] [Input("version", "The base python version to remove.")] public static void RemoveBaseVersion(PythonVersion version = PythonVersion.v3_10) { diff --git a/Python_Engine/Compute/UnzipFile.cs b/Python_Engine/Compute/UnzipFile.cs index f34d802..bb038db 100644 --- a/Python_Engine/Compute/UnzipFile.cs +++ b/Python_Engine/Compute/UnzipFile.cs @@ -29,6 +29,7 @@ namespace BH.Engine.Python { public static partial class Compute { + //This method is no longer used by python toolkit, and perhaps should be removed or moved to the file toolkit instead. [Description("Extract the contents of an archive.")] [Input("archivePath", "The archive to extract.")] [Input("targetDirectory", "The destination directory to extract into.")] diff --git a/Python_Engine/Convert/ToPython.cs b/Python_Engine/Convert/ToPython.cs deleted file mode 100644 index e7ab80f..0000000 --- a/Python_Engine/Convert/ToPython.cs +++ /dev/null @@ -1,41 +0,0 @@ -/* - * This file is part of the Buildings and Habitats object Model (BHoM) - * Copyright (c) 2015 - 2024, the respective contributors. All rights reserved. - * - * Each contributor holds copyright over their respective contributions. - * The project versioning (Git) records all such contribution source information. - * - * - * The BHoM is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3.0 of the License, or - * (at your option) any later version. - * - * The BHoM is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with this code. If not, see . - */ - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using BH.oM.Base.Attributes; - -namespace BH.Engine.Python -{ - public static partial class Convert - { - // TODO - REMOVE THIS METHOD, NO LONGER REQUIRED, PLUS IT HAS AN OLD ToBeRemoved TAG - [ToBeRemoved("5.3", "This method included in order to fool versioning into playing nicely.")] - public static bool ToPython(this T[,] input) - { - return false; - } - } -} diff --git a/Python_Engine/Python_Engine.csproj b/Python_Engine/Python_Engine.csproj index 55dd242..ce916ff 100644 --- a/Python_Engine/Python_Engine.csproj +++ b/Python_Engine/Python_Engine.csproj @@ -39,6 +39,10 @@ + + + + diff --git a/Python_Engine/Query/Directory.cs b/Python_Engine/Query/Directory.cs index 68dd52b..d38e769 100644 --- a/Python_Engine/Query/Directory.cs +++ b/Python_Engine/Query/Directory.cs @@ -30,14 +30,14 @@ namespace BH.Engine.Python public static partial class Query { [Description("The location where BHoM extensions reside.")] - [Output("The location where BHoM extensions reside.")] + [Output("path", "The location where BHoM extensions reside.")] public static string DirectoryExtensions() { return Path.Combine(System.IO.Directory.GetParent(BH.Engine.Base.Query.BHoMFolder()).FullName, "Extensions"); } [Description("The location where any BHoM-related Python kernels reside.")] - [Output("The location where any BHoM-related Python kernels reside.")] + [Output("path", "The location where any BHoM-related Python kernels reside.")] public static string DirectoryKernels() { @@ -48,7 +48,7 @@ public static string DirectoryKernels() } [Description("The location where any BHoM-related Python code resides.")] - [Output("The location where any BHoM-related Python code resides.")] + [Output("path", "The location where any BHoM-related Python code resides.")] public static string DirectoryCode() { string dir = Path.Combine(Query.DirectoryExtensions(), "PythonCode"); @@ -58,7 +58,7 @@ public static string DirectoryCode() } [Description("The location where any BHoM-related Python environment (or virtualenv) resides.")] - [Output("The location where any BHoM-related Python environment (or virtualenv) resides.")] + [Output("path", "The location where any BHoM-related Python environment (or virtualenv) resides.")] public static string DirectoryEnvironments() { string dir = Path.Combine(Query.DirectoryExtensions(), "PythonEnvironments"); @@ -69,7 +69,8 @@ public static string DirectoryEnvironments() [PreviousVersion("8.0", "BH.Engine.Python.Query.DirectoryBaseEnvironment()")] [Description("The location where the base Python environment exists.")] - [Output("The location where the base Python environment exists.")] + [Input("version", "The python version to get the base environment for.")] + [Output("path", "The location where the base Python environment exists.")] public static string DirectoryBaseEnvironment(this PythonVersion version) { return Path.Combine(Query.DirectoryEnvironments(), Query.ToolkitName(), version.ToString()); diff --git a/Python_Engine/Query/ToolkitName.cs b/Python_Engine/Query/ToolkitName.cs index db48ddc..63daee9 100644 --- a/Python_Engine/Query/ToolkitName.cs +++ b/Python_Engine/Query/ToolkitName.cs @@ -29,7 +29,7 @@ namespace BH.Engine.Python public static partial class Query { [Description("The name of this toolkit.")] - [Output("The name of this toolkit.")] + [Output("name", "The name of this toolkit.")] public static string ToolkitName() { return "Python_Toolkit"; From f1b93e8f4a594734b69b956251155f960f57ea99 Mon Sep 17 00:00:00 2001 From: Thomas Edward Kingstone Date: Wed, 2 Oct 2024 12:12:16 +0100 Subject: [PATCH 18/24] revert ToPython removal (versioning doesn't play nice) --- .../Compute/BasePythonEnvironment.cs | 1 + Python_Engine/Convert/ToPython.cs | 41 +++++++++++++++++++ Python_Engine/Python_Engine.csproj | 4 -- 3 files changed, 42 insertions(+), 4 deletions(-) create mode 100644 Python_Engine/Convert/ToPython.cs diff --git a/Python_Engine/Compute/BasePythonEnvironment.cs b/Python_Engine/Compute/BasePythonEnvironment.cs index 4c32e9d..9ab74f1 100644 --- a/Python_Engine/Compute/BasePythonEnvironment.cs +++ b/Python_Engine/Compute/BasePythonEnvironment.cs @@ -33,6 +33,7 @@ namespace BH.Engine.Python { public static partial class Compute { + [PreviousVersion("8.0", "BH.Engine.Python.Compute.BasePythonEnvironment(System.String, System.String)")] [Description("Retrieve or reinstall the base Python Environment for BHoM workflows.")] [Input("version", "The target version of python to be installed or retrieved.")] [Input("reload", "Reload the base Python environment rather than recreating it, if it already exists.")] diff --git a/Python_Engine/Convert/ToPython.cs b/Python_Engine/Convert/ToPython.cs new file mode 100644 index 0000000..c271f19 --- /dev/null +++ b/Python_Engine/Convert/ToPython.cs @@ -0,0 +1,41 @@ +/* + * This file is part of the Buildings and Habitats object Model (BHoM) + * Copyright (c) 2015 - 2024, the respective contributors. All rights reserved. + * + * Each contributor holds copyright over their respective contributions. + * The project versioning (Git) records all such contribution source information. + * + * + * The BHoM is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3.0 of the License, or + * (at your option) any later version. + * + * The BHoM is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this code. If not, see . + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using BH.oM.Base.Attributes; + +namespace BH.Engine.Python +{ + public static partial class Convert + { + // TODO - REMOVE THIS METHOD, NO LONGER REQUIRED, PLUS IT HAS AN OLD ToBeRemoved TAG + [ToBeRemoved("5.3", "This method included in order to fool versioning into playing nicely.")] + public static bool ToPython(this T[,] input) + { + return false; + } + } +} \ No newline at end of file diff --git a/Python_Engine/Python_Engine.csproj b/Python_Engine/Python_Engine.csproj index ce916ff..55dd242 100644 --- a/Python_Engine/Python_Engine.csproj +++ b/Python_Engine/Python_Engine.csproj @@ -39,10 +39,6 @@ - - - - From 642f2f5b723b756e62b0a4ef4e2ff7386e038d0d Mon Sep 17 00:00:00 2001 From: Thomas Edward Kingstone Date: Wed, 2 Oct 2024 13:48:06 +0100 Subject: [PATCH 19/24] Update Python_Engine/Compute/BasePythonEnvironment.cs --- Python_Engine/Compute/BasePythonEnvironment.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python_Engine/Compute/BasePythonEnvironment.cs b/Python_Engine/Compute/BasePythonEnvironment.cs index 9ab74f1..9bcd311 100644 --- a/Python_Engine/Compute/BasePythonEnvironment.cs +++ b/Python_Engine/Compute/BasePythonEnvironment.cs @@ -33,7 +33,7 @@ namespace BH.Engine.Python { public static partial class Compute { - [PreviousVersion("8.0", "BH.Engine.Python.Compute.BasePythonEnvironment(System.String, System.String)")] + [PreviousVersion("8.0", "BH.Engine.Python.Compute.BasePythonEnvironment(System.Boolean, System.Boolean)")] [Description("Retrieve or reinstall the base Python Environment for BHoM workflows.")] [Input("version", "The target version of python to be installed or retrieved.")] [Input("reload", "Reload the base Python environment rather than recreating it, if it already exists.")] From 660a338ca46a77c6dd9d0ae339d6bbb1f24135d3 Mon Sep 17 00:00:00 2001 From: Thomas Edward Kingstone Date: Tue, 22 Oct 2024 12:03:08 +0100 Subject: [PATCH 20/24] Made the warning for creating virtual environments clearer (need the base version installed when requesting that version) --- Python_Engine/Compute/VirtualEnvironment.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python_Engine/Compute/VirtualEnvironment.cs b/Python_Engine/Compute/VirtualEnvironment.cs index 6e2453e..9f6c840 100644 --- a/Python_Engine/Compute/VirtualEnvironment.cs +++ b/Python_Engine/Compute/VirtualEnvironment.cs @@ -56,7 +56,7 @@ public static PythonEnvironment VirtualEnvironment(this PythonVersion version, s string baseEnvironmentExecutable = Path.Combine(Query.DirectoryBaseEnvironment(version), "python.exe"); if (!File.Exists(baseEnvironmentExecutable)) { - BH.Engine.Base.Compute.RecordWarning("The base Python environment for this version doesn't seem to be installed. Install it first in order to run this method."); + BH.Engine.Base.Compute.RecordWarning($"The base Python environment for the desired version ({version}) doesn't seem to be installed. Install it first using BasePythonEnvironment in order to run this method."); return null; } From 631ad9ee3d67a681974319c149d535ea1690d032 Mon Sep 17 00:00:00 2001 From: Thomas Edward Kingstone Date: Tue, 29 Oct 2024 10:01:57 +0000 Subject: [PATCH 21/24] Removed error if the environment didn't exist - now downloads the required base version if it doesn't exist and leaves a note to say this. --- Python_Engine/Compute/VirtualEnvironment.cs | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/Python_Engine/Compute/VirtualEnvironment.cs b/Python_Engine/Compute/VirtualEnvironment.cs index 9f6c840..12ee877 100644 --- a/Python_Engine/Compute/VirtualEnvironment.cs +++ b/Python_Engine/Compute/VirtualEnvironment.cs @@ -46,17 +46,10 @@ public static PythonEnvironment VirtualEnvironment(this PythonVersion version, s BH.Engine.Base.Compute.RecordError("A BHoM Python virtual environment cannot cannot contain invalid filepath characters."); return null; } - if (version == PythonVersion.Undefined) - { - BH.Engine.Base.Compute.RecordError("Please provide a version of Python."); - return null; - } - // check that base environment is installed and return null and raise error if it isn't - string baseEnvironmentExecutable = Path.Combine(Query.DirectoryBaseEnvironment(version), "python.exe"); - if (!File.Exists(baseEnvironmentExecutable)) + if (version == PythonVersion.Undefined) { - BH.Engine.Base.Compute.RecordWarning($"The base Python environment for the desired version ({version}) doesn't seem to be installed. Install it first using BasePythonEnvironment in order to run this method."); + BH.Engine.Base.Compute.RecordError("Please provide a valid Python version."); return null; } @@ -77,15 +70,20 @@ public static PythonEnvironment VirtualEnvironment(this PythonVersion version, s } if (!File.Exists(versionExecutable)) + { // The output here should be the same, but to be sure replace the value. + BH.Engine.Base.Compute.RecordNote($"The base environment for the requested version {version} has been installed as it was not present."); versionExecutable = version.DownloadPythonVersion(); + if (versionExecutable == null) + return null; + } // create the venv from the base environment Process process = new Process() { StartInfo = new ProcessStartInfo() { - FileName = Modify.AddQuotesIfRequired(baseEnvironmentExecutable), + FileName = Modify.AddQuotesIfRequired(versionExecutable), Arguments = $"-m virtualenv --python={Modify.AddQuotesIfRequired(versionExecutable)} {Modify.AddQuotesIfRequired(targetDirectory)}", RedirectStandardError = true, UseShellExecute = false, From 6561cb0ae9a4ceb0e6632f84c92a8d057d283a0d Mon Sep 17 00:00:00 2001 From: Thomas Edward Kingstone Date: Tue, 29 Oct 2024 11:07:35 +0000 Subject: [PATCH 22/24] add a comment and if spacing --- Python_Engine/Compute/VirtualEnvironment.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Python_Engine/Compute/VirtualEnvironment.cs b/Python_Engine/Compute/VirtualEnvironment.cs index 12ee877..7f32610 100644 --- a/Python_Engine/Compute/VirtualEnvironment.cs +++ b/Python_Engine/Compute/VirtualEnvironment.cs @@ -74,7 +74,8 @@ public static PythonEnvironment VirtualEnvironment(this PythonVersion version, s // The output here should be the same, but to be sure replace the value. BH.Engine.Base.Compute.RecordNote($"The base environment for the requested version {version} has been installed as it was not present."); versionExecutable = version.DownloadPythonVersion(); - if (versionExecutable == null) + + if (versionExecutable == null) // If the executable is null, then python wasn't installed correctly, so return null - the error message for downloading the version should suffice. return null; } From 3ba64bbc7b649108316470d682f7511ec7386bff Mon Sep 17 00:00:00 2001 From: Thomas Edward Kingstone Date: Tue, 29 Oct 2024 11:56:53 +0000 Subject: [PATCH 23/24] properly install the base environment when asked for --- Python_Engine/Compute/VirtualEnvironment.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python_Engine/Compute/VirtualEnvironment.cs b/Python_Engine/Compute/VirtualEnvironment.cs index 7f32610..230ecb2 100644 --- a/Python_Engine/Compute/VirtualEnvironment.cs +++ b/Python_Engine/Compute/VirtualEnvironment.cs @@ -73,7 +73,7 @@ public static PythonEnvironment VirtualEnvironment(this PythonVersion version, s { // The output here should be the same, but to be sure replace the value. BH.Engine.Base.Compute.RecordNote($"The base environment for the requested version {version} has been installed as it was not present."); - versionExecutable = version.DownloadPythonVersion(); + versionExecutable = BasePythonEnvironment(version, run: true)?.Executable; if (versionExecutable == null) // If the executable is null, then python wasn't installed correctly, so return null - the error message for downloading the version should suffice. return null; From 37eb521bc251ee17474104f8cc65ba5f8a24a479 Mon Sep 17 00:00:00 2001 From: Thomas Edward Kingstone Date: Wed, 30 Oct 2024 13:44:23 +0000 Subject: [PATCH 24/24] change to run installer check again --- Python_Engine/Compute/BasePythonEnvironment.cs | 1 - Python_Engine/Compute/VirtualEnvironment.cs | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Python_Engine/Compute/BasePythonEnvironment.cs b/Python_Engine/Compute/BasePythonEnvironment.cs index 9bcd311..2735a7b 100644 --- a/Python_Engine/Compute/BasePythonEnvironment.cs +++ b/Python_Engine/Compute/BasePythonEnvironment.cs @@ -72,7 +72,6 @@ public static PythonEnvironment BasePythonEnvironment( // install essential packages into base environment InstallPackages(exe, new List() { "virtualenv", "jupyterlab", "black", "pylint" }); - InstallPackageLocal(exe, Path.Combine(Query.DirectoryCode(), Query.ToolkitName())); return new PythonEnvironment() { Name = Query.ToolkitName(), Executable = exe }; diff --git a/Python_Engine/Compute/VirtualEnvironment.cs b/Python_Engine/Compute/VirtualEnvironment.cs index 230ecb2..8927220 100644 --- a/Python_Engine/Compute/VirtualEnvironment.cs +++ b/Python_Engine/Compute/VirtualEnvironment.cs @@ -113,6 +113,7 @@ public static PythonEnvironment VirtualEnvironment(this PythonVersion version, s UseShellExecute = false, } }; + using (Process p = Process.Start(process2.StartInfo)) { string standardError = p.StandardError.ReadToEnd(); @@ -120,7 +121,6 @@ public static PythonEnvironment VirtualEnvironment(this PythonVersion version, s if (p.ExitCode != 0) BH.Engine.Base.Compute.RecordError($"Error registering the \"{name}\" virtual environment.\n{standardError}"); } - // replace text in a file return new PythonEnvironment() { Executable = targetExecutable, Name = name }; }