Update python packaging to support building universal wheel for python (

#815) * update * update * update * update & rebase
microsoft · Oct 14, 2020 · 1a35fbc · 1a35fbc
1 parent 1df3370
commit 1a35fbc
Show file tree

Hide file tree

Showing 6 changed files with 138 additions and 6 deletions.
diff --git a/src/BuildScriptGenerator/Python/PythonBashBuildSnippet.sh.tpl b/src/BuildScriptGenerator/Python/PythonBashBuildSnippet.sh.tpl
@@ -103,12 +103,24 @@ fi
 	echo $APP_PACKAGES_PATH > $SITE_PACKAGES_PATH"/oryx.pth"
 {{ end }}
 
+
 {{ if RunPythonPackageCommand }}
 	echo
 	echo "Running python packaging commands ...."
 	echo
-	echo "Creating python package wheel ...."
-	$python setup.py sdist --formats=gztar,zip,tar bdist_wheel --universal
+	echo "Determining python package wheel ...."
+
+	{{ if PythonPackageWheelProperty }}
+		echo "Creating universal package wheel ...."
+	{{ end }}
+
+	if [ -z "{{ PythonPackageWheelProperty }}" ]
+	then 
+		echo "Creating non universal package wheel ...."
+		$python setup.py sdist --formats=gztar,zip,tar bdist_wheel
+	else		
+		$python setup.py sdist --formats=gztar,zip,tar bdist_wheel --universal
+	fi
 	echo "Now creating python package egg ...."
 	$python setup.py bdist_egg
 	echo

diff --git a/src/BuildScriptGenerator/Python/PythonBashBuildSnippetProperties.cs b/src/BuildScriptGenerator/Python/PythonBashBuildSnippetProperties.cs
@@ -18,7 +18,8 @@ public PythonBashBuildSnippetProperties(
             bool enableCollectStatic,
             string compressVirtualEnvCommand,
             string compressedVirtualEnvFileName,
-            bool runPythonPackageCommand)
+            bool runPythonPackageCommand,
+            string pythonPackageWheelProperty = null)
         {
             VirtualEnvironmentName = virtualEnvironmentName;
             VirtualEnvironmentModule = virtualEnvironmentModule;
@@ -28,6 +29,7 @@ public PythonBashBuildSnippetProperties(
             CompressVirtualEnvCommand = compressVirtualEnvCommand;
             CompressedVirtualEnvFileName = compressedVirtualEnvFileName;
             RunPythonPackageCommand = runPythonPackageCommand;
+            PythonPackageWheelProperty = pythonPackageWheelProperty;
         }
 
         public string VirtualEnvironmentName { get; set; }
@@ -48,5 +50,7 @@ public PythonBashBuildSnippetProperties(
         public string CompressedVirtualEnvFileName { get; set; }
 
         public bool RunPythonPackageCommand { get; set;}
+
+        public string PythonPackageWheelProperty { get; set; }
     }
 }
diff --git a/src/BuildScriptGenerator/Python/PythonManifestFilePropertyKeys.cs b/src/BuildScriptGenerator/Python/PythonManifestFilePropertyKeys.cs
@@ -10,5 +10,6 @@ internal static class PythonManifestFilePropertyKeys
         internal const string CompressedVirtualEnvFile = "compressedVirtualEnvFile";
         internal const string VirtualEnvName = "virtualEnvName";
         internal const string PackageDir = "packagedir";
+        internal const string PackageWheel = "packagewheel";
     }
 }
diff --git a/src/BuildScriptGenerator/Python/PythonPlatform.cs b/src/BuildScriptGenerator/Python/PythonPlatform.cs
@@ -31,6 +31,12 @@ namespace Microsoft.Oryx.BuildScriptGenerator.Python
     [BuildProperty(
         TargetPackageDirectoryPropertyKey,
         "If provided, packages will be downloaded to the given directory instead of to a virtual environment.")]
+    [BuildProperty(
+        PythonPackageWheelPropertyKey,
+        "If provided, package wheels will be built with universal flag. For example," +
+        "folder. Option is '" + UniversalWheel + ". Default is to not Non universal wheel. " +
+        "For example, when this property is enabled, wheel build command will be "+
+        "'python setup.py bdist_wheel --universal'")]
     internal class PythonPlatform : IProgrammingPlatform
     {
         /// <summary>
@@ -48,6 +54,11 @@ internal class PythonPlatform : IProgrammingPlatform
         /// </summary>
         internal const string CompressVirtualEnvPropertyKey = "compress_virtualenv";
 
+        /// <summary>
+        /// The Package Wheel Property.
+        /// </summary>
+        internal const string PythonPackageWheelPropertyKey = "packagewheel";
+
         /// <summary>
         /// The zip option.
         /// </summary>
@@ -57,6 +68,12 @@ internal class PythonPlatform : IProgrammingPlatform
         /// The tar-gz option.
         /// </summary>
         internal const string TarGzOption = "tar-gz";
+
+        /// <summary>
+        /// The Universal Wheel option.
+        /// </summary>
+        internal const string UniversalWheel = "universal";
+
         private readonly BuildScriptGeneratorOptions _commonOptions;
         private readonly PythonScriptGeneratorOptions _pythonScriptGeneratorOptions;
         private readonly IPythonVersionProvider _versionProvider;
@@ -144,6 +161,24 @@ public BuildScriptSnippet GenerateBashBuildScriptSnippet(
             var packageDir = GetPackageDirectory(context);
             var virtualEnvName = GetVirtualEnvironmentName(context);
             var isPythonPackageCommandEnabled = _commonOptions.ShouldPackage;
+            var pythonPackageWheelType = GetPythonPackageWheelType(context);
+
+            if (!isPythonPackageCommandEnabled && !string.IsNullOrWhiteSpace(pythonPackageWheelType))
+            {
+                throw new InvalidUsageException($"Option '{PythonPackageWheelPropertyKey}' can't exist" +
+                    $"without package command being enabled. Please provide --package along with wheel type");
+            }
+
+            if (isPythonPackageCommandEnabled &&
+                !string.IsNullOrWhiteSpace(pythonPackageWheelType))
+            {
+                if (!string.Equals(pythonPackageWheelType.ToLower(), "universal"))
+                {
+                    throw new InvalidUsageException($"Option '{PythonPackageWheelPropertyKey}' can only have 'universal' as value.'");
+                }
+
+                manifestFileProperties[PythonManifestFilePropertyKeys.PackageWheel] = pythonPackageWheelType;
+            }
 
             if (!string.IsNullOrWhiteSpace(packageDir) && !string.IsNullOrWhiteSpace(virtualEnvName))
             {
@@ -205,7 +240,8 @@ public BuildScriptSnippet GenerateBashBuildScriptSnippet(
                 enableCollectStatic: _pythonScriptGeneratorOptions.EnableCollectStatic,
                 compressVirtualEnvCommand: compressVirtualEnvCommand,
                 compressedVirtualEnvFileName: compressedVirtualEnvFileName,
-                runPythonPackageCommand: isPythonPackageCommandEnabled);
+                runPythonPackageCommand: isPythonPackageCommandEnabled,
+                pythonPackageWheelProperty: pythonPackageWheelType);
             string script = TemplateHelper.Render(
                 TemplateHelper.TemplateResource.PythonSnippet,
                 scriptProps,
@@ -519,6 +555,16 @@ private string GetVirtualEnvironmentName(BuildScriptGeneratorContext context)
             return virtualEnvName;
         }
 
+        private string GetPythonPackageWheelType(BuildScriptGeneratorContext context)
+        {
+            if (context.Properties == null ||
+                !context.Properties.TryGetValue(PythonPackageWheelPropertyKey, out var packageWheelProperty))
+            {
+                packageWheelProperty = string.Empty;
+            }
+
+            return packageWheelProperty;
+        }
         private string GetMaxSatisfyingVersionAndVerify(string version)
         {
             var supportedVersions = SupportedVersions;

diff --git a/tests/BuildScriptGenerator.Tests/Python/PythonBashBuildSnippetTest.cs b/tests/BuildScriptGenerator.Tests/Python/PythonBashBuildSnippetTest.cs
@@ -22,7 +22,8 @@ public void GeneratedSnippet_ContainsCollectStatic_IfDisableCollectStatic_IsFals
                 enableCollectStatic: true,
                 compressVirtualEnvCommand: null,
                 compressedVirtualEnvFileName: null,
-                runPythonPackageCommand: false);
+                runPythonPackageCommand: false
+                );
 
             // Act
             var text = TemplateHelper.Render(TemplateHelper.TemplateResource.PythonSnippet, snippetProps);
@@ -51,5 +52,73 @@ public void GeneratedSnippet_DoesNotContainCollectStatic_IfDisableCollectStatic_
             // Assert
             Assert.DoesNotContain("manage.py collectstatic", text);
         }
+
+        [Fact]
+        public void GeneratedSnippet_DoesNotContainPackageWheelType_If_PackageWheelType_IsNotProvided()
+        {
+            // Arrange
+            var snippetProps = new PythonBashBuildSnippetProperties(
+                virtualEnvironmentName: null,
+                virtualEnvironmentModule: null,
+                virtualEnvironmentParameters: null,
+                packagesDirectory: "packages_dir",
+                enableCollectStatic: false,
+                compressVirtualEnvCommand: null,
+                compressedVirtualEnvFileName: null,
+                runPythonPackageCommand: true);
+
+            // Act
+            var text = TemplateHelper.Render(TemplateHelper.TemplateResource.PythonSnippet, snippetProps);
+
+            // Assert
+            Assert.DoesNotContain("Creating universal package wheel", text);
+            Assert.Contains("setup.py sdist --formats=gztar,zip,tar bdist_wheel", text);
+        }
+
+        [Fact]
+        public void GeneratedSnippet_DoesNotContainPackageWheelType_When_PackageCommand_IsNotPresent()
+        {
+            // Arrange
+            var snippetProps = new PythonBashBuildSnippetProperties(
+                virtualEnvironmentName: null,
+                virtualEnvironmentModule: null,
+                virtualEnvironmentParameters: null,
+                packagesDirectory: "packages_dir",
+                enableCollectStatic: false,
+                compressVirtualEnvCommand: null,
+                compressedVirtualEnvFileName: null,
+                runPythonPackageCommand: false,
+                pythonPackageWheelProperty: "universal");
+
+            // Act
+            var text = TemplateHelper.Render(TemplateHelper.TemplateResource.PythonSnippet, snippetProps);
+
+            // Assert
+            Assert.DoesNotContain("Creating universal package wheel", text);
+            Assert.DoesNotContain("Creating non universal package wheel", text);
+        }
+
+        [Fact]
+        public void GeneratedSnippet_ContainsPackageWheelType_When_PackageCommandAndPackageWheelType_IsPresent()
+        {
+            // Arrange
+            var snippetProps = new PythonBashBuildSnippetProperties(
+                virtualEnvironmentName: null,
+                virtualEnvironmentModule: null,
+                virtualEnvironmentParameters: null,
+                packagesDirectory: "packages_dir",
+                enableCollectStatic: false,
+                compressVirtualEnvCommand: null,
+                compressedVirtualEnvFileName: null,
+                runPythonPackageCommand: true,
+                pythonPackageWheelProperty: "universal");
+
+            // Act
+            var text = TemplateHelper.Render(TemplateHelper.TemplateResource.PythonSnippet, snippetProps);
+
+            // Assert
+            Assert.Contains("Creating universal package wheel", text);
+            Assert.Contains("setup.py sdist --formats=gztar,zip,tar bdist_wheel --universal", text);
+        }
     }
 }
diff --git a/tests/Oryx.BuildImage.Tests/Python/PythonPackageTest.cs b/tests/Oryx.BuildImage.Tests/Python/PythonPackageTest.cs
@@ -103,7 +103,7 @@ public void CanBuildPython3Packages(
                 .ToString();
 
             // Act
-            var image = Settings.BuildImageWithRootAccess;
+            var image = Settings.LtsVersionsBuildImageWithRootAccess;
             var result = _dockerCli.Run(image, "/bin/bash", new[] { "-c", script });
 
             // Assert contained file names