From c8ab45556b453f981cf40377104515b24f0029b2 Mon Sep 17 00:00:00 2001 From: Jonathan Pryor Date: Fri, 5 Jun 2020 19:35:28 -0400 Subject: [PATCH] [ci] Install JetBrains OpenJDK 8, 11 (#4773) Context: https://github.com/xamarin/xamarin-android/pull/4567 We want to support using OpenJDK 11 to build Xamarin.Android projects. For better or worse, we can't "change the entire world," not all at once. In particular, the Android Designer Integration Tests do not currently support building with or running under JDK 11. "Split the difference", kind of: Instead of installing Corretto JDK 8 into `$HOME/android-toolchain/jdk`, install JetBrains OpenJDK 8 and 11 into the new paths: * `$HOME/android-toolchain/jdk-1.8` * `$HOME/android-toolchain/jdk-11` We will continue *using* JDK 1.8 by default, until PR #4567 is ready to completely merge, at which point we'll use JDK 11 to build Xamarin.Android, while continuing to use JDK 1.8 with the Designer. Co-authored-by: Marek Habersack --- .../automation/azure-pipelines-oss.yaml | 3 +- build-tools/automation/azure-pipelines.yaml | 3 +- .../ConfigAndData/Configurables.Linux.cs | 2 + .../ConfigAndData/Configurables.MacOS.cs | 5 +- .../ConfigAndData/Configurables.Windows.cs | 17 +- .../xaprepare/ConfigAndData/Configurables.cs | 41 ++- .../xaprepare/OperatingSystems/OS.cs | 2 +- .../Scenarios/Scenario_AndroidToolchain.cs | 5 +- .../xaprepare/Scenarios/Scenario_Standard.cs | 3 +- .../Step_InstallCorrettoOpenJDK.Linux.cs | 14 - .../Step_InstallCorrettoOpenJDK.Windows.cs | 17 -- .../Steps/Step_InstallCorrettoOpenJDK.cs | 160 ---------- .../Step_InstallJetBrainsOpenJDK.Linux.cs | 13 + .../Step_InstallJetBrainsOpenJDK.MacOS.cs | 14 + .../Step_InstallJetBrainsOpenJDK.Unix.cs | 13 + .../Step_InstallJetBrainsOpenJDK.Windows.cs | 52 ++++ .../Steps/Step_InstallJetBrainsOpenJDK.cs | 273 ++++++++++++++++++ .../Step_PrepareExternalJavaInterop.Unix.cs | 1 + .../Steps/Step_PrepareImageDependencies.cs | 1 - .../xaprepare/xaprepare/xaprepare.csproj | 9 +- 20 files changed, 419 insertions(+), 229 deletions(-) delete mode 100644 build-tools/xaprepare/xaprepare/Steps/Step_InstallCorrettoOpenJDK.Linux.cs delete mode 100644 build-tools/xaprepare/xaprepare/Steps/Step_InstallCorrettoOpenJDK.Windows.cs delete mode 100644 build-tools/xaprepare/xaprepare/Steps/Step_InstallCorrettoOpenJDK.cs create mode 100644 build-tools/xaprepare/xaprepare/Steps/Step_InstallJetBrainsOpenJDK.Linux.cs create mode 100644 build-tools/xaprepare/xaprepare/Steps/Step_InstallJetBrainsOpenJDK.MacOS.cs create mode 100644 build-tools/xaprepare/xaprepare/Steps/Step_InstallJetBrainsOpenJDK.Unix.cs create mode 100644 build-tools/xaprepare/xaprepare/Steps/Step_InstallJetBrainsOpenJDK.Windows.cs create mode 100644 build-tools/xaprepare/xaprepare/Steps/Step_InstallJetBrainsOpenJDK.cs diff --git a/build-tools/automation/azure-pipelines-oss.yaml b/build-tools/automation/azure-pipelines-oss.yaml index 000df5ce26f..760da6a867c 100644 --- a/build-tools/automation/azure-pipelines-oss.yaml +++ b/build-tools/automation/azure-pipelines-oss.yaml @@ -17,6 +17,7 @@ pr: # Predefined variables: https://docs.microsoft.com/en-us/azure/devops/pipelines/build/variables?view=azure-devops&tabs=yaml # https://dev.azure.com/xamarin/public/_apps/hub/ms.vss-ciworkflow.build-ci-hub?_a=edit-build-definition&id=48&view=Tab_Variables variables: + XA.Jdk.Folder: jdk-1.8 XA.Build.MacOSSPool: VSEng-Xamarin-RedmondMacMojaveBuildPool-Android-OSS XA.Build.LinuxOSSPool: Xamarin-Android-Ubuntu-Public @@ -157,7 +158,7 @@ stages: - script: echo "##vso[task.setvariable variable=HOME]$(Agent.HomeDirectory)" displayName: set HOME to agent directory - - script: echo "##vso[task.setvariable variable=PATH]$PATH:$(Agent.HomeDirectory)/android-toolchain/jdk/bin" + - script: echo "##vso[task.setvariable variable=PATH]$PATH:$(Agent.HomeDirectory)/android-toolchain/$(XA.Jdk.Folder)/bin" displayName: append jdk tools to PATH - script: make jenkins V=1 PREPARE_CI_PR=1 PREPARE_AUTOPROVISION=1 CONFIGURATION=$(XA.Build.Configuration) diff --git a/build-tools/automation/azure-pipelines.yaml b/build-tools/automation/azure-pipelines.yaml index c0bab5b02b5..0014a0f8c8e 100644 --- a/build-tools/automation/azure-pipelines.yaml +++ b/build-tools/automation/azure-pipelines.yaml @@ -44,6 +44,7 @@ resources: # Global variables variables: + XA.Jdk.Folder: jdk-1.8 NuGetArtifactName: nupkgs InstallerArtifactName: installers TestAssembliesArtifactName: test-assemblies @@ -96,7 +97,7 @@ stages: - checkout: self submodules: recursive - - script: echo "##vso[task.setvariable variable=JAVA_HOME]$HOME/Library/Android/jdk" + - script: echo "##vso[task.setvariable variable=JAVA_HOME]$HOME/Library/Android/$(XA.Jdk.Folder)" displayName: set JAVA_HOME - template: yaml-templates/use-dot-net.yaml diff --git a/build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.Linux.cs b/build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.Linux.cs index d65f02e2955..4cdd9f67ab7 100644 --- a/build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.Linux.cs +++ b/build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.Linux.cs @@ -4,6 +4,8 @@ namespace Xamarin.Android.Prepare { partial class Configurables { + const string JetBrainsOpenJDKOperatingSystem = "linux-x64"; + partial class Urls { public static readonly Uri Corretto = new Uri ($"{Corretto_BaseUri}{CorrettoUrlPathVersion}/amazon-corretto-{CorrettoDistVersion}-linux-x64.tar.gz"); diff --git a/build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.MacOS.cs b/build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.MacOS.cs index 5b75ed09f4b..c94e38eea45 100644 --- a/build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.MacOS.cs +++ b/build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.MacOS.cs @@ -4,10 +4,7 @@ namespace Xamarin.Android.Prepare { partial class Configurables { - partial class Urls - { - public static readonly Uri Corretto = new Uri ($"{Corretto_BaseUri}{CorrettoUrlPathVersion}/amazon-corretto-{CorrettoDistVersion}-macosx-x64.tar.gz"); - } + const string JetBrainsOpenJDKOperatingSystem = "osx-x64"; partial class Defaults { diff --git a/build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.Windows.cs b/build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.Windows.cs index f11e83793ce..a192fdf7564 100644 --- a/build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.Windows.cs +++ b/build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.Windows.cs @@ -5,26 +5,11 @@ namespace Xamarin.Android.Prepare { partial class Configurables { - partial class Urls - { - public static Uri Corretto => GetWindowsCorrettoUrl (); - - public static readonly Uri Corretto64 = new Uri ($"{Corretto_BaseUri}{CorrettoUrlPathVersion}/amazon-corretto-{CorrettoDistVersion}-windows-x64-jdk.zip"); - public static readonly Uri Corretto32 = new Uri ($"{Corretto_BaseUri}{CorrettoUrlPathVersion}/amazon-corretto-{CorrettoDistVersion}-windows-x86-jdk.zip"); - - static Uri GetWindowsCorrettoUrl () - { - if (Context.Instance.OS == null) - return Corretto64; - - return Context.Instance.OS.Is64Bit ? Corretto64 : Corretto32; - } - } + const string JetBrainsOpenJDKOperatingSystem = "windows-x64"; partial class Defaults { public const string NativeLibraryExtension = ".dll"; - public static readonly Version CorrettoVersion = Version.Parse (CorrettoDistVersion); } partial class Paths diff --git a/build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.cs b/build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.cs index b980bfa080d..f3ee0bbd5bd 100644 --- a/build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.cs +++ b/build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.cs @@ -15,6 +15,14 @@ namespace Xamarin.Android.Prepare // partial class Configurables { + const string JetBrainsOpenJDK11Version = "11.0.4"; + const string JetBrainsOpenJDK11Release = "546.1"; + static readonly string JetBrainsOpenJDK11DownloadVersion = JetBrainsOpenJDK11Version.Replace ('.', '_'); + + const string JetBrainsOpenJDK8Version = "8.202"; + const string JetBrainsOpenJDK8Release = "1483.37"; + static readonly string JetBrainsOpenJDK8DownloadVersion = JetBrainsOpenJDK8Version.Replace ('.', 'u'); + const string CorrettoDistVersion = "8.242.08.1"; const string CorrettoUrlPathVersion = CorrettoDistVersion; @@ -22,8 +30,15 @@ partial class Configurables public static partial class Urls { - // Keep the trailing slash here - OS-specific code assumes it's there. - public const string Corretto_BaseUri = "https://corretto.aws/downloads/resources/"; + // https://bintray.com/jetbrains/intellij-jdk/download_file?file_path=jbrsdk-8u202-linux-x64-b1483.37.tar.gz + // https://bintray.com/jetbrains/intellij-jdk/download_file?file_path=jbrsdk-8u202-osx-x64-b1483.37.tar.gz + // https://bintray.com/jetbrains/intellij-jdk/download_file?file_path=jbrsdk-8u202-windows-x64-b1483.37.tar.gz + public static readonly Uri JetBrainsOpenJDK8 = new Uri ($"https://bintray.com/jetbrains/intellij-jdk/download_file?file_path=jbrsdk-{JetBrainsOpenJDK8DownloadVersion}-{JetBrainsOpenJDKOperatingSystem}-b{JetBrainsOpenJDK8Release}.tar.gz"); + + // https://bintray.com/jetbrains/intellij-jdk/download_file?file_path=jbrsdk-11_0_4-linux-x64-b546.1.tar.gz + // https://bintray.com/jetbrains/intellij-jdk/download_file?file_path=jbrsdk-11_0_4-osx-x64-b546.1.tar.gz + // https://bintray.com/jetbrains/intellij-jdk/download_file?file_path=jbrsdk-11_0_4-windows-x64-b546.1.tar.gz + public static readonly Uri JetBrainsOpenJDK11 = new Uri ($"https://bintray.com/jetbrains/intellij-jdk/download_file?file_path=jbrsdk-{JetBrainsOpenJDK11DownloadVersion}-{JetBrainsOpenJDKOperatingSystem}-b{JetBrainsOpenJDK11Release}.tar.gz"); /// /// Base URL for all Android SDK and NDK downloads. Used in @@ -39,6 +54,14 @@ public static partial class Defaults { public static readonly char[] PropertyListSeparator = new [] { ':' }; + public static readonly string JdkFolder = "jdk-1.8"; + + public static readonly Version JetBrainsOpenJDK11Version = new Version (Configurables.JetBrainsOpenJDK11Version); + public static readonly Version JetBrainsOpenJDK11Release = new Version (Configurables.JetBrainsOpenJDK11Release); + + public static readonly Version JetBrainsOpenJDK8Version = new Version (Configurables.JetBrainsOpenJDK8Version); + public static readonly Version JetBrainsOpenJDK8Release = new Version (Configurables.JetBrainsOpenJDK8Release); + // Mono runtimes public const string DebugFileExtension = ".pdb"; public const string MonoHostMingwRuntimeNativeLibraryExtension = WindowsDLLSuffix; @@ -104,9 +127,9 @@ public static partial class Defaults public const int DefaultMaximumParallelTasks = 5; /// - /// The maximum JDK version we support. Note: this will probably go away with Corretto + /// The maximum JDK version we support. /// - public const int MaxJDKVersion = 8; + public static readonly Version MaxJDKVersion = new Version (11, 99, 0); /// /// Prefix for all the log files created by the bootstrapper. @@ -314,10 +337,12 @@ public static partial class Paths public static string Mingw32CmakePath => GetCachedPath (ref mingw32CmakePath, () => Path.Combine (BuildBinDir, "mingw-32.cmake")); public static string Mingw64CmakePath => GetCachedPath (ref mingw64CmakePath, () => Path.Combine (BuildBinDir, "mingw-64.cmake")); - // Corretto OpenJDK - public static string CorrettoCacheDir => GetCachedPath (ref correttoCacheDir, () => ctx.Properties.GetRequiredValue (KnownProperties.AndroidToolchainCacheDirectory)); - public static string CorrettoInstallDir => GetCachedPath (ref correttoInstallDir, () => Path.Combine (ctx.Properties.GetRequiredValue (KnownProperties.AndroidToolchainDirectory), "jdk")); + // JetBrains OpenJDK + public static string OpenJDK8InstallDir => GetCachedPath (ref openJDK8InstallDir, () => Path.Combine (ctx.Properties.GetRequiredValue (KnownProperties.AndroidToolchainDirectory), "jdk-1.8")); + public static string OpenJDK8CacheDir => GetCachedPath (ref openJDK8CacheDir, () => ctx.Properties.GetRequiredValue (KnownProperties.AndroidToolchainCacheDirectory)); + public static string OpenJDK11InstallDir => GetCachedPath (ref openJDK11InstallDir, () => Path.Combine (ctx.Properties.GetRequiredValue (KnownProperties.AndroidToolchainDirectory), "jdk-11")); + public static string OpenJDK11CacheDir => GetCachedPath (ref openJDK11CacheDir, () => ctx.Properties.GetRequiredValue (KnownProperties.AndroidToolchainCacheDirectory)); // bundle public static string BCLTestsArchiveName = "bcl-tests.zip"; @@ -403,6 +428,8 @@ static string GetCachedPath (ref string? variable, Func creator) static string? monoSdksTpnExternalPath; static string? monoSDKSIncludeDestDir; static string? monoLlvmTpnPath; + static string? openJDK8InstallDir, openJDK11InstallDir; + static string? openJDK8CacheDir, openJDK11CacheDir; } } } diff --git a/build-tools/xaprepare/xaprepare/OperatingSystems/OS.cs b/build-tools/xaprepare/xaprepare/OperatingSystems/OS.cs index fed5e822f2c..24ca2c5aeba 100644 --- a/build-tools/xaprepare/xaprepare/OperatingSystems/OS.cs +++ b/build-tools/xaprepare/xaprepare/OperatingSystems/OS.cs @@ -184,7 +184,7 @@ protected virtual bool InitOS () JavaHome = Context.Instance.Properties.GetValue (KnownProperties.JavaSdkDirectory)?.Trim () ?? String.Empty; if (String.IsNullOrEmpty (JavaHome)) { var androidToolchainDirectory = Context.Instance.Properties.GetValue (KnownProperties.AndroidToolchainDirectory)?.Trim (); - JavaHome = Path.Combine (androidToolchainDirectory, "jdk"); + JavaHome = Path.Combine (androidToolchainDirectory, Configurables.Defaults.JdkFolder); } string extension = IsWindows ? ".exe" : string.Empty; diff --git a/build-tools/xaprepare/xaprepare/Scenarios/Scenario_AndroidToolchain.cs b/build-tools/xaprepare/xaprepare/Scenarios/Scenario_AndroidToolchain.cs index be9795c6dc2..ab7a5e8bcb2 100644 --- a/build-tools/xaprepare/xaprepare/Scenarios/Scenario_AndroidToolchain.cs +++ b/build-tools/xaprepare/xaprepare/Scenarios/Scenario_AndroidToolchain.cs @@ -6,12 +6,13 @@ namespace Xamarin.Android.Prepare partial class Scenario_AndroidToolchain : ScenarioNoStandardEndSteps { public Scenario_AndroidToolchain () - : base ("AndroidToolchain", "Install Android SDK, NDK and Corretto JDK.") + : base ("AndroidToolchain", "Install Android SDK, NDK and OpenJDK.") {} protected override void AddSteps (Context context) { - Steps.Add (new Step_InstallCorrettoOpenJDK ()); + Steps.Add (new Step_InstallJetBrainsOpenJDK8 ()); + Steps.Add (new Step_InstallJetBrainsOpenJDK11 ()); Steps.Add (new Step_Android_SDK_NDK ()); // disable installation of missing programs... diff --git a/build-tools/xaprepare/xaprepare/Scenarios/Scenario_Standard.cs b/build-tools/xaprepare/xaprepare/Scenarios/Scenario_Standard.cs index f4c3f1531f9..ea889380dee 100644 --- a/build-tools/xaprepare/xaprepare/Scenarios/Scenario_Standard.cs +++ b/build-tools/xaprepare/xaprepare/Scenarios/Scenario_Standard.cs @@ -19,7 +19,8 @@ protected override void AddSteps (Context context) throw new ArgumentNullException (nameof (context)); Steps.Add (new Step_ShowEnabledRuntimes ()); - Steps.Add (new Step_InstallCorrettoOpenJDK ()); + Steps.Add (new Step_InstallJetBrainsOpenJDK8 ()); + Steps.Add (new Step_InstallJetBrainsOpenJDK11 ()); Steps.Add (new Step_Android_SDK_NDK ()); Steps.Add (new Step_GenerateFiles (atBuildStart: true)); Steps.Add (new Step_PrepareProps ()); diff --git a/build-tools/xaprepare/xaprepare/Steps/Step_InstallCorrettoOpenJDK.Linux.cs b/build-tools/xaprepare/xaprepare/Steps/Step_InstallCorrettoOpenJDK.Linux.cs deleted file mode 100644 index 1fc5813dc60..00000000000 --- a/build-tools/xaprepare/xaprepare/Steps/Step_InstallCorrettoOpenJDK.Linux.cs +++ /dev/null @@ -1,14 +0,0 @@ -using System; - -namespace Xamarin.Android.Prepare -{ - partial class Step_InstallCorrettoOpenJDK - { - string GetArchiveRootDirectoryName () - { - Version v = Configurables.Defaults.CorrettoVersion; - - return $"amazon-corretto-{v.Major}.{v.Minor}.{v.Build:00}.{v.Revision}-linux-x64"; - } - } -} diff --git a/build-tools/xaprepare/xaprepare/Steps/Step_InstallCorrettoOpenJDK.Windows.cs b/build-tools/xaprepare/xaprepare/Steps/Step_InstallCorrettoOpenJDK.Windows.cs deleted file mode 100644 index f54de803e03..00000000000 --- a/build-tools/xaprepare/xaprepare/Steps/Step_InstallCorrettoOpenJDK.Windows.cs +++ /dev/null @@ -1,17 +0,0 @@ -using System; -using System.Threading; - -namespace Xamarin.Android.Prepare -{ - partial class Step_InstallCorrettoOpenJDK - { - static readonly TimeSpan BeforeMoveSleepTime = TimeSpan.FromSeconds (30); - - string GetArchiveRootDirectoryName () - { - Version v = Configurables.Defaults.CorrettoVersion; - - return $"jdk1.8.0_{v.Minor}"; - } - } -} diff --git a/build-tools/xaprepare/xaprepare/Steps/Step_InstallCorrettoOpenJDK.cs b/build-tools/xaprepare/xaprepare/Steps/Step_InstallCorrettoOpenJDK.cs deleted file mode 100644 index 82babb1e59c..00000000000 --- a/build-tools/xaprepare/xaprepare/Steps/Step_InstallCorrettoOpenJDK.cs +++ /dev/null @@ -1,160 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Net; -using System.Reflection; -using System.Threading.Tasks; -using System.Xml.Linq; - -namespace Xamarin.Android.Prepare -{ - partial class Step_InstallCorrettoOpenJDK : StepWithDownloadProgress - { - // Paths relative to JDK installation root, just for a cursory check whether we have a sane JDK instance - // NOTE: file extensions are not necessary here - static readonly List jdkFiles = new List { - Path.Combine ("bin", "java"), - Path.Combine ("bin", "javac"), - Path.Combine ("include", "jni.h"), - }; - - public Step_InstallCorrettoOpenJDK () - : base ("Installing OpenJDK (Amazon Corretto 8)") - {} - - protected override async Task Execute (Context context) - { - string corettoInstallDir = Configurables.Paths.CorrettoInstallDir; - if (CorrettoExistsAndIsValid (corettoInstallDir, out string installedVersion)) { - Log.Status ("Corretto version "); - Log.Status (installedVersion, ConsoleColor.Yellow); - Log.StatusLine (" already installed in: ", corettoInstallDir, tailColor: ConsoleColor.Cyan); - return true; - } - - Log.StatusLine ($"Corretto JDK {Configurables.Defaults.CorrettoVersion} will be installed"); - Uri correttoURL = Configurables.Urls.Corretto; - if (correttoURL == null) - throw new InvalidOperationException ("Corretto URL must not be null"); - - string packageName = Path.GetFileName (correttoURL.LocalPath); - string localPackagePath = Path.Combine (Configurables.Paths.CorrettoCacheDir, packageName); - if (!await DownloadCorretto (context, localPackagePath, correttoURL)) - return false; - - string tempDir = $"{corettoInstallDir}.temp"; - try { - if (!await Utilities.Unpack (localPackagePath, tempDir, cleanDestinatioBeforeUnpacking: true)) { - Log.ErrorLine ("Failed to install Corretto"); - return false; - } - - string rootDirName = GetArchiveRootDirectoryName (); - string rootDir = Path.Combine (tempDir, rootDirName); - Log.DebugLine ($"{context.OS.Type} root directory name of Corretto OpenJDK package is: {rootDirName}"); - if (!Directory.Exists (rootDir)) { - Log.ErrorLine ($"Corretto root directory not found after unpacking: {rootDirName}"); - return false; - } - - Utilities.MoveDirectoryContentsRecursively (rootDir, corettoInstallDir); - } finally { - Utilities.DeleteDirectorySilent (tempDir); - // Clean up zip after extraction if running on a hosted azure pipelines agent. - if (context.IsRunningOnHostedAzureAgent) - Utilities.DeleteFileSilent (localPackagePath); - } - - return true; - } - - async Task DownloadCorretto (Context context, string localPackagePath, Uri url) - { - if (File.Exists (localPackagePath)) { - Log.StatusLine ("Corretto archive already downloaded"); - return true; - } - - Log.StatusLine ("Downloading Corretto from ", url.ToString (), tailColor: ConsoleColor.White); - (bool success, ulong size, HttpStatusCode status) = await Utilities.GetDownloadSizeWithStatus (url); - if (!success) { - if (status == HttpStatusCode.NotFound) - Log.ErrorLine ("Corretto archive URL not found"); - else - Log.ErrorLine ($"Failed to obtain Corretto size. HTTP status code: {status} ({(int)status})"); - return false; - } - - DownloadStatus downloadStatus = Utilities.SetupDownloadStatus (context, size, context.InteractiveSession); - Log.StatusLine ($" {context.Characters.Link} {url}", ConsoleColor.White); - await Download (context, url, localPackagePath, "Corretto", Path.GetFileName (localPackagePath), downloadStatus); - - if (!File.Exists (localPackagePath)) { - Log.ErrorLine ($"Download of Corretto from {url} failed."); - return false; - } - - return true; - } - - bool CorrettoExistsAndIsValid (string installDir, out string installedVersion) - { - installedVersion = String.Empty; - if (!Directory.Exists (installDir)) { - Log.DebugLine ($"Corretto directory {installDir} does not exist"); - return false; - } - - string corettoVersionFile = Path.Combine (installDir, "version.txt"); - if (!File.Exists (corettoVersionFile)) { - Log.DebugLine ($"Corretto version file {corettoVersionFile} does not exist"); - return false; - } - - string[] lines = File.ReadAllLines (corettoVersionFile); - if (lines == null || lines.Length == 0) { - Log.DebugLine ($"Corretto version file {corettoVersionFile} is empty, cannot determine version"); - return false; - } - - string cv = lines [0].Trim (); - if (String.IsNullOrEmpty (cv)) { - Log.DebugLine ($"Corretto version is empty"); - return false; - } - installedVersion = cv; - - if (!Version.TryParse (cv, out Version cversion)) { - Log.DebugLine ($"Unable to parse Corretto version from: {cv}"); - return false; - } - - if (cversion != Configurables.Defaults.CorrettoVersion) { - Log.DebugLine ($"Invalid Corretto version. Need {Configurables.Defaults.CorrettoVersion}, found {cversion}"); - return false; - } - - foreach (string f in jdkFiles) { - string file = Path.Combine (installDir, f); - if (!File.Exists (file)) { - bool foundExe = false; - foreach (string exe in Utilities.FindExecutable (f)) { - file = Path.Combine (installDir, exe); - if (File.Exists (file)) { - foundExe = true; - break; - } - } - - if (!foundExe) { - Log.DebugLine ($"JDK file {file} missing from Corretto"); - return false; - } - } - } - - return true; - } - } -} diff --git a/build-tools/xaprepare/xaprepare/Steps/Step_InstallJetBrainsOpenJDK.Linux.cs b/build-tools/xaprepare/xaprepare/Steps/Step_InstallJetBrainsOpenJDK.Linux.cs new file mode 100644 index 00000000000..d41d9cf0ad8 --- /dev/null +++ b/build-tools/xaprepare/xaprepare/Steps/Step_InstallJetBrainsOpenJDK.Linux.cs @@ -0,0 +1,13 @@ +using System; +using System.Threading.Tasks; + +namespace Xamarin.Android.Prepare +{ + partial class Step_InstallJetBrainsOpenJDK + { + void MoveContents (string sourceDir, string destinationDir) + { + Utilities.MoveDirectoryContentsRecursively (sourceDir, destinationDir); + } + } +} diff --git a/build-tools/xaprepare/xaprepare/Steps/Step_InstallJetBrainsOpenJDK.MacOS.cs b/build-tools/xaprepare/xaprepare/Steps/Step_InstallJetBrainsOpenJDK.MacOS.cs new file mode 100644 index 00000000000..0cd7d04819d --- /dev/null +++ b/build-tools/xaprepare/xaprepare/Steps/Step_InstallJetBrainsOpenJDK.MacOS.cs @@ -0,0 +1,14 @@ +using System; +using System.IO; + +namespace Xamarin.Android.Prepare +{ + partial class Step_InstallJetBrainsOpenJDK + { + void MoveContents (string sourceDir, string destinationDir) + { + string realSourceDir = Path.Combine (sourceDir, "Contents", "Home"); + Utilities.MoveDirectoryContentsRecursively (realSourceDir, destinationDir); + } + } +} diff --git a/build-tools/xaprepare/xaprepare/Steps/Step_InstallJetBrainsOpenJDK.Unix.cs b/build-tools/xaprepare/xaprepare/Steps/Step_InstallJetBrainsOpenJDK.Unix.cs new file mode 100644 index 00000000000..951f837aad6 --- /dev/null +++ b/build-tools/xaprepare/xaprepare/Steps/Step_InstallJetBrainsOpenJDK.Unix.cs @@ -0,0 +1,13 @@ +using System; +using System.Threading.Tasks; + +namespace Xamarin.Android.Prepare +{ + partial class Step_InstallJetBrainsOpenJDK + { + async Task Unpack (string fullArchivePath, string destinationDirectory, bool cleanDestinationBeforeUnpacking = false) + { + return await Utilities.Unpack (fullArchivePath, destinationDirectory, cleanDestinatioBeforeUnpacking: true); + } + } +} diff --git a/build-tools/xaprepare/xaprepare/Steps/Step_InstallJetBrainsOpenJDK.Windows.cs b/build-tools/xaprepare/xaprepare/Steps/Step_InstallJetBrainsOpenJDK.Windows.cs new file mode 100644 index 00000000000..0a018dde010 --- /dev/null +++ b/build-tools/xaprepare/xaprepare/Steps/Step_InstallJetBrainsOpenJDK.Windows.cs @@ -0,0 +1,52 @@ +using System; +using System.IO; +using System.Threading.Tasks; + +namespace Xamarin.Android.Prepare +{ + partial class Step_InstallJetBrainsOpenJDK + { + async Task Unpack (string fullArchivePath, string destinationDirectory, bool cleanDestinationBeforeUnpacking = false) + { + // https://bintray.com/jetbrains/intellij-jdk/download_file?file_path=jbrsdk-8u202-windows-x64-b1483.37.tar.gz + // doesn't contain a single root directory! This causes the + // "JetBrains root directory not found after unpacking" check to fail on Windows. + // "Fix" things by setting destinationDirectory to contain RootDirName, allowing + // the check to succeed. + if (JdkVersion == Configurables.Defaults.JetBrainsOpenJDK8Version) { + destinationDirectory = Path.Combine (destinationDirectory, RootDirName); + } + + // On Windows we don't have Tar available and the Windows package is a .tar.gz + // 7zip can unpack tar.gz but it's a two-stage process - first it decompresses the package, then it can be + // invoked again to extract the actual tar contents. + + if (cleanDestinationBeforeUnpacking) + Utilities.DeleteDirectorySilent (destinationDirectory); + Utilities.CreateDirectory (destinationDirectory); + + var sevenZip = new SevenZipRunner (Context.Instance); + Log.DebugLine ($"Uncompressing {fullArchivePath} to {destinationDirectory}"); + if (!await sevenZip.Extract (fullArchivePath, destinationDirectory)) { + Log.DebugLine ($"Failed to decompress {fullArchivePath}"); + return false; + } + + string tarPath = Path.Combine (destinationDirectory, Path.GetFileNameWithoutExtension (fullArchivePath)); + bool ret = await sevenZip.Extract (tarPath, destinationDirectory); + Utilities.DeleteFileSilent (tarPath); + + if (!ret) { + Log.DebugLine ($"Failed to extract TAR contents from {tarPath}"); + return false; + } + + return true; + } + + void MoveContents (string sourceDir, string destinationDir) + { + Utilities.MoveDirectoryContentsRecursively (sourceDir, destinationDir); + } + } +} diff --git a/build-tools/xaprepare/xaprepare/Steps/Step_InstallJetBrainsOpenJDK.cs b/build-tools/xaprepare/xaprepare/Steps/Step_InstallJetBrainsOpenJDK.cs new file mode 100644 index 00000000000..7381a7dfc2c --- /dev/null +++ b/build-tools/xaprepare/xaprepare/Steps/Step_InstallJetBrainsOpenJDK.cs @@ -0,0 +1,273 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Reflection; +using System.Threading.Tasks; +using System.Xml.Linq; + +namespace Xamarin.Android.Prepare +{ + abstract partial class Step_InstallJetBrainsOpenJDK : StepWithDownloadProgress + { + const string ProductName = "JetBrains OpenJDK"; + const string XAVersionInfoFile = "xa_jdk_version.txt"; + const string URLQueryFilePathField = "file_path"; + + static readonly char[] QuerySeparator = new char[] { ';', '&' }; + + // Paths relative to JDK installation root, just for a cursory check whether we have a sane JDK instance + // NOTE: file extensions are not necessary here + static readonly List jdkFiles = new List { + Path.Combine ("bin", "java"), + Path.Combine ("bin", "javac"), + Path.Combine ("include", "jni.h"), + }; + + public Step_InstallJetBrainsOpenJDK (string description) + : base (description) + {} + + protected abstract string JdkInstallDir {get;} + protected abstract Version JdkVersion {get;} + protected abstract Version JdkRelease {get;} + protected abstract Uri JdkUrl {get;} + protected abstract string JdkCacheDir {get;} + protected abstract string RootDirName {get;} + + protected override async Task Execute (Context context) + { + string jdkInstallDir = JdkInstallDir; + if (OpenJDKExistsAndIsValid (jdkInstallDir, out string installedVersion)) { + Log.Status ($"{ProductName} version "); + Log.Status (installedVersion, ConsoleColor.Yellow); + Log.StatusLine (" already installed in: ", jdkInstallDir, tailColor: ConsoleColor.Cyan); + return true; + } + + Log.StatusLine ($"JetBrains JDK {JdkVersion} r{JdkRelease} will be installed"); + Uri jdkURL = JdkUrl; + if (jdkURL == null) + throw new InvalidOperationException ($"{ProductName} URL must not be null"); + + string[] queryParams = jdkURL.Query.TrimStart ('?').Split (QuerySeparator, StringSplitOptions.RemoveEmptyEntries); + if (queryParams.Length == 0) { + Log.ErrorLine ($"Unable to extract file name from {ProductName} URL as it contains no query component"); + return false; + } + + string? packageName = null; + foreach (string p in queryParams) { + if (!p.StartsWith (URLQueryFilePathField, StringComparison.Ordinal)) { + continue; + } + + int idx = p.IndexOf ('='); + if (idx < 0) { + Log.DebugLine ($"{ProductName} URL query field '{URLQueryFilePathField}' has no value, unable to detect file name"); + break; + } + + packageName = p.Substring (idx + 1).Trim (); + } + + if (String.IsNullOrEmpty (packageName)) { + Log.ErrorLine ($"Unable to extract file name from {ProductName} URL"); + return false; + } + + string localPackagePath = Path.Combine (JdkCacheDir, packageName); + if (!await DownloadOpenJDK (context, localPackagePath, jdkURL)) + return false; + + string tempDir = $"{jdkInstallDir}.temp"; + try { + if (!await Unpack (localPackagePath, tempDir, cleanDestinationBeforeUnpacking: true)) { + Log.ErrorLine ($"Failed to install {ProductName}"); + return false; + } + + string rootDir = Path.Combine (tempDir, RootDirName); + if (!Directory.Exists (rootDir)) { + Log.ErrorLine ($"JetBrains root directory not found after unpacking: {RootDirName}"); + return false; + } + + MoveContents (rootDir, jdkInstallDir); + File.WriteAllText (Path.Combine (jdkInstallDir, XAVersionInfoFile), $"{JdkRelease}{Environment.NewLine}"); + } finally { + Utilities.DeleteDirectorySilent (tempDir); + // Clean up zip after extraction if running on a hosted azure pipelines agent. + if (context.IsRunningOnHostedAzureAgent) + Utilities.DeleteFileSilent (localPackagePath); + } + + return true; + } + + async Task DownloadOpenJDK (Context context, string localPackagePath, Uri url) + { + if (File.Exists (localPackagePath)) { + Log.StatusLine ($"{ProductName} archive already downloaded"); + return true; + } + + Log.StatusLine ($"Downloading {ProductName} from ", url.ToString (), tailColor: ConsoleColor.White); + (bool success, ulong size, HttpStatusCode status) = await Utilities.GetDownloadSizeWithStatus (url); + if (!success) { + if (status == HttpStatusCode.NotFound) + Log.ErrorLine ($"{ProductName} archive URL not found"); + else + Log.ErrorLine ($"Failed to obtain {ProductName} size. HTTP status code: {status} ({(int)status})"); + return false; + } + + DownloadStatus downloadStatus = Utilities.SetupDownloadStatus (context, size, context.InteractiveSession); + Log.StatusLine ($" {context.Characters.Link} {url}", ConsoleColor.White); + await Download (context, url, localPackagePath, ProductName, Path.GetFileName (localPackagePath), downloadStatus); + + if (!File.Exists (localPackagePath)) { + Log.ErrorLine ($"Download of {ProductName} from {url} failed."); + return false; + } + + return true; + } + + bool OpenJDKExistsAndIsValid (string installDir, out string installedVersion) + { + installedVersion = null; + if (!Directory.Exists (installDir)) { + Log.DebugLine ($"{ProductName} directory {installDir} does not exist"); + return false; + } + + string corettoVersionFile = Path.Combine (installDir, "version.txt"); + if (File.Exists (corettoVersionFile)) { + Log.DebugLine ($"Corretto version file {corettoVersionFile} found, will replace Corretto with {ProductName}"); + return false; + } + + string jetBrainsReleaseFile = Path.Combine (installDir, "release"); + if (!File.Exists (jetBrainsReleaseFile)) { + Log.DebugLine ($"{ProductName} release file {jetBrainsReleaseFile} does not exist, cannot determine version"); + return false; + } + + string[] lines = File.ReadAllLines (jetBrainsReleaseFile); + if (lines == null || lines.Length == 0) { + Log.DebugLine ($"{ProductName} release file {jetBrainsReleaseFile} is empty, cannot determine version"); + return false; + } + + string cv = null; + foreach (string l in lines) { + string line = l.Trim (); + if (!line.StartsWith ("JAVA_VERSION=", StringComparison.Ordinal)) { + continue; + } + + cv = line.Substring (line.IndexOf ('=') + 1).Trim ('"'); + break; + } + + if (String.IsNullOrEmpty (cv)) { + Log.DebugLine ($"Unable to find version of {ProductName} in release file {jetBrainsReleaseFile}"); + return false; + } + + string xaVersionFile = Path.Combine (installDir, XAVersionInfoFile); + if (!File.Exists (xaVersionFile)) { + installedVersion = cv; + Log.DebugLine ($"Unable to find Xamarin.Android version file {xaVersionFile}"); + return false; + } + + lines = File.ReadAllLines (xaVersionFile); + if (lines == null || lines.Length == 0) { + Log.DebugLine ($"Xamarin.Android version file {xaVersionFile} is empty, cannot determine release version"); + return false; + } + + string rv = lines[0].Trim (); + if (String.IsNullOrEmpty (rv)) { + Log.DebugLine ($"Xamarin.Android version file {xaVersionFile} does not contain release version information"); + return false; + } + + installedVersion = $"{cv} r{rv}"; + + if (!Version.TryParse (cv, out Version cversion)) { + Log.DebugLine ($"Unable to parse {ProductName} version from: {cv}"); + return false; + } + + if (cversion != JdkVersion) { + Log.DebugLine ($"Invalid {ProductName} version. Need {JdkVersion}, found {cversion}"); + return false; + } + + if (!Version.TryParse (rv, out cversion)) { + Log.DebugLine ($"Unable to parse {ProductName} release version from: {rv}"); + return false; + } + + if (cversion != JdkRelease) { + Log.DebugLine ($"Invalid {ProductName} version. Need {JdkRelease}, found {cversion}"); + return false; + } + + foreach (string f in jdkFiles) { + string file = Path.Combine (installDir, f); + if (!File.Exists (file)) { + bool foundExe = false; + foreach (string exe in Utilities.FindExecutable (f)) { + file = Path.Combine (installDir, exe); + if (File.Exists (file)) { + foundExe = true; + break; + } + } + + if (!foundExe) { + Log.DebugLine ($"JDK file {file} missing from {ProductName}"); + return false; + } + } + } + + return true; + } + } + + class Step_InstallJetBrainsOpenJDK8 : Step_InstallJetBrainsOpenJDK { + + public Step_InstallJetBrainsOpenJDK8 () + : base ("Installing {ProductName} 1.8") + { + } + + protected override string JdkInstallDir => Configurables.Paths.OpenJDK8InstallDir; + protected override Version JdkVersion => Configurables.Defaults.JetBrainsOpenJDK8Version; + protected override Version JdkRelease => Configurables.Defaults.JetBrainsOpenJDK8Release; + protected override Uri JdkUrl => Configurables.Urls.JetBrainsOpenJDK8; + protected override string JdkCacheDir => Configurables.Paths.OpenJDK8CacheDir; + protected override string RootDirName => "jdk"; + } + + class Step_InstallJetBrainsOpenJDK11 : Step_InstallJetBrainsOpenJDK { + + public Step_InstallJetBrainsOpenJDK11 () + : base ("Installing {ProductName} 11") + { + } + + protected override string JdkInstallDir => Configurables.Paths.OpenJDK11InstallDir; + protected override Version JdkVersion => Configurables.Defaults.JetBrainsOpenJDK11Version; + protected override Version JdkRelease => Configurables.Defaults.JetBrainsOpenJDK11Release; + protected override Uri JdkUrl => Configurables.Urls.JetBrainsOpenJDK11; + protected override string JdkCacheDir => Configurables.Paths.OpenJDK11CacheDir; + protected override string RootDirName => "jbrsdk"; + } +} diff --git a/build-tools/xaprepare/xaprepare/Steps/Step_PrepareExternalJavaInterop.Unix.cs b/build-tools/xaprepare/xaprepare/Steps/Step_PrepareExternalJavaInterop.Unix.cs index 2da742d8e1c..ce59185c1fb 100644 --- a/build-tools/xaprepare/xaprepare/Steps/Step_PrepareExternalJavaInterop.Unix.cs +++ b/build-tools/xaprepare/xaprepare/Steps/Step_PrepareExternalJavaInterop.Unix.cs @@ -18,6 +18,7 @@ async Task ExecuteOSSpecific (Context context) workingDirectory: javaInteropDir, arguments: new List { "prepare", + "V=1", $"CONFIGURATION={context.Configuration}", $"JAVA_HOME={context.OS.JavaHome}", $"JI_MAX_JDK={Configurables.Defaults.MaxJDKVersion}", diff --git a/build-tools/xaprepare/xaprepare/Steps/Step_PrepareImageDependencies.cs b/build-tools/xaprepare/xaprepare/Steps/Step_PrepareImageDependencies.cs index c07926ab611..acc8f22f107 100644 --- a/build-tools/xaprepare/xaprepare/Steps/Step_PrepareImageDependencies.cs +++ b/build-tools/xaprepare/xaprepare/Steps/Step_PrepareImageDependencies.cs @@ -18,7 +18,6 @@ protected override async Task Execute (Context context) var toolchainDirs = new List { Path.GetFileName (context.Properties.GetRequiredValue (KnownProperties.AndroidSdkDirectory)), Path.GetFileName (context.Properties.GetRequiredValue (KnownProperties.AndroidNdkDirectory)), - Path.GetFileName (Configurables.Paths.CorrettoInstallDir), }; var androidToolchain = new AndroidToolchain (); diff --git a/build-tools/xaprepare/xaprepare/xaprepare.csproj b/build-tools/xaprepare/xaprepare/xaprepare.csproj index e677b21e462..58965358111 100644 --- a/build-tools/xaprepare/xaprepare/xaprepare.csproj +++ b/build-tools/xaprepare/xaprepare/xaprepare.csproj @@ -143,7 +143,7 @@ - + @@ -203,6 +203,7 @@ + @@ -229,7 +230,7 @@ - + @@ -246,7 +247,7 @@ - + @@ -268,7 +269,7 @@ - +