Allow for grabbing dependencies of NuGet packages now

src/Pharmacist.Core/Extractors/NuGetExtractor.cs

@@ -30,11 +30,10 @@ public class NuGetExtractor : IExtractor
         /// </summary>
         /// <param name="targetFramework">The target framework to extract.</param>
         /// <param name="package">The package to extract the information from.</param>
-        /// <param name="supportPackages">The packages for support purposes.</param>
         /// <returns>A task to monitor the progress.</returns>
-        public async Task Extract(NuGetFramework targetFramework, PackageIdentity package, IEnumerable<PackageIdentity> supportPackages = null)
+        public async Task Extract(NuGetFramework targetFramework, PackageIdentity package)
         {
-            var results = (await NuGetPackageHelper.DownloadPackageAndGetLibFilesAndFolder(package, supportPackages, targetFramework).ConfigureAwait(false)).ToList();
+            var results = (await NuGetPackageHelper.DownloadPackageAndGetLibFilesAndFolder(package, targetFramework).ConfigureAwait(false)).ToList();
             Assemblies.AddRange(results.SelectMany(x => x.files));
             SearchDirectories.AddRange(results.Select(x => x.folder));
         }

src/Pharmacist.Core/NuGet/NuGetPackageHelper.cs

@@ -4,6 +4,7 @@
 // See the LICENSE file in the project root for full license information.
 
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Globalization;
 using System.IO;
@@ -48,72 +49,100 @@ static NuGetPackageHelper()
         /// Downloads the specified packages and returns the files and directories where the package NuGet package lives.
         /// </summary>
         /// <param name="packageIdentity">The identity of the packages to find.</param>
-        /// <param name="supportPackageIdentities">Any support libraries where the directories should be included but not the files.</param>
         /// <param name="framework">Optional framework parameter which will force NuGet to evaluate as the specified Framework. If null it will use .NET Standard 2.0.</param>
         /// <param name="nugetSource">Optional v3 nuget source. Will default to default nuget.org servers.</param>
         /// <param name="token">A cancellation token.</param>
         /// <returns>The directory where the NuGet packages are unzipped to.</returns>
-        public static async Task<IEnumerable<(string folder, IEnumerable<string> files)>> DownloadPackageAndGetLibFilesAndFolder(PackageIdentity packageIdentity, IEnumerable<PackageIdentity> supportPackageIdentities = null, NuGetFramework framework = null, PackageSource nugetSource = null, CancellationToken token = default)
+        public static async Task<IEnumerable<(string folder, IEnumerable<string> files)>> DownloadPackageAndGetLibFilesAndFolder(PackageIdentity packageIdentity, NuGetFramework framework = null, PackageSource nugetSource = null, CancellationToken token = default)
         {
             // If the user hasn't selected a default framework to extract, select .NET Standard 2.0
             framework = framework ?? FrameworkConstants.CommonFrameworks.NetStandard20;
 
-            // Get our support libraries together. We will grab the default for the framework passed in if it's packaged based.
-            var defaultSupportLibrary = framework?.GetSupportLibraries() ?? Enumerable.Empty<PackageIdentity>();
-            supportPackageIdentities = (supportPackageIdentities ?? Array.Empty<PackageIdentity>()).Concat(defaultSupportLibrary).Distinct();
+            // Use the provided nuget package source, or use nuget.org
+            var source = new SourceRepository(nugetSource ?? new PackageSource("https://api.nuget.org/v3/index.json"), _providers);
+
+            var librariesToCopy = await GetPackagesToCopy(packageIdentity, source, framework, token).ConfigureAwait(false);
 
-            // Combine together the primary/secondary packages, boolean value to indicate if we should include in our output.
-            var packagesToDownload = new SortedSet<(PackageIdentity packageIdentity, bool includeFiles)>(NuGetPackageIdentityComparer.Default) { (packageIdentity, true) };
-            packagesToDownload.AddRange(supportPackageIdentities.Select(x => (x, false)));
+            var output = librariesToCopy.Select(x => CopyPackageFiles(x.packageIdentity, x.downloadResourceResult, framework, x.includeFilesInOutput, token));
 
-            return await Task.WhenAll(packagesToDownload
-                .Select(x => CopyPackageLibraryItems(x.packageIdentity, nugetSource, framework, x.includeFiles, token)))
-                .ConfigureAwait(false);
+            return output.ToList();
         }
 
-        private static async Task<(string folder, IEnumerable<string> files)> CopyPackageLibraryItems(PackageIdentity package, PackageSource nugetSource, NuGetFramework framework, bool includeFilesInOutput, CancellationToken token)
+        private static async Task<IEnumerable<(PackageIdentity packageIdentity, DownloadResourceResult downloadResourceResult, bool includeFilesInOutput)>> GetPackagesToCopy(PackageIdentity startingPackage, SourceRepository source, NuGetFramework framework, CancellationToken token)
         {
-            var directory = Path.Combine(PackageDirectory, package.Id, package.Version.ToNormalizedString());
+            // Get the download resource from the nuget client API. This is basically a DI locator.
+            var downloadResource = source.GetResource<DownloadResource>(token);
 
-            EnsureDirectory(directory);
+            var packagesToCopy = new Dictionary<string, (PackageIdentity packageIdentity, DownloadResourceResult downloadResourceResult, bool includeFilesInOutput)>(StringComparer.InvariantCultureIgnoreCase);
 
-            // Use the provided nuget package source, or use nuget.org
-            var source = new SourceRepository(nugetSource ?? new PackageSource("https://api.nuget.org/v3/index.json"), _providers);
+            var stack = new ConcurrentStack<PackageIdentity>(new[] { startingPackage });
+            var currentItems = new PackageIdentity[32];
+            while (!stack.IsEmpty)
+            {
+                var count = stack.TryPopRange(currentItems);
 
-            // Get the download resource from the nuget client API. This is basically a DI locator.
-            var downloadResource = await source.GetResourceAsync<DownloadResource>(token).ConfigureAwait(false);
+                // Download the resource into the global packages path. We get a result which allows us to copy or do other operations based on the files.
+                (DownloadResourceResult downloadResourceResult, PackageIdentity packageIdentity, bool includeFilesInOutput)[] results = await Task.WhenAll(currentItems.Take(count).Select(async item =>
+                                          (await downloadResource.GetDownloadResourceResultAsync(item, _downloadContext, _globalPackagesPath, _logger, token).ConfigureAwait(false), item, item.Equals(startingPackage))))
+                                          .ConfigureAwait(false);
 
-            // Download the resource into the global packages path. We get a result which allows us to copy or do other operations based on the files.
-            var downloadResults = await downloadResource.GetDownloadResourceResultAsync(
-                                              package,
-                                              _downloadContext,
-                                              _globalPackagesPath,
-                                              _logger,
-                                              token).ConfigureAwait(false);
+                foreach (var result in results.Where(x => x.downloadResourceResult.Status == DownloadResourceResultStatus.Available || x.downloadResourceResult.Status == DownloadResourceResultStatus.AvailableWithoutStream))
+                {
+                    packagesToCopy[result.packageIdentity.Id] = (result.packageIdentity, result.downloadResourceResult, result.includeFilesInOutput);
+                }
 
-            if (downloadResults.Status != DownloadResourceResultStatus.Available && downloadResults.Status != DownloadResourceResultStatus.AvailableWithoutStream)
-            {
-                return default;
+                var dependencies = results.SelectMany(x => GetDependencyPackages(x.downloadResourceResult, framework)
+                    .Where(
+                        dependentPackage =>
+                        {
+                            if (!packagesToCopy.TryGetValue(dependentPackage.Id, out var value))
+                            {
+                                return true;
+                            }
+
+                            return dependentPackage.Version > value.packageIdentity.Version;
+                        })).ToArray();
+
+                if (dependencies.Length > 0)
+                {
+                    stack.PushRange(dependencies);
+                }
             }
 
+            return packagesToCopy.Select(x => (x.Value.packageIdentity, x.Value.downloadResourceResult, x.Value.includeFilesInOutput));
+        }
+
+        private static (string folder, IEnumerable<string> files) CopyPackageFiles(PackageIdentity package, DownloadResourceResult downloadResults, NuGetFramework framework, bool includeFilesInOutput, CancellationToken token)
+        {
+            var directory = Path.Combine(PackageDirectory, package.Id, package.Version.ToNormalizedString());
+
+            EnsureDirectory(directory);
+
             // Get all the folders in our lib and build directory of our nuget. These are the general contents we include in our projects.
             var groups = downloadResults.PackageReader.GetFileGroups(PackagingConstants.Folders.Lib).Concat(
-                              downloadResults.PackageReader.GetFileGroups(PackagingConstants.Folders.Build).Concat(
-                              downloadResults.PackageReader.GetFileGroups(PackagingConstants.Folders.Ref)));
+                          downloadResults.PackageReader.GetFileGroups(PackagingConstants.Folders.Build).Concat(
+                          downloadResults.PackageReader.GetFileGroups(PackagingConstants.Folders.Ref)));
 
             // Select our groups that match our selected framework and have content.
-            var groupFiles = groups.Where(x => !x.HasEmptyFolder && x.TargetFramework == framework).SelectMany(x => x.Items).ToList();
+            var groupFiles = groups.Where(x => !x.HasEmptyFolder && x.TargetFramework.EqualToOrLessThan(framework)).SelectMany(x => x.Items).ToList();
 
             // Extract the files, don't bother copying the XML file contents.
             var packageFileExtractor = new PackageFileExtractor(groupFiles, XmlDocFileSaveMode.Skip);
 
             // Copy the files to our extractor cache directory.
-            var outputFiles = await downloadResults.PackageReader.CopyFilesAsync(directory, groupFiles, packageFileExtractor.ExtractPackageFile, _logger, token).ConfigureAwait(false);
+            var outputFiles = downloadResults.PackageReader.CopyFiles(directory, groupFiles, packageFileExtractor.ExtractPackageFile, _logger, token);
 
             // Return the folder, if we aren't excluding files return all the assemblies.
             return (directory, includeFilesInOutput ? outputFiles.Where(x => x.EndsWith(".dll", StringComparison.OrdinalIgnoreCase)) : Enumerable.Empty<string>());
         }
 
+        private static IEnumerable<PackageIdentity> GetDependencyPackages(DownloadResourceResult downloadResults, NuGetFramework framework)
+        {
+            return downloadResults.PackageReader.GetPackageDependencies()
+                .Where(dependency => dependency.TargetFramework.EqualToOrLessThan(framework))
+                .SelectMany(x => x.Packages.Select(package => new PackageIdentity(package.Id, package.VersionRange.MinVersion)));
+        }
+
         private static void EnsureDirectory(string packageUnzipPath)
         {
             if (!Directory.Exists(packageUnzipPath))
@@ -122,14 +151,23 @@ private static void EnsureDirectory(string packageUnzipPath)
             }
         }
 
+        private static bool EqualToOrLessThan(this NuGetFramework firstFramework, NuGetFramework secondFramework)
+        {
+            if (!NuGetFramework.FrameworkNameComparer.Equals(firstFramework, secondFramework))
+            {
+                return false;
+            }
+
+            return firstFramework.Version <= secondFramework.Version;
+        }
+
         private static IEnumerable<FrameworkSpecificGroup> GetFileGroups(this PackageReaderBase reader, string folder)
         {
-            Dictionary<NuGetFramework, List<string>> groups = new Dictionary<NuGetFramework, List<string>>(new NuGetFrameworkFullComparer());
-            bool allowSubFolders = true;
+            var groups = new Dictionary<NuGetFramework, List<string>>(new NuGetFrameworkFullComparer());
             foreach (string file in reader.GetFiles(folder))
             {
-                NuGetFramework frameworkFromPath = reader.GetFrameworkFromPath(file, allowSubFolders);
-                if (!groups.TryGetValue(frameworkFromPath, out List<string> stringList))
+                var frameworkFromPath = reader.GetFrameworkFromPath(file, true);
+                if (!groups.TryGetValue(frameworkFromPath, out var stringList))
                 {
                     stringList = new List<string>();
                     groups.Add(frameworkFromPath, stringList);
@@ -138,17 +176,17 @@ private static IEnumerable<FrameworkSpecificGroup> GetFileGroups(this PackageRea
                 stringList.Add(file);
             }
 
-            foreach (NuGetFramework targetFramework in groups.Keys.OrderBy(e => e, new NuGetFrameworkSorter()))
+            foreach (var targetFramework in groups.Keys.OrderBy(e => e, new NuGetFrameworkSorter()))
             {
                 yield return new FrameworkSpecificGroup(targetFramework, groups[targetFramework].OrderBy(e => e, StringComparer.OrdinalIgnoreCase));
             }
         }
 
-        private static NuGetFramework GetFrameworkFromPath(this PackageReaderBase reader, string path, bool allowSubFolders = false)
+        private static NuGetFramework GetFrameworkFromPath(this IPackageCoreReader reader, string path, bool allowSubFolders = false)
         {
             var nuGetFramework = NuGetFramework.AnyFramework;
-            string[] strArray = path.Split(new char[1] { '/' }, StringSplitOptions.RemoveEmptyEntries);
-            if (strArray.Length == 3 || strArray.Length > 3 & allowSubFolders)
+            var strArray = path.Split(new char[] { '/' }, StringSplitOptions.RemoveEmptyEntries);
+            if ((strArray.Length == 3 || strArray.Length > 3) && allowSubFolders)
             {
                 string folderName = strArray[1];
                 NuGetFramework folder;
@@ -169,16 +207,5 @@ private static NuGetFramework GetFrameworkFromPath(this PackageReaderBase reader
 
             return nuGetFramework;
         }
-
-        private class NuGetPackageIdentityComparer : IComparer<(PackageIdentity, bool)>
-        {
-            public static NuGetPackageIdentityComparer Default { get; } = new NuGetPackageIdentityComparer();
-
-            /// <inheritdoc />
-            public int Compare((PackageIdentity, bool) x, (PackageIdentity, bool) y)
-            {
-                return x.Item1.CompareTo(y.Item1);
-            }
-        }
     }
 }

src/Pharmacist.Core/Pharmacist.Core.csproj

@@ -9,6 +9,7 @@
   </ItemGroup>
 
   <ItemGroup>
+    <PackageReference Include="ConcurrentHashSet" Version="1.0.2" />
     <PackageReference Include="ICSharpCode.Decompiler" Version="4.0.0.4521" />
     <PackageReference Include="Microsoft.CodeAnalysis.CSharp" Version="3.0.0" />
     <PackageReference Include="nuget.protocol" Version="5.0.0" />

src/Pharmacist.Tests/NuGetPackageHelperTests.cs

@@ -13,6 +13,8 @@
 
 using Pharmacist.Core.NuGet;
 
+using Shouldly;
+
 using Xunit;
 
 namespace Pharmacist.Tests
@@ -146,6 +148,19 @@ public async Task MultipleTizenAttemptsWork()
             await GetAndCheckTizenPackage().ConfigureAwait(false);
         }
 
+        [Fact]
+        public async Task CanGetNuGetProtocolAndDependencies()
+        {
+            var package = new PackageIdentity("NuGet.Protocol", new NuGetVersion("5.0.0"));
+            var framework = FrameworkConstants.CommonFrameworks.NetStandard20;
+
+            var result = (await NuGetPackageHelper
+                              .DownloadPackageAndGetLibFilesAndFolder(package, framework: framework)
+                              .ConfigureAwait(false)).ToList();
+
+            result.ShouldNotBeEmpty();
+        }
+
         private static async Task GetAndCheckTizenPackage()
         {
             var package = new PackageIdentity("Tizen.NET.API4", new NuGetVersion("4.0.1.14152"));