Support indexing mulitple release assets on GitHub

Netkan/CKAN-netkan.csproj

@@ -85,6 +85,7 @@
     <Compile Include="Sources\Github\GithubApi.cs" />
     <Compile Include="Sources\Github\GithubRef.cs" />
     <Compile Include="Sources\Github\GithubRelease.cs" />
+    <Compile Include="Sources\Github\GithubReleaseAsset.cs" />
     <Compile Include="Sources\Github\GithubRepo.cs" />
     <Compile Include="Sources\Github\IGithubApi.cs" />
     <Compile Include="Sources\Jenkins\IJenkinsApi.cs" />

Netkan/Model/Metadata.cs

@@ -1,5 +1,5 @@
 using System;
-using System.Linq;
+using System.Linq;
 using CKAN.Versioning;
 using Newtonsoft.Json.Linq;
 

Netkan/Sources/Github/GithubApi.cs

@@ -2,7 +2,7 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Net;
-using System.Text.RegularExpressions;
+using System.Text.RegularExpressions;
 using log4net;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
@@ -80,31 +80,36 @@ public IEnumerable<GithubRelease> GetAllReleases(GithubRef reference)
                         var version = new ModuleVersion((string)release["tag_name"]);
                         var author = (string)release["author"]["login"];
 
-                        Uri       download = null;
-                        DateTime? updated  = null;
+                        List<GithubReleaseAsset> allAssets;
 
                         if (reference.UseSourceArchive)
                         {
                             Log.Debug("Using GitHub source archive");
-                            download = new Uri((string)release["zipball_url"]);
-                            updated = (DateTime)release["published_at"];
+                            Uri download = new Uri((string)release["zipball_url"]);
+                            DateTime?  updated = (DateTime)release["published_at"];
+                            allAssets = new List<GithubReleaseAsset> { new GithubReleaseAsset(version.ToString(), download, updated) };
                         }
                         else
                         {
+                            allAssets = new List<GithubReleaseAsset>();
                             var assets = (JArray)release["assets"];
 
                             foreach (var asset in assets.Where(asset => reference.Filter.IsMatch((string)asset["name"])))
                             {
                                 Log.DebugFormat("Using GitHub asset: {0}", asset["name"]);
-                                download = new Uri((string)asset["browser_download_url"]);
-                                updated = (DateTime)asset["updated_at"];
-                                break;
+                                Uri download = new Uri((string)asset["browser_download_url"]);
+                                DateTime? updated = (DateTime)asset["updated_at"];
+
+                                allAssets.Add(new GithubReleaseAsset(
+                                        (string)asset["name"], download, updated
+                                    )
+                                );
                             }
                         }
 
-                        if (download != null)
+                        if (allAssets.Any())
                         {
-                            yield return new GithubRelease(author, version, download, updated);
+                            yield return new GithubRelease(author, version, allAssets);
                         }
                     }
                 }
@@ -116,13 +121,13 @@ public IEnumerable<GithubRelease> GetAllReleases(GithubRef reference)
                 }
             }
         }
-        
+
         public List<GithubUser> getOrgMembers(GithubUser organization)
         {
             return JsonConvert.DeserializeObject<List<GithubUser>>(
                 Call($"orgs/{organization.Login}/public_members")
             );
-        }        
+        }
 
         /// <summary>
         /// Download a URL via the GitHubAPI.

Netkan/Sources/Github/GithubRef.cs

@@ -6,14 +6,15 @@ namespace CKAN.NetKAN.Sources.Github
     internal sealed class GithubRef : RemoteRef
     {
         private static readonly Regex Pattern = new Regex(
-            @"^(?<account>[^/]+)/(?<project>[^/]+)(?:/asset_match/(?<filter>.+))?$",
+            @"^(?<account>[^/]+)/(?<project>[^/]+)(?:(/asset_match/(?<filter>.+))|(/version_from_asset/(?<versionFromAsset>.+)))?$",
             RegexOptions.Compiled
         );
 
         public string Account { get; private set; }
         public string Project { get; private set; }
         public string Repository { get; private set; }
         public Regex Filter { get; private set; }
+        public Regex VersionFromAsset { get; private set; }
         public bool UseSourceArchive { get; private set; }
         public bool UsePrerelease { get; private set; }
 
@@ -35,6 +36,10 @@ public GithubRef(RemoteRef remoteRef, bool useSourceArchive, bool usePrerelease)
                     new Regex(match.Groups["filter"].Value, RegexOptions.Compiled) :
                     Constants.DefaultAssetMatchPattern;
 
+                VersionFromAsset = match.Groups["versionFromAsset"].Success ?
+                    new Regex(match.Groups["versionFromAsset"].Value, RegexOptions.Compiled) :
+                    null;
+
                 UseSourceArchive = useSourceArchive;
                 UsePrerelease = usePrerelease;
             }

Netkan/Sources/Github/GithubRelease.cs

@@ -1,21 +1,20 @@
 using System;
+using System.Collections.Generic;
 using CKAN.Versioning;
 
 namespace CKAN.NetKAN.Sources.Github
 {
     public sealed class GithubRelease
     {
-        public string           Author       { get; }
-        public ModuleVersion    Version      { get; }
-        public Uri              Download     { get; }
-        public DateTime?        AssetUpdated { get; }
+        public string        Author { get; }
+        public ModuleVersion Tag    { get; }
+        public List<GithubReleaseAsset> Assets { get; }
 
-        public GithubRelease(string author, ModuleVersion version, Uri download, DateTime? updated)
+        public GithubRelease(string author, ModuleVersion tag, List<GithubReleaseAsset> assets)
         {
             Author       = author;
-            Version      = version;
-            Download     = download;
-            AssetUpdated = updated;
+            Tag          = tag;
+            Assets       = assets;
         }
     }
 }

Netkan/Sources/Github/GithubReleaseAsset.cs

@@ -0,0 +1,20 @@
+using System;
+using System.Collections.Generic;
+using CKAN.Versioning;
+
+namespace CKAN.NetKAN.Sources.Github
+{
+    public sealed class GithubReleaseAsset
+    {
+        public String        Name     { get; }
+        public Uri           Download { get; }
+        public DateTime?     Updated  { get; }
+
+        public GithubReleaseAsset(string name, Uri download, DateTime? updated)
+        {
+            Name     = name;
+            Download = download;
+            Updated  = updated;
+        }
+    }
+}

Netkan/Transformers/GithubTransformer.cs

@@ -1,11 +1,13 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Text.RegularExpressions;
 using log4net;
 using Newtonsoft.Json.Linq;
 using CKAN.NetKAN.Extensions;
 using CKAN.NetKAN.Model;
 using CKAN.NetKAN.Sources.Github;
+using CKAN.Versioning;
 
 namespace CKAN.NetKAN.Transformers
 {
@@ -72,12 +74,43 @@ public IEnumerable<Metadata> Transform(Metadata metadata, TransformOptions opts)
                 bool returnedAny = false;
                 foreach (GithubRelease rel in versions)
                 {
-                    returnedAny = true;
-                    yield return TransformOne(metadata, metadata.Json(), ghRef, ghRepo, rel);
+                    if (ghRef.VersionFromAsset != null)
+                    {
+                        Log.DebugFormat("Found version_from_asset regex, inflating all assets");
+                        foreach (var asset in rel.Assets)
+                        {
+                            var match = ghRef.VersionFromAsset.Match(asset.Name);
+                            if (!match.Success)
+                                continue;
+
+                            var extractedVersion = match.Groups["version"];
+                            if (!extractedVersion.Success)
+                                throw new Exception("version_from_asset contains no 'version' capturing group");
+
+                            returnedAny = true;
+                            yield return TransformOne(metadata, metadata.Json(), ghRef, ghRepo, rel, asset, extractedVersion.Value);
+                        }
+                    }
+                    else
+                    {
+                        returnedAny = true;
+                        yield return TransformOne(metadata, metadata.Json(), ghRef, ghRepo, rel, rel.Assets.FirstOrDefault(), rel.Tag.ToString());
+                    }
                 }
                 if (!returnedAny)
                 {
-                    Log.WarnFormat("No releases found for {0}", ghRef.Repository);
+                    if (ghRef.Filter != Constants.DefaultAssetMatchPattern)
+                    {
+                        Log.WarnFormat("No releases found for {0} with asset_match {1}", ghRef.Repository, ghRef.Filter);
+                    }
+                    else if (ghRef.VersionFromAsset != null)
+                    {
+                        Log.WarnFormat("No releases found for {0} with version_from_asset {1}", ghRef.Repository, ghRef.VersionFromAsset);
+                    }
+                    else
+                    {
+                        Log.WarnFormat("No releases found for {0}", ghRef.Repository);
+                    }
                     yield return metadata;
                 }
             }
@@ -87,7 +120,10 @@ public IEnumerable<Metadata> Transform(Metadata metadata, TransformOptions opts)
             }
         }
 
-        private Metadata TransformOne(Metadata metadata, JObject json, GithubRef ghRef, GithubRepo ghRepo, GithubRelease ghRelease)
+        private Metadata TransformOne(
+            Metadata metadata, JObject json, GithubRef ghRef, GithubRepo ghRepo, GithubRelease ghRelease,
+            GithubReleaseAsset ghAsset, String version
+        )
         {
             if (!string.IsNullOrWhiteSpace(ghRepo.Description))
                 json.SafeAdd("abstract", ghRepo.Description);
@@ -114,11 +150,11 @@ private Metadata TransformOne(Metadata metadata, JObject json, GithubRef ghRef,
 
             if (ghRelease != null)
             {
-                json.SafeAdd("version",  ghRelease.Version.ToString());
+                json.SafeAdd("version",  version);
                 json.SafeAdd("author",   () => getAuthors(ghRepo, ghRelease));
                 json.Remove("$kref");
-                json.SafeAdd("download", ghRelease.Download.ToString());
-                json.SafeAdd(Metadata.UpdatedPropertyName, ghRelease.AssetUpdated);
+                json.SafeAdd("download", ghAsset.Download.ToString());
+                json.SafeAdd(Metadata.UpdatedPropertyName, ghAsset.Updated);
 
                 if (ghRef.Project.Contains("_"))
                 {
@@ -146,6 +182,11 @@ private Metadata TransformOne(Metadata metadata, JObject json, GithubRef ghRef,
                     json.SafeAdd("name", repoName);
                 }
 
+                json.SafeMerge(
+                    "x_netkan_version_pieces",
+                    JObject.FromObject(new Dictionary<string, string>{ {"tag", ghRelease.Tag.ToString()} })
+                );
+
                 Log.DebugFormat("Transformed metadata:{0}{1}", Environment.NewLine, json);
 
                 return new Metadata(json);

Netkan/Transformers/VersionEditTransformer.cs

@@ -1,9 +1,11 @@
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using System.Text.RegularExpressions;
 using log4net;
 using Newtonsoft.Json.Linq;
 using CKAN.NetKAN.Model;
+using Newtonsoft.Json;
 
 namespace CKAN.NetKAN.Transformers
 {
@@ -29,8 +31,25 @@ public IEnumerable<Metadata> Transform(Metadata metadata, TransformOptions opts)
                 var findRegex = new Regex(versionEditInfo.Find);
                 if (findRegex.IsMatch(versionEditInfo.Version))
                 {
-                    json["version"] = new Regex(versionEditInfo.Find)
+                    string version = new Regex(versionEditInfo.Find)
                         .Replace(versionEditInfo.Version, versionEditInfo.Replace);
+
+                    var versionPieces = json["x_netkan_version_pieces"]?.Value<JObject>()
+                        .ToObject<Dictionary<string, string>>();
+
+                    if (versionPieces != null)
+                    {
+                        version = versionPieces.Aggregate(
+                            version,
+                            (v, kvp) =>
+                            {
+                                Log.Debug($"Replacing ${{{kvp.Key}}} with {kvp.Value} in {v}");
+                                return v.Replace($"${{{kvp.Key}}}", kvp.Value);
+                            });
+                    }
+
+
+                    json["version"] = version;
                 }
                 else if (versionEditInfo.Strict)
                 {

Spec.md

@@ -731,7 +731,7 @@ When used, the following fields will be auto-filled if not already present:
 - `resources.curse`
 - `ksp_version`
 
-###### `#/ckan/github/:user/:repo[/asset_match/:filter_regexp]`
+###### `#/ckan/github/:user/:repo[(/asset_match/:filter_regexp)|(/version_from_asset/:version_regexp)]`
 
 Indicates that data should be fetched from GitHub, using the `:user` and `:repo` provided.
 For example: `#/ckan/github/pjf/DogeCoinFlag`.
@@ -749,10 +749,21 @@ When used, the following fields will be auto-filled if not already present:
 - `download_content_type`
 - `resources.repository`
 
-Optionally, one asset `:filter_regexp` directive *may* be provided:
+Optionally, one of `asset_match` with `:filter_regexp` *or* `version_from_asset` with `:version_regexp` *may* be provided:
 
-- `filter_regexp`: A string which is treated as  case-sensitive C# regular expressions which are matched against the
+- `asset_match` with `filter_regexp`: A string which is treated as case-sensitive C# regular expressions which are matched against the
   name of the released artifact.
+- `version_from_asset` with `:version_regexp`: A string which is treated as case-sensitive C# regular expressions which are matched
+  against the names of all release artifacts. Every matching artifact will result in a separate metadata output. The `:version_regexp`
+  must have a named capturing group `version`, which is used as the `version` of each asset's module.
+
+An example `.netkan` excerpt:
+
+```json
+{
+    "$kref": "#/ckan/github/pjf/DogeCoinFlag/version_from_asset/^DogeCoinFlag-(?<version>.+).zip$"
+}
+```
 
 An `x_netkan_github` field may be provided to customize how the metadata is fetched from GitHub. It is an `object` with the following fields:
 
@@ -954,6 +965,8 @@ an `object` with the following fields:
 - `replace` (type: `string`, regex substitution) (default: `"${version}"`)<br/>
   Specifies a [regex substitution string](https://msdn.microsoft.com/en-us/library/ewy2t5e0%28v=vs.110%29.aspx) which
   will be used as the value of the new `version` field.
+  The following special variables are supported and will be filled in by various metadata aggregated during inflation:
+  - `${tag}` (`$kref #/ckan/github` only): Replaced with the release tag from GitHub.
 - `strict` (type: `boolean`, default: `true`)<br/>
   Specifies if NetKAN should produce an error if `find` fails to produce a match against the `version` field.
 

Tests/NetKAN/Sources/Github/GithubApiTests.cs

@@ -1,5 +1,6 @@
 using System;
 using System.IO;
+using System.Linq;
 using NUnit.Framework;
 using CKAN;
 using CKAN.NetKAN.Services;
@@ -48,8 +49,9 @@ public void GetsLatestReleaseCorrectly()
 
             // Assert
             Assert.IsNotNull(githubRelease.Author);
-            Assert.IsNotNull(githubRelease.Download);
-            Assert.IsNotNull(githubRelease.Version);
+            Assert.IsNotNull(githubRelease.Tag);
+            Assert.IsNotNull(githubRelease.Assets.FirstOrDefault());
+            Assert.IsNotNull(githubRelease.Assets.FirstOrDefault()?.Download);
         }
     }
 }