Merge #3060 Support fallback URLs for netkan validate-ckan

KSP-CKAN · May 24, 2020 · 6273ca5 · 6273ca5
2 parents c451a20 + 5532e85
commit 6273ca5
Show file tree

Hide file tree

Showing 14 changed files with 99 additions and 30 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -24,6 +24,7 @@ All notable changes to this project will be documented in this file.
 
 - [Netkan] Purge stale cache entries for SpaceDock (#2859 by: HebaruSan; reviewed: DasSkelett)
 - [Netkan] NetKAN warnings (#3045 by: HebaruSan; reviewed: DasSkelett)
+- [Netkan] Support fallback URLs for netkan validate-ckan (#3060 by: HebaruSan; reviewed: DasSkelett)
 
 ## v1.27.2 (Chandrasekhar)
 

diff --git a/Netkan/Model/Metadata.cs b/Netkan/Model/Metadata.cs
@@ -1,4 +1,5 @@
 using System;
+using System.Linq;
 using CKAN.Versioning;
 using Newtonsoft.Json.Linq;
 
@@ -101,7 +102,7 @@ public Metadata(JObject json)
             {
                 Staged = (bool)stagedToken;
             }
-            
+
             JToken stagingReasonToken;
             if (json.TryGetValue(StagingReasonPropertyName, out stagingReasonToken))
             {
@@ -116,6 +117,58 @@ public Metadata(JObject json)
                 RemoteTimestamp = t;
             }
         }
+
+        public string[] Licenses
+        {
+            get
+            {
+                var lic = _json["license"];
+                switch (lic.Type)
+                {
+                    case JTokenType.Array:
+                        return lic.Children()
+                            .Select(t => (string)t)
+                            .ToArray();
+
+                    case JTokenType.String:
+                        return new string[] { (string)lic };
+                }
+                return new string[] { };
+            }
+        }
+
+        public bool Redistributable
+        {
+            get
+            {
+                return Licenses.Any(lic => new License(lic).Redistributable);
+            }
+        }
+
+        public Uri FallbackDownload
+        {
+            get
+            {
+                if (Identifier == null || Version == null || !Redistributable)
+                {
+                    return null;
+                }
+                string verStr = Version.ToString().Replace(':', '-');
+                var hashes = (JObject)_json["download_hash"];
+                if (hashes == null)
+                {
+                    return null;
+                }
+                var sha1 = (string)hashes["sha1"];
+                if (sha1 == null)
+                {
+                    return null;
+                }
+                return new Uri(
+                    $"https://archive.org/download/{Identifier}-{verStr}/{sha1.Substring(0, 8)}-{Identifier}-{verStr}.zip"
+                );
+            }
+        }
 
         public JObject Json()
         {

diff --git a/Netkan/Services/CachingHttpService.cs b/Netkan/Services/CachingHttpService.cs
@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using log4net;
+using CKAN.NetKAN.Model;
 
 namespace CKAN.NetKAN.Services
 {
@@ -20,8 +21,33 @@ public CachingHttpService(NetFileCache cache, bool overwrite = false)
             _overwriteCache = overwrite;
         }
 
-        public string DownloadPackage(Uri url, string identifier, DateTime? updated)
+        public string DownloadModule(Metadata metadata)
         {
+            try
+            {
+                return DownloadPackage(metadata.Download, metadata.Identifier, metadata.RemoteTimestamp);
+            }
+            catch (Exception exc)
+            {
+                var fallback = metadata.FallbackDownload;
+                if (fallback == null)
+                {
+                    throw;
+                }
+                else
+                {
+                    log.InfoFormat("Trying fallback URL: {0}", fallback);
+                    return DownloadPackage(fallback, metadata.Identifier, metadata.RemoteTimestamp, metadata.Download);
+                }
+            }
+        }
+
+        private string DownloadPackage(Uri url, string identifier, DateTime? updated, Uri primaryUrl = null)
+        {
+            if (primaryUrl == null)
+            {
+                primaryUrl = url;
+            }
             if (_overwriteCache && !_requestedURLs.Contains(url))
             {
                 // Discard cached file if command line says so,
@@ -31,7 +57,7 @@ public string DownloadPackage(Uri url, string identifier, DateTime? updated)
 
             _requestedURLs.Add(url);
 
-            var cachedFile = _cache.GetCachedFilename(url, updated);
+            var cachedFile = _cache.GetCachedFilename(primaryUrl, updated);
 
             if (!string.IsNullOrWhiteSpace(cachedFile))
             {
@@ -72,7 +98,7 @@ public string DownloadPackage(Uri url, string identifier, DateTime? updated)
                 }
 
                 return _cache.Store(
-                    url,
+                    primaryUrl,
                     downloadedFile,
                     string.Format("netkan-{0}.{1}", identifier, extension),
                     move: true

diff --git a/Netkan/Services/IHttpService.cs b/Netkan/Services/IHttpService.cs
@@ -1,11 +1,12 @@
 using System;
 using System.Collections.Generic;
+using CKAN.NetKAN.Model;
 
 namespace CKAN.NetKAN.Services
 {
     internal interface IHttpService
     {
-        string DownloadPackage(Uri url, string identifier, DateTime? updated);
+        string DownloadModule(Metadata metadata);
         string DownloadText(Uri url);
         string DownloadText(Uri url, string authToken, string mimeType);
 

diff --git a/Netkan/Transformers/AvcTransformer.cs b/Netkan/Transformers/AvcTransformer.cs
@@ -62,7 +62,7 @@ public IEnumerable<Metadata> Transform(Metadata metadata, TransformOptions opts)
                     json.Remove("version");
                 }
 
-                var file = _http.DownloadPackage(metadata.Download, metadata.Identifier, metadata.RemoteTimestamp);
+                var file = _http.DownloadModule(metadata);
                 var avc = _moduleService.GetInternalAvc(mod, file, metadata.Vref.Id);
 
                 if (avc != null)

diff --git a/Netkan/Transformers/DownloadAttributeTransformer.cs b/Netkan/Transformers/DownloadAttributeTransformer.cs
@@ -35,7 +35,7 @@ public IEnumerable<Metadata> Transform(Metadata metadata, TransformOptions opts)
                 Log.InfoFormat("Executing Download attribute transformation with {0}", metadata.Kref);
                 Log.DebugFormat("Input metadata:{0}{1}", Environment.NewLine, json);
 
-                string file = _http.DownloadPackage(metadata.Download, metadata.Identifier, metadata.RemoteTimestamp);
+                string file = _http.DownloadModule(metadata);
 
                 if (file != null)
                 {

diff --git a/Netkan/Transformers/InternalCkanTransformer.cs b/Netkan/Transformers/InternalCkanTransformer.cs
@@ -32,7 +32,7 @@ public IEnumerable<Metadata> Transform(Metadata metadata, TransformOptions opts)
             {
                 var json = metadata.Json();
 
-                string file = _http.DownloadPackage(metadata.Download, metadata.Identifier, metadata.RemoteTimestamp);
+                string file = _http.DownloadModule(metadata);
 
                 var internalJson = _moduleService.GetInternalCkan(file);
 

diff --git a/Netkan/Transformers/LocalizationsTransformer.cs b/Netkan/Transformers/LocalizationsTransformer.cs
@@ -50,11 +50,7 @@ public IEnumerable<Metadata> Transform(Metadata metadata, TransformOptions opts)
             else
             {
                 CkanModule mod  = CkanModule.FromJson(json.ToString());
-                ZipFile    zip  = new ZipFile(_http.DownloadPackage(
-                    metadata.Download,
-                    metadata.Identifier,
-                    metadata.RemoteTimestamp
-                ));
+                ZipFile    zip  = new ZipFile(_http.DownloadModule(metadata));
 
                 log.Debug("Extracting locales");
                 // Extract the locale names from the ZIP's cfg files

diff --git a/Netkan/Validators/InstallsFilesValidator.cs b/Netkan/Validators/InstallsFilesValidator.cs
@@ -20,7 +20,7 @@ public void Validate(Metadata metadata)
             var mod = CkanModule.FromJson(metadata.Json().ToString());
             if (!mod.IsDLC)
             {
-                var file = _http.DownloadPackage(metadata.Download, metadata.Identifier, metadata.RemoteTimestamp);
+                var file = _http.DownloadModule(metadata);
 
                 // Make sure this would actually generate an install.
                 if (!_moduleService.HasInstallableFiles(mod, file))

diff --git a/Netkan/Validators/ModuleManagerDependsValidator.cs b/Netkan/Validators/ModuleManagerDependsValidator.cs
@@ -25,11 +25,7 @@ public void Validate(Metadata metadata)
             CkanModule mod  = CkanModule.FromJson(json.ToString());
             if (!mod.IsDLC)
             {
-                var package = _http.DownloadPackage(
-                    metadata.Download,
-                    metadata.Identifier,
-                    metadata.RemoteTimestamp
-                );
+                var package = _http.DownloadModule(metadata);
                 if (!string.IsNullOrEmpty(package))
                 {
                     ZipFile zip  = new ZipFile(package);

diff --git a/Netkan/Validators/PluginCompatibilityValidator.cs b/Netkan/Validators/PluginCompatibilityValidator.cs
@@ -25,11 +25,7 @@ public void Validate(Metadata metadata)
             CkanModule mod  = CkanModule.FromJson(json.ToString());
             if (!mod.IsDLC)
             {
-                var package = _http.DownloadPackage(
-                    metadata.Download,
-                    metadata.Identifier,
-                    metadata.RemoteTimestamp
-                );
+                var package = _http.DownloadModule(metadata);
                 if (!string.IsNullOrEmpty(package))
                 {
                     ZipFile zip  = new ZipFile(package);

diff --git a/Netkan/Validators/VrefValidator.cs b/Netkan/Validators/VrefValidator.cs
@@ -31,7 +31,7 @@ public void Validate(Metadata metadata)
 
             if (!mod.IsDLC)
             {
-                var file = _http.DownloadPackage(metadata.Download, metadata.Identifier, metadata.RemoteTimestamp);
+                var file = _http.DownloadModule(metadata);
                 if (!string.IsNullOrEmpty(file))
                 {
                     bool hasVref = (metadata.Vref != null);

diff --git a/Tests/NetKAN/Transformers/DownloadAttributeTransformerTests.cs b/Tests/NetKAN/Transformers/DownloadAttributeTransformerTests.cs
@@ -28,7 +28,7 @@ public void AddsDownloadAttributes()
             var mHttp = new Mock<IHttpService>();
             var mFileService = new Mock<IFileService>();
 
-            mHttp.Setup(i => i.DownloadPackage(It.IsAny<Uri>(), It.IsAny<string>(), It.IsAny<DateTime?>()))
+            mHttp.Setup(i => i.DownloadModule(It.IsAny<Metadata>()))
                 .Returns(downloadFilePath);
 
             mFileService.Setup(i => i.GetFileHashSha1(downloadFilePath))
@@ -75,7 +75,7 @@ public void DoesNothingIfFileDoesNotExist()
             var mHttp = new Mock<IHttpService>();
             var mFileService = new Mock<IFileService>();
 
-            mHttp.Setup(i => i.DownloadPackage(It.IsAny<Uri>(), It.IsAny<string>(), It.IsAny<DateTime?>()))
+            mHttp.Setup(i => i.DownloadModule(It.IsAny<Metadata>()))
                 .Returns((string)null);
 
             var sut = new DownloadAttributeTransformer(mHttp.Object, mFileService.Object);

diff --git a/Tests/NetKAN/Transformers/InternalCkanTransformerTests.cs b/Tests/NetKAN/Transformers/InternalCkanTransformerTests.cs
@@ -27,7 +27,7 @@ public void AddsMiddingProperties()
             var mHttp = new Mock<IHttpService>();
             var mModuleService = new Mock<IModuleService>();
 
-            mHttp.Setup(i => i.DownloadPackage(It.IsAny<Uri>(), It.IsAny<string>(), It.IsAny<DateTime?>()))
+            mHttp.Setup(i => i.DownloadModule(It.IsAny<Metadata>()))
                 .Returns(filePath);
 
             mModuleService.Setup(i => i.GetInternalCkan(filePath))
@@ -62,7 +62,7 @@ public void DoesNotOverrideExistingProperties()
             var mHttp = new Mock<IHttpService>();
             var mModuleService = new Mock<IModuleService>();
 
-            mHttp.Setup(i => i.DownloadPackage(It.IsAny<Uri>(), It.IsAny<string>(), It.IsAny<DateTime?>()))
+            mHttp.Setup(i => i.DownloadModule(It.IsAny<Metadata>()))
                 .Returns(filePath);
 
             mModuleService.Setup(i => i.GetInternalCkan(filePath))
@@ -100,7 +100,7 @@ public void HigherOfTwoSpecVersionsIsChosen(
             var mHttp = new Mock<IHttpService>();
             var mModuleService = new Mock<IModuleService>();
 
-            mHttp.Setup(i => i.DownloadPackage(It.IsAny<Uri>(), It.IsAny<string>(), It.IsAny<DateTime?>()))
+            mHttp.Setup(i => i.DownloadModule(It.IsAny<Metadata>()))
                 .Returns(filePath);
 
             mModuleService.Setup(i => i.GetInternalCkan(filePath))