From 4789ef5a36829fcce1210e7d39cdbe6f4a03116a Mon Sep 17 00:00:00 2001 From: Leon Wright Date: Tue, 3 May 2016 13:46:15 +0800 Subject: [PATCH 01/13] Add download hash to Metadata - #1682 --- Netkan/CKAN-netkan.csproj | 3 + Netkan/Program.cs | 2 + Netkan/Services/FileHash.cs | 21 +++++ Netkan/Services/IFileHash.cs | 7 ++ .../Transformers/DownloadHashTransformer.cs | 50 ++++++++++ Netkan/Transformers/NetkanTransformer.cs | 2 + Tests/NetKAN/Services/FileHashTests.cs | 25 +++++ .../DownloadHashTransformerTests.cs | 94 +++++++++++++++++++ Tests/Tests.csproj | 2 + 9 files changed, 206 insertions(+) create mode 100644 Netkan/Services/FileHash.cs create mode 100644 Netkan/Services/IFileHash.cs create mode 100644 Netkan/Transformers/DownloadHashTransformer.cs create mode 100644 Tests/NetKAN/Services/FileHashTests.cs create mode 100644 Tests/NetKAN/Transformers/DownloadHashTransformerTests.cs diff --git a/Netkan/CKAN-netkan.csproj b/Netkan/CKAN-netkan.csproj index 2265d35fec..32fcda2025 100644 --- a/Netkan/CKAN-netkan.csproj +++ b/Netkan/CKAN-netkan.csproj @@ -38,6 +38,8 @@ + + @@ -61,6 +63,7 @@ + diff --git a/Netkan/Program.cs b/Netkan/Program.cs index 931085db0d..8863b4351c 100644 --- a/Netkan/Program.cs +++ b/Netkan/Program.cs @@ -46,6 +46,7 @@ public static int Main(string[] args) var moduleService = new ModuleService(); var fileService = new FileService(); + var fileHash = new FileHash(); var http = new CachingHttpService(FindCache(new KSPManager(new ConsoleUser(false)))); var netkan = ReadNetkan(); @@ -57,6 +58,7 @@ public static int Main(string[] args) var transformer = new NetkanTransformer( http, fileService, + fileHash, moduleService, Options.GitHubToken, Options.PreRelease diff --git a/Netkan/Services/FileHash.cs b/Netkan/Services/FileHash.cs new file mode 100644 index 0000000000..63eabdc264 --- /dev/null +++ b/Netkan/Services/FileHash.cs @@ -0,0 +1,21 @@ +using System; +using System.IO; +using System.Security.Cryptography; + +namespace CKAN.NetKAN.Services +{ + internal sealed class FileHash : IFileHash + { + public string GetFileHash(string filePath) + { + using (FileStream fs = new FileStream(@filePath, FileMode.Open)) + using (BufferedStream bs = new BufferedStream(fs)) + using (var sha1 = new SHA1Managed()) + { + byte[] hash = sha1.ComputeHash(bs); + + return BitConverter.ToString(hash).Replace("-", ""); + } + } + } +} diff --git a/Netkan/Services/IFileHash.cs b/Netkan/Services/IFileHash.cs new file mode 100644 index 0000000000..fb54df25aa --- /dev/null +++ b/Netkan/Services/IFileHash.cs @@ -0,0 +1,7 @@ +namespace CKAN.NetKAN.Services +{ + internal interface IFileHash + { + string GetFileHash(string filePath); + } +} diff --git a/Netkan/Transformers/DownloadHashTransformer.cs b/Netkan/Transformers/DownloadHashTransformer.cs new file mode 100644 index 0000000000..f2dd6fe89d --- /dev/null +++ b/Netkan/Transformers/DownloadHashTransformer.cs @@ -0,0 +1,50 @@ +using System; +using CKAN.NetKAN.Model; +using CKAN.NetKAN.Services; +using log4net; + +namespace CKAN.NetKAN.Transformers +{ + /// + /// An that adds a hash of the downloaded file. + /// + internal sealed class DownloadHashTransformer : ITransformer + { + private static readonly ILog Log = LogManager.GetLogger(typeof(DownloadHashTransformer)); + + private readonly IHttpService _http; + private readonly IFileHash _fileHash; + + public string Name { get { return "download_hash"; } } + + public DownloadHashTransformer(IHttpService http, IFileHash fileHash) + { + _http = http; + _fileHash = fileHash; + } + + public Metadata Transform(Metadata metadata) + { + if (metadata.Download != null) + { + var json = metadata.Json(); + + Log.InfoFormat("Executing Download Hash transformation with {0}", metadata.Kref); + Log.DebugFormat("Input metadata:{0}{1}", Environment.NewLine, json); + + var file = _http.DownloadPackage(metadata.Download, metadata.Identifier); + + if (file != null) + { + json["download_hash"] = _fileHash.GetFileHash(file); + } + + Log.DebugFormat("Transformed metadata:{0}{1}", Environment.NewLine, json); + + return new Metadata(json); + } + + return metadata; + } + } +} diff --git a/Netkan/Transformers/NetkanTransformer.cs b/Netkan/Transformers/NetkanTransformer.cs index 32a684acd5..de2acb5c4f 100644 --- a/Netkan/Transformers/NetkanTransformer.cs +++ b/Netkan/Transformers/NetkanTransformer.cs @@ -19,6 +19,7 @@ internal sealed class NetkanTransformer : ITransformer public NetkanTransformer( IHttpService http, IFileService fileService, + IFileHash fileHash, IModuleService moduleService, string githubToken, bool prerelease @@ -40,6 +41,7 @@ bool prerelease // specify a before or after property. new VersionedOverrideTransformer(before: new string[] { null }, after: new string[] { null }), new DownloadSizeTransformer(http, fileService), + new DownloadHashTransformer(http, fileHash), new GeneratedByTransformer(), new OptimusPrimeTransformer(), new StripNetkanMetadataTransformer(), diff --git a/Tests/NetKAN/Services/FileHashTests.cs b/Tests/NetKAN/Services/FileHashTests.cs new file mode 100644 index 0000000000..eda3e0c45a --- /dev/null +++ b/Tests/NetKAN/Services/FileHashTests.cs @@ -0,0 +1,25 @@ +using CKAN.NetKAN.Services; +using NUnit.Framework; +using Tests.Data; + +namespace Tests.NetKAN.Services +{ + [TestFixture] + public sealed class FileHashTests + { + [Test] + public void GetsFileHashCorrectly() + { + // Arrange + var sut = new FileHash(); + + // Act + var result = sut.GetFileHash(TestData.DogeCoinFlagZip()); + + // Assert + Assert.That(result, Is.EqualTo("47B6ED5F502AD914744882858345BE030A29E1AA"), + "FileService should return the correct file hash." + ); + } + } +} diff --git a/Tests/NetKAN/Transformers/DownloadHashTransformerTests.cs b/Tests/NetKAN/Transformers/DownloadHashTransformerTests.cs new file mode 100644 index 0000000000..57ae0b0096 --- /dev/null +++ b/Tests/NetKAN/Transformers/DownloadHashTransformerTests.cs @@ -0,0 +1,94 @@ +using System; +using CKAN.NetKAN.Model; +using CKAN.NetKAN.Services; +using CKAN.NetKAN.Transformers; +using Moq; +using Newtonsoft.Json.Linq; +using NUnit.Framework; + +namespace Tests.NetKAN.Transformers +{ + [TestFixture] + public sealed class DownloadHashTransformerTests + { + [Test] + public void AddsDownloadHash() + { + // Arrange + const string downloadFilePath = "/DoesNotExist.zip"; + const string downloadHash = "47B6ED5F502AD914744882858345BE030A29E1AA"; + + var mHttp = new Mock(); + var mFileHash = new Mock(); + + mHttp.Setup(i => i.DownloadPackage(It.IsAny(), It.IsAny())) + .Returns(downloadFilePath); + + mFileHash.Setup(i => i.GetFileHash(downloadFilePath)) + .Returns(downloadHash); + + var sut = new DownloadHashTransformer(mHttp.Object, mFileHash.Object); + + var json = new JObject(); + json["spec_version"] = 1; + json["download"] = "https://awesomemod.example/AwesomeMod.zip"; + + // Act + var result = sut.Transform(new Metadata(json)); + var transformedJson = result.Json(); + + // Assert + Assert.That((string)transformedJson["download_hash"], Is.EqualTo(downloadHash), + "DownloadHashTransformer should add a download_hash property equal to the calculated hash of the file." + ); + } + + [Test] + public void DoesNothingIfFileDoesNotExist() + { + // Arrange + var mHttp = new Mock(); + var mFileHash = new Mock(); + + mHttp.Setup(i => i.DownloadPackage(It.IsAny(), It.IsAny())) + .Returns((string)null); + + var sut = new DownloadHashTransformer(mHttp.Object, mFileHash.Object); + + var json = new JObject(); + json["spec_version"] = 1; + json["download"] = "https://awesomemod.example/AwesomeMod.zip"; + + // Act + var result = sut.Transform(new Metadata(json)); + var transformedJson = result.Json(); + + // Assert + Assert.That(transformedJson, Is.EqualTo(json), + "DownloadHashTransformer should do nothing if the file does not exist." + ); + } + + [Test] + public void DoesNothingIfDownloadDoesNotExist() + { + // Arrange + var mHttp = new Mock(); + var mFileHash = new Mock(); + + var sut = new DownloadHashTransformer(mHttp.Object, mFileHash.Object); + + var json = new JObject(); + json["spec_version"] = 1; + + // Act + var result = sut.Transform(new Metadata(json)); + var transformedJson = result.Json(); + + // Assert + Assert.That(transformedJson, Is.EqualTo(json), + "DownloadHashTransformer should do nothing if the download property does not exist." + ); + } + } +} diff --git a/Tests/Tests.csproj b/Tests/Tests.csproj index 3eec213fd1..cecf20fb99 100644 --- a/Tests/Tests.csproj +++ b/Tests/Tests.csproj @@ -144,11 +144,13 @@ + + From 581b0a552476927e440abd000830d3bd564a0ddb Mon Sep 17 00:00:00 2001 From: Leon Wright Date: Tue, 3 May 2016 14:02:45 +0800 Subject: [PATCH 02/13] Add change set to CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f52626e4c6..38869b0041 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ All notable changes to this project will be documented in this file. - [NetKAN] Allow specifying when an override is executed (#1684 by: dbent; fixes: #1674) - [NetKAN] Redirects to the download file are now resolved when using HTTP $krefs (#1696 by: dbent, reviewed: techman83) - [NetKAN] Remote AVC files will be used in preference to ones stored in the archive if they have the same version (#1701 by: dbent, reviewed: techman83) +- [NetKAN] Add download hash to Metadata (#1703 by: techman83; addresses: #1682) ## v1.16.1 From 2ab29bd9b4d5ae20a97737497ec0ee13d69c4d08 Mon Sep 17 00:00:00 2001 From: Leon Wright Date: Tue, 3 May 2016 14:07:02 +0800 Subject: [PATCH 03/13] Add download_hash to Spec documentation. --- Spec.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Spec.md b/Spec.md index e0d20c4033..c9b144d36f 100644 --- a/Spec.md +++ b/Spec.md @@ -509,6 +509,13 @@ downloading from the `download` URL. It is recommended that this field is only generated by automated tools (where it is encouraged), and not filled in by hand. +##### download_hash + +If supplied, `download_hash` is the SHA1 calculated hash to expect of +the resulting file downloaded. It is recommended that this field is +only generated by automated tools (where it is encouraged), +and not filled in by hand. + #### Extensions Any field starting with `x_` (an x, followed by an underscore) is considered @@ -547,6 +554,7 @@ When used, the following fields will be auto-filled if not already present: - `version` - `download` - `download_size` +- `download_hash` - `resources.homepage` - `resources.spacedock` - `resources.repository` @@ -564,6 +572,7 @@ When used, the following fields will be auto-filled if not already present: - `version` - `download` - `download_size` +- `download_hash` - `resources.repository` Optionally, one asset `:filter_regexp` directive *may* be provided: @@ -581,6 +590,7 @@ The following fields will be auto-filled if not already present: - `version` - `download` - `download_size` +- `download_hash` - `resources.ci` An `x_netkan_jenkins` field may be provided to customize how the metadata is fetched from the Jenkins server. It is @@ -621,6 +631,7 @@ When used, the following fields will be auto-filled if not already present: - `download` - `download_size` +- `download_hash` This method depends on the existence of an AVC `.version` file in the download file to determine: @@ -757,6 +768,7 @@ The possible values of `before` and `after` are: - `$all` - `avc` - `download_size` +- `download_hash` - `epoch` - `forced_v` - `generated_by` From fe78ea6adc695825cf73c15bcb06a1b7445ba3d5 Mon Sep 17 00:00:00 2001 From: Leon Wright Date: Tue, 3 May 2016 14:10:00 +0800 Subject: [PATCH 04/13] Add download hash to schema. --- CKAN.schema | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CKAN.schema b/CKAN.schema index 9f592f3843..14b338cf00 100644 --- a/CKAN.schema +++ b/CKAN.schema @@ -64,6 +64,10 @@ "description" : "The size of the download in bytes", "type" : "integer" }, + "download_hash" : { + "description" : "The hash of the downloaded file", + "type" : "string" + }, "license" : { "description" : "Machine readable license, or array of licenses", "$ref" : "#/definitions/licenses" From cd6a38c5c76bd7595fbbb03d6c48041163dfc869 Mon Sep 17 00:00:00 2001 From: Leon Wright Date: Wed, 4 May 2016 00:31:07 +0800 Subject: [PATCH 05/13] Output array of hashes + merge FileHash into FileService --- Netkan/CKAN-netkan.csproj | 2 - Netkan/Program.cs | 2 - Netkan/Services/FileHash.cs | 21 ---------- Netkan/Services/FileService.cs | 28 ++++++++++++- Netkan/Services/IFileHash.cs | 7 ---- Netkan/Services/IFileService.cs | 2 + .../Transformers/DownloadHashTransformer.cs | 22 +++++++++-- Netkan/Transformers/NetkanTransformer.cs | 3 +- .../Transformers/PropertySortTransformer.cs | 1 + Tests/NetKAN/Services/FileHashTests.cs | 25 ------------ Tests/NetKAN/Services/FileServiceTests.cs | 30 ++++++++++++++ .../DownloadHashTransformerTests.cs | 39 +++++++++++++------ Tests/Tests.csproj | 1 - 13 files changed, 106 insertions(+), 77 deletions(-) delete mode 100644 Netkan/Services/FileHash.cs delete mode 100644 Netkan/Services/IFileHash.cs delete mode 100644 Tests/NetKAN/Services/FileHashTests.cs diff --git a/Netkan/CKAN-netkan.csproj b/Netkan/CKAN-netkan.csproj index 32fcda2025..2b2075ae26 100644 --- a/Netkan/CKAN-netkan.csproj +++ b/Netkan/CKAN-netkan.csproj @@ -38,8 +38,6 @@ - - diff --git a/Netkan/Program.cs b/Netkan/Program.cs index 8863b4351c..931085db0d 100644 --- a/Netkan/Program.cs +++ b/Netkan/Program.cs @@ -46,7 +46,6 @@ public static int Main(string[] args) var moduleService = new ModuleService(); var fileService = new FileService(); - var fileHash = new FileHash(); var http = new CachingHttpService(FindCache(new KSPManager(new ConsoleUser(false)))); var netkan = ReadNetkan(); @@ -58,7 +57,6 @@ public static int Main(string[] args) var transformer = new NetkanTransformer( http, fileService, - fileHash, moduleService, Options.GitHubToken, Options.PreRelease diff --git a/Netkan/Services/FileHash.cs b/Netkan/Services/FileHash.cs deleted file mode 100644 index 63eabdc264..0000000000 --- a/Netkan/Services/FileHash.cs +++ /dev/null @@ -1,21 +0,0 @@ -using System; -using System.IO; -using System.Security.Cryptography; - -namespace CKAN.NetKAN.Services -{ - internal sealed class FileHash : IFileHash - { - public string GetFileHash(string filePath) - { - using (FileStream fs = new FileStream(@filePath, FileMode.Open)) - using (BufferedStream bs = new BufferedStream(fs)) - using (var sha1 = new SHA1Managed()) - { - byte[] hash = sha1.ComputeHash(bs); - - return BitConverter.ToString(hash).Replace("-", ""); - } - } - } -} diff --git a/Netkan/Services/FileService.cs b/Netkan/Services/FileService.cs index ba4f8856ff..e10eb664ad 100644 --- a/Netkan/Services/FileService.cs +++ b/Netkan/Services/FileService.cs @@ -1,4 +1,6 @@ -using System.IO; +using System; +using System.IO; +using System.Security.Cryptography; namespace CKAN.NetKAN.Services { @@ -8,5 +10,29 @@ public long GetSizeBytes(string filePath) { return new FileInfo(filePath).Length; } + + public string GetFileHashSha1(string filePath) + { + using (FileStream fs = new FileStream(@filePath, FileMode.Open)) + using (BufferedStream bs = new BufferedStream(fs)) + using (var sha1 = new SHA1Managed()) + { + byte[] hash = sha1.ComputeHash(bs); + + return BitConverter.ToString(hash).Replace("-", ""); + } + } + + public string GetFileHashSha256(string filePath) + { + using (FileStream fs = new FileStream(@filePath, FileMode.Open)) + using (BufferedStream bs = new BufferedStream(fs)) + using (var sha256 = new SHA256Managed()) + { + byte[] hash = sha256.ComputeHash(bs); + + return BitConverter.ToString(hash).Replace("-", ""); + } + } } } diff --git a/Netkan/Services/IFileHash.cs b/Netkan/Services/IFileHash.cs deleted file mode 100644 index fb54df25aa..0000000000 --- a/Netkan/Services/IFileHash.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace CKAN.NetKAN.Services -{ - internal interface IFileHash - { - string GetFileHash(string filePath); - } -} diff --git a/Netkan/Services/IFileService.cs b/Netkan/Services/IFileService.cs index e9ac6f7454..0fc781c3bb 100644 --- a/Netkan/Services/IFileService.cs +++ b/Netkan/Services/IFileService.cs @@ -3,5 +3,7 @@ internal interface IFileService { long GetSizeBytes(string filePath); + string GetFileHashSha1(string filePath); + string GetFileHashSha256(string filePath); } } diff --git a/Netkan/Transformers/DownloadHashTransformer.cs b/Netkan/Transformers/DownloadHashTransformer.cs index f2dd6fe89d..f169a1561f 100644 --- a/Netkan/Transformers/DownloadHashTransformer.cs +++ b/Netkan/Transformers/DownloadHashTransformer.cs @@ -2,6 +2,7 @@ using CKAN.NetKAN.Model; using CKAN.NetKAN.Services; using log4net; +using Newtonsoft.Json.Linq; namespace CKAN.NetKAN.Transformers { @@ -13,14 +14,14 @@ internal sealed class DownloadHashTransformer : ITransformer private static readonly ILog Log = LogManager.GetLogger(typeof(DownloadHashTransformer)); private readonly IHttpService _http; - private readonly IFileHash _fileHash; + private readonly IFileService _fileService; public string Name { get { return "download_hash"; } } - public DownloadHashTransformer(IHttpService http, IFileHash fileHash) + public DownloadHashTransformer(IHttpService http, IFileService fileService) { _http = http; - _fileHash = fileHash; + _fileService = fileService; } public Metadata Transform(Metadata metadata) @@ -36,7 +37,20 @@ public Metadata Transform(Metadata metadata) if (file != null) { - json["download_hash"] = _fileHash.GetFileHash(file); + var sha1 = new JObject(); + sha1.Add("type", "sha1"); + sha1.Add("digest", _fileService.GetFileHashSha1(file)); + + var sha256 = new JObject(); + sha256.Add("type", "sha256"); + sha256.Add("digest", _fileService.GetFileHashSha256(file)); + + json["download_hash"] = new JArray(); + + var download_hashJson = (JArray)json["download_hash"]; + + download_hashJson.Add(sha1); + download_hashJson.Add(sha256); } Log.DebugFormat("Transformed metadata:{0}{1}", Environment.NewLine, json); diff --git a/Netkan/Transformers/NetkanTransformer.cs b/Netkan/Transformers/NetkanTransformer.cs index de2acb5c4f..217fae282f 100644 --- a/Netkan/Transformers/NetkanTransformer.cs +++ b/Netkan/Transformers/NetkanTransformer.cs @@ -19,7 +19,6 @@ internal sealed class NetkanTransformer : ITransformer public NetkanTransformer( IHttpService http, IFileService fileService, - IFileHash fileHash, IModuleService moduleService, string githubToken, bool prerelease @@ -41,7 +40,7 @@ bool prerelease // specify a before or after property. new VersionedOverrideTransformer(before: new string[] { null }, after: new string[] { null }), new DownloadSizeTransformer(http, fileService), - new DownloadHashTransformer(http, fileHash), + new DownloadHashTransformer(http, fileService), new GeneratedByTransformer(), new OptimusPrimeTransformer(), new StripNetkanMetadataTransformer(), diff --git a/Netkan/Transformers/PropertySortTransformer.cs b/Netkan/Transformers/PropertySortTransformer.cs index f272a3d9b8..45a5106aaf 100644 --- a/Netkan/Transformers/PropertySortTransformer.cs +++ b/Netkan/Transformers/PropertySortTransformer.cs @@ -40,6 +40,7 @@ internal sealed class PropertySortTransformer : ITransformer { "install", 22 }, { "download", 23 }, { "download_size", 24 }, + { "download_hash", 25 }, { "x_generated_by", int.MaxValue } }; diff --git a/Tests/NetKAN/Services/FileHashTests.cs b/Tests/NetKAN/Services/FileHashTests.cs deleted file mode 100644 index eda3e0c45a..0000000000 --- a/Tests/NetKAN/Services/FileHashTests.cs +++ /dev/null @@ -1,25 +0,0 @@ -using CKAN.NetKAN.Services; -using NUnit.Framework; -using Tests.Data; - -namespace Tests.NetKAN.Services -{ - [TestFixture] - public sealed class FileHashTests - { - [Test] - public void GetsFileHashCorrectly() - { - // Arrange - var sut = new FileHash(); - - // Act - var result = sut.GetFileHash(TestData.DogeCoinFlagZip()); - - // Assert - Assert.That(result, Is.EqualTo("47B6ED5F502AD914744882858345BE030A29E1AA"), - "FileService should return the correct file hash." - ); - } - } -} diff --git a/Tests/NetKAN/Services/FileServiceTests.cs b/Tests/NetKAN/Services/FileServiceTests.cs index e34581b227..d8b5bd00dd 100644 --- a/Tests/NetKAN/Services/FileServiceTests.cs +++ b/Tests/NetKAN/Services/FileServiceTests.cs @@ -21,5 +21,35 @@ public void GetsFileSizeCorrectly() "FileService should return the correct file size." ); } + + [Test] + public void GetsFileHashSha1Correctly() + { + // Arrange + var sut = new FileService(); + + // Act + var result = sut.GetFileHashSha1(TestData.DogeCoinFlagZip()); + + // Assert + Assert.That(result, Is.EqualTo("47B6ED5F502AD914744882858345BE030A29E1AA"), + "FileService should return the correct file SHA1 hash." + ); + } + + [Test] + public void GetsFileHashSha256Correctly() + { + // Arrange + var sut = new FileService(); + + // Act + var result = sut.GetFileHashSha256(TestData.DogeCoinFlagZip()); + + // Assert + Assert.That(result, Is.EqualTo("EC955DB772FBA8CAA62BF61C180D624C350D792C6F573D35A5EAEE3898DCF7C1"), + "FileService should return the correct file SHA256 hash." + ); + } } } diff --git a/Tests/NetKAN/Transformers/DownloadHashTransformerTests.cs b/Tests/NetKAN/Transformers/DownloadHashTransformerTests.cs index 57ae0b0096..fcd82d8772 100644 --- a/Tests/NetKAN/Transformers/DownloadHashTransformerTests.cs +++ b/Tests/NetKAN/Transformers/DownloadHashTransformerTests.cs @@ -16,18 +16,22 @@ public void AddsDownloadHash() { // Arrange const string downloadFilePath = "/DoesNotExist.zip"; - const string downloadHash = "47B6ED5F502AD914744882858345BE030A29E1AA"; + const string downloadHashSha1 = "47B6ED5F502AD914744882858345BE030A29E1AA"; + const string downloadHashSha256 = "EC955DB772FBA8CAA62BF61C180D624C350D792C6F573D35A5EAEE3898DCF7C1"; var mHttp = new Mock(); - var mFileHash = new Mock(); + var mFileService = new Mock(); mHttp.Setup(i => i.DownloadPackage(It.IsAny(), It.IsAny())) .Returns(downloadFilePath); - mFileHash.Setup(i => i.GetFileHash(downloadFilePath)) - .Returns(downloadHash); + mFileService.Setup(i => i.GetFileHashSha1(downloadFilePath)) + .Returns(downloadHashSha1); - var sut = new DownloadHashTransformer(mHttp.Object, mFileHash.Object); + mFileService.Setup(i => i.GetFileHashSha256(downloadFilePath)) + .Returns(downloadHashSha256); + + var sut = new DownloadHashTransformer(mHttp.Object, mFileService.Object); var json = new JObject(); json["spec_version"] = 1; @@ -37,10 +41,21 @@ public void AddsDownloadHash() var result = sut.Transform(new Metadata(json)); var transformedJson = result.Json(); + // Collection Comparison + var sha1 = new JObject(); + sha1.Add("type", "sha1"); + sha1.Add("digest", downloadHashSha1); + + var sha256 = new JObject(); + sha256.Add("type", "sha256"); + sha256.Add("digest", downloadHashSha256); + + var download_hashJson = new JArray(); + download_hashJson.Add(sha1); + download_hashJson.Add(sha256); + // Assert - Assert.That((string)transformedJson["download_hash"], Is.EqualTo(downloadHash), - "DownloadHashTransformer should add a download_hash property equal to the calculated hash of the file." - ); + CollectionAssert.AreEquivalent(transformedJson["download_hash"],download_hashJson); } [Test] @@ -48,12 +63,12 @@ public void DoesNothingIfFileDoesNotExist() { // Arrange var mHttp = new Mock(); - var mFileHash = new Mock(); + var mFileService = new Mock(); mHttp.Setup(i => i.DownloadPackage(It.IsAny(), It.IsAny())) .Returns((string)null); - var sut = new DownloadHashTransformer(mHttp.Object, mFileHash.Object); + var sut = new DownloadHashTransformer(mHttp.Object, mFileService.Object); var json = new JObject(); json["spec_version"] = 1; @@ -74,9 +89,9 @@ public void DoesNothingIfDownloadDoesNotExist() { // Arrange var mHttp = new Mock(); - var mFileHash = new Mock(); + var mFileService = new Mock(); - var sut = new DownloadHashTransformer(mHttp.Object, mFileHash.Object); + var sut = new DownloadHashTransformer(mHttp.Object, mFileService.Object); var json = new JObject(); json["spec_version"] = 1; diff --git a/Tests/Tests.csproj b/Tests/Tests.csproj index cecf20fb99..06af6fb695 100644 --- a/Tests/Tests.csproj +++ b/Tests/Tests.csproj @@ -144,7 +144,6 @@ - From 3e7b2efa6da979be8b46f3c42faca8a91b2cdc60 Mon Sep 17 00:00:00 2001 From: Leon Wright Date: Wed, 4 May 2016 01:02:45 +0800 Subject: [PATCH 06/13] Spec + Schema updated --- CKAN.schema | 4 ++-- Spec.md | 13 +++++++++---- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/CKAN.schema b/CKAN.schema index 14b338cf00..5a5e0e41f5 100644 --- a/CKAN.schema +++ b/CKAN.schema @@ -65,8 +65,8 @@ "type" : "integer" }, "download_hash" : { - "description" : "The hash of the downloaded file", - "type" : "string" + "description" : "An array of hashes of the downloaded file", + "type" : "array" }, "license" : { "description" : "Machine readable license, or array of licenses", diff --git a/Spec.md b/Spec.md index c9b144d36f..844ce97e14 100644 --- a/Spec.md +++ b/Spec.md @@ -511,10 +511,15 @@ and not filled in by hand. ##### download_hash -If supplied, `download_hash` is the SHA1 calculated hash to expect of -the resulting file downloaded. It is recommended that this field is -only generated by automated tools (where it is encouraged), -and not filled in by hand. +If supplied, `download_hash` is an array of hash digests. Currently +SHA1 and SHA256 calculated hashes of the resulting file downloaded. +It is recommended that this field is only generated by automated +tools (where it is encouraged), and not filled in by hand. + + "download_hash": { + "sha1": "1F4B3F21A77D4A302E3417A7C7A24A0B63740FC5", + "sha256": "E3B0C44298FC1C149AFBF4C8996FB92427AE41E4649B934CA495991B7852B855" + } #### Extensions From db4a6454d6542ef5219809b95df2053b62ac2e74 Mon Sep 17 00:00:00 2001 From: Leon Wright Date: Wed, 4 May 2016 01:19:06 +0800 Subject: [PATCH 07/13] Change array for "type":"digest" objects --- Netkan/Transformers/DownloadHashTransformer.cs | 18 +++++------------- .../DownloadHashTransformerTests.cs | 17 +++++------------ 2 files changed, 10 insertions(+), 25 deletions(-) diff --git a/Netkan/Transformers/DownloadHashTransformer.cs b/Netkan/Transformers/DownloadHashTransformer.cs index f169a1561f..f61228356e 100644 --- a/Netkan/Transformers/DownloadHashTransformer.cs +++ b/Netkan/Transformers/DownloadHashTransformer.cs @@ -1,6 +1,7 @@ using System; using CKAN.NetKAN.Model; using CKAN.NetKAN.Services; +using CKAN.NetKAN.Extensions; using log4net; using Newtonsoft.Json.Linq; @@ -37,20 +38,11 @@ public Metadata Transform(Metadata metadata) if (file != null) { - var sha1 = new JObject(); - sha1.Add("type", "sha1"); - sha1.Add("digest", _fileService.GetFileHashSha1(file)); - - var sha256 = new JObject(); - sha256.Add("type", "sha256"); - sha256.Add("digest", _fileService.GetFileHashSha256(file)); - - json["download_hash"] = new JArray(); + json["download_hash"] = new JObject(); - var download_hashJson = (JArray)json["download_hash"]; - - download_hashJson.Add(sha1); - download_hashJson.Add(sha256); + var download_hashJson = (JObject)json["download_hash"]; + download_hashJson.SafeAdd("sha1", _fileService.GetFileHashSha1(file)); + download_hashJson.SafeAdd("sha256", _fileService.GetFileHashSha256(file)); } Log.DebugFormat("Transformed metadata:{0}{1}", Environment.NewLine, json); diff --git a/Tests/NetKAN/Transformers/DownloadHashTransformerTests.cs b/Tests/NetKAN/Transformers/DownloadHashTransformerTests.cs index fcd82d8772..25b087922d 100644 --- a/Tests/NetKAN/Transformers/DownloadHashTransformerTests.cs +++ b/Tests/NetKAN/Transformers/DownloadHashTransformerTests.cs @@ -1,6 +1,7 @@ -using System; +using System; using CKAN.NetKAN.Model; using CKAN.NetKAN.Services; +using CKAN.NetKAN.Extensions; using CKAN.NetKAN.Transformers; using Moq; using Newtonsoft.Json.Linq; @@ -42,17 +43,9 @@ public void AddsDownloadHash() var transformedJson = result.Json(); // Collection Comparison - var sha1 = new JObject(); - sha1.Add("type", "sha1"); - sha1.Add("digest", downloadHashSha1); - - var sha256 = new JObject(); - sha256.Add("type", "sha256"); - sha256.Add("digest", downloadHashSha256); - - var download_hashJson = new JArray(); - download_hashJson.Add(sha1); - download_hashJson.Add(sha256); + var download_hashJson = new JObject(); + download_hashJson.SafeAdd("sha1", downloadHashSha1); + download_hashJson.SafeAdd("sha256", downloadHashSha256); // Assert CollectionAssert.AreEquivalent(transformedJson["download_hash"],download_hashJson); From 19049e3e5e836580b3f36f997865c8c0617fe47c Mon Sep 17 00:00:00 2001 From: Leon Wright Date: Wed, 4 May 2016 01:25:31 +0800 Subject: [PATCH 08/13] Update schema to reflect download_hash changes --- CKAN.schema | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/CKAN.schema b/CKAN.schema index 5a5e0e41f5..534c9294d8 100644 --- a/CKAN.schema +++ b/CKAN.schema @@ -65,8 +65,18 @@ "type" : "integer" }, "download_hash" : { - "description" : "An array of hashes of the downloaded file", - "type" : "array" + "description" : "A object of hashes of the downloaded file", + "type" : "object", + "properties" : { + "sha1" : { + "description" : "SHA1 hash of the file", + "type" : "string", + }, + "sha256" : { + "description" : "SHA256 hash of the file", + "type" : "string", + } + } }, "license" : { "description" : "Machine readable license, or array of licenses", From e8357a3e3bbc703d5bec34bf16c6dbe2ccb8971f Mon Sep 17 00:00:00 2001 From: Leon Wright Date: Wed, 4 May 2016 01:32:41 +0800 Subject: [PATCH 09/13] Extraneous commas --- CKAN.schema | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CKAN.schema b/CKAN.schema index 534c9294d8..1275bf274e 100644 --- a/CKAN.schema +++ b/CKAN.schema @@ -70,11 +70,11 @@ "properties" : { "sha1" : { "description" : "SHA1 hash of the file", - "type" : "string", + "type" : "string" }, "sha256" : { "description" : "SHA256 hash of the file", - "type" : "string", + "type" : "string" } } }, From d5ecd52d42bbde8532faed727cec3df7cfb5e06e Mon Sep 17 00:00:00 2001 From: Leon Wright Date: Wed, 4 May 2016 13:00:52 +0800 Subject: [PATCH 10/13] Add 'download_content_type' to Spec/Schema --- CKAN.schema | 4 ++++ Spec.md | 13 +++++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/CKAN.schema b/CKAN.schema index 1275bf274e..5d25f7040c 100644 --- a/CKAN.schema +++ b/CKAN.schema @@ -78,6 +78,10 @@ } } }, + "download_content_type" : { + "description" : "The content type of the download", + "type" : "string" + }, "license" : { "description" : "Machine readable license, or array of licenses", "$ref" : "#/definitions/licenses" diff --git a/Spec.md b/Spec.md index 844ce97e14..c71aec28d6 100644 --- a/Spec.md +++ b/Spec.md @@ -521,6 +521,15 @@ tools (where it is encouraged), and not filled in by hand. "sha256": "E3B0C44298FC1C149AFBF4C8996FB92427AE41E4649B934CA495991B7852B855" } +##### download_content_type + +If supplied, `download_content_type` is the content type of the file +downloaded from the `download` URL. It is recommended that this field is +only generated by automated tools (where it is encouraged), +and not filled in by hand. + + "download_content_type": "application/zip" + #### Extensions Any field starting with `x_` (an x, followed by an underscore) is considered @@ -637,6 +646,7 @@ When used, the following fields will be auto-filled if not already present: - `download` - `download_size` - `download_hash` +- `download_content_type` This method depends on the existence of an AVC `.version` file in the download file to determine: @@ -772,8 +782,7 @@ The possible values of `before` and `after` are: - `$none` - `$all` - `avc` -- `download_size` -- `download_hash` +- `download_attributes` - `epoch` - `forced_v` - `generated_by` From 5f3da7983e317aa8ade242ecbed10442c4bee5b2 Mon Sep 17 00:00:00 2001 From: Leon Wright Date: Wed, 4 May 2016 13:02:12 +0800 Subject: [PATCH 11/13] Merge DownloadHash/DownloadSize into DownloadAttribute and add content_type --- Netkan/CKAN-netkan.csproj | 3 +- Netkan/Services/FileService.cs | 29 ++++++ Netkan/Services/IFileService.cs | 1 + ...mer.cs => DownloadAttributeTransformer.cs} | 16 +-- .../Transformers/DownloadSizeTransformer.cs | 50 --------- Netkan/Transformers/NetkanTransformer.cs | 3 +- .../Transformers/PropertySortTransformer.cs | 1 + Tests/Data/FileIdentifier/random.bin | Bin 0 -> 1024 bytes Tests/Data/FileIdentifier/test_gzip.gz | Bin 0 -> 1349 bytes Tests/NetKAN/Services/FileServiceTests.cs | 95 +++++++++++++++++- ...s => DownloadAttributeTransformerTests.cs} | 40 +++++--- .../DownloadSizeTransformerTests.cs | 94 ----------------- Tests/Tests.csproj | 3 +- 13 files changed, 165 insertions(+), 170 deletions(-) rename Netkan/Transformers/{DownloadHashTransformer.cs => DownloadAttributeTransformer.cs} (69%) delete mode 100644 Netkan/Transformers/DownloadSizeTransformer.cs create mode 100644 Tests/Data/FileIdentifier/random.bin create mode 100644 Tests/Data/FileIdentifier/test_gzip.gz rename Tests/NetKAN/Transformers/{DownloadHashTransformerTests.cs => DownloadAttributeTransformerTests.cs} (56%) delete mode 100644 Tests/NetKAN/Transformers/DownloadSizeTransformerTests.cs diff --git a/Netkan/CKAN-netkan.csproj b/Netkan/CKAN-netkan.csproj index 2b2075ae26..287e786430 100644 --- a/Netkan/CKAN-netkan.csproj +++ b/Netkan/CKAN-netkan.csproj @@ -60,8 +60,7 @@ - - + diff --git a/Netkan/Services/FileService.cs b/Netkan/Services/FileService.cs index e10eb664ad..b26944bb94 100644 --- a/Netkan/Services/FileService.cs +++ b/Netkan/Services/FileService.cs @@ -34,5 +34,34 @@ public string GetFileHashSha256(string filePath) return BitConverter.ToString(hash).Replace("-", ""); } } + + public string GetMimetype(string filePath) + { + string mimetype; + + switch (FileIdentifier.IdentifyFile(filePath)) + { + case FileType.ASCII: + mimetype = "text/plain"; + break; + case FileType.GZip: + mimetype = "application/x-gzip"; + break; + case FileType.Tar: + mimetype = "application/x-tar"; + break; + case FileType.TarGz: + mimetype = "application/x-compressed-tar"; + break; + case FileType.Zip: + mimetype = "application/zip"; + break; + default: + mimetype = "application/octet-stream"; + break; + } + + return mimetype; + } } } diff --git a/Netkan/Services/IFileService.cs b/Netkan/Services/IFileService.cs index 0fc781c3bb..08a21a72cf 100644 --- a/Netkan/Services/IFileService.cs +++ b/Netkan/Services/IFileService.cs @@ -5,5 +5,6 @@ internal interface IFileService long GetSizeBytes(string filePath); string GetFileHashSha1(string filePath); string GetFileHashSha256(string filePath); + string GetMimetype(string filePath); } } diff --git a/Netkan/Transformers/DownloadHashTransformer.cs b/Netkan/Transformers/DownloadAttributeTransformer.cs similarity index 69% rename from Netkan/Transformers/DownloadHashTransformer.cs rename to Netkan/Transformers/DownloadAttributeTransformer.cs index f61228356e..aec5c7891c 100644 --- a/Netkan/Transformers/DownloadHashTransformer.cs +++ b/Netkan/Transformers/DownloadAttributeTransformer.cs @@ -8,18 +8,18 @@ namespace CKAN.NetKAN.Transformers { /// - /// An that adds a hash of the downloaded file. + /// An that adds the size of the download. /// - internal sealed class DownloadHashTransformer : ITransformer + internal sealed class DownloadAttributeTransformer : ITransformer { - private static readonly ILog Log = LogManager.GetLogger(typeof(DownloadHashTransformer)); + private static readonly ILog Log = LogManager.GetLogger(typeof(DownloadAttributeTransformer)); private readonly IHttpService _http; private readonly IFileService _fileService; - public string Name { get { return "download_hash"; } } + public string Name { get { return "download_attributes"; } } - public DownloadHashTransformer(IHttpService http, IFileService fileService) + public DownloadAttributeTransformer(IHttpService http, IFileService fileService) { _http = http; _fileService = fileService; @@ -31,18 +31,22 @@ public Metadata Transform(Metadata metadata) { var json = metadata.Json(); - Log.InfoFormat("Executing Download Hash transformation with {0}", metadata.Kref); + Log.InfoFormat("Executing Download Attribute transformation with {0}", metadata.Kref); Log.DebugFormat("Input metadata:{0}{1}", Environment.NewLine, json); var file = _http.DownloadPackage(metadata.Download, metadata.Identifier); if (file != null) { + json["download_size"] = _fileService.GetSizeBytes(file); + json["download_hash"] = new JObject(); var download_hashJson = (JObject)json["download_hash"]; download_hashJson.SafeAdd("sha1", _fileService.GetFileHashSha1(file)); download_hashJson.SafeAdd("sha256", _fileService.GetFileHashSha256(file)); + + json["download_content_type"] = _fileService.GetMimetype(file); } Log.DebugFormat("Transformed metadata:{0}{1}", Environment.NewLine, json); diff --git a/Netkan/Transformers/DownloadSizeTransformer.cs b/Netkan/Transformers/DownloadSizeTransformer.cs deleted file mode 100644 index b7d1217848..0000000000 --- a/Netkan/Transformers/DownloadSizeTransformer.cs +++ /dev/null @@ -1,50 +0,0 @@ -using System; -using CKAN.NetKAN.Model; -using CKAN.NetKAN.Services; -using log4net; - -namespace CKAN.NetKAN.Transformers -{ - /// - /// An that adds the size of the download. - /// - internal sealed class DownloadSizeTransformer : ITransformer - { - private static readonly ILog Log = LogManager.GetLogger(typeof(DownloadSizeTransformer)); - - private readonly IHttpService _http; - private readonly IFileService _fileService; - - public string Name { get { return "download_size"; } } - - public DownloadSizeTransformer(IHttpService http, IFileService fileService) - { - _http = http; - _fileService = fileService; - } - - public Metadata Transform(Metadata metadata) - { - if (metadata.Download != null) - { - var json = metadata.Json(); - - Log.InfoFormat("Executing Download Size transformation with {0}", metadata.Kref); - Log.DebugFormat("Input metadata:{0}{1}", Environment.NewLine, json); - - var file = _http.DownloadPackage(metadata.Download, metadata.Identifier); - - if (file != null) - { - json["download_size"] = _fileService.GetSizeBytes(file); - } - - Log.DebugFormat("Transformed metadata:{0}{1}", Environment.NewLine, json); - - return new Metadata(json); - } - - return metadata; - } - } -} diff --git a/Netkan/Transformers/NetkanTransformer.cs b/Netkan/Transformers/NetkanTransformer.cs index 217fae282f..49e44b307b 100644 --- a/Netkan/Transformers/NetkanTransformer.cs +++ b/Netkan/Transformers/NetkanTransformer.cs @@ -39,8 +39,7 @@ bool prerelease // This is the "default" VersionedOverrideTransformer for compatability with overrides that don't // specify a before or after property. new VersionedOverrideTransformer(before: new string[] { null }, after: new string[] { null }), - new DownloadSizeTransformer(http, fileService), - new DownloadHashTransformer(http, fileService), + new DownloadAttributeTransformer(http, fileService), new GeneratedByTransformer(), new OptimusPrimeTransformer(), new StripNetkanMetadataTransformer(), diff --git a/Netkan/Transformers/PropertySortTransformer.cs b/Netkan/Transformers/PropertySortTransformer.cs index 45a5106aaf..ddda9a946a 100644 --- a/Netkan/Transformers/PropertySortTransformer.cs +++ b/Netkan/Transformers/PropertySortTransformer.cs @@ -41,6 +41,7 @@ internal sealed class PropertySortTransformer : ITransformer { "download", 23 }, { "download_size", 24 }, { "download_hash", 25 }, + { "download_content_type", 26 }, { "x_generated_by", int.MaxValue } }; diff --git a/Tests/Data/FileIdentifier/random.bin b/Tests/Data/FileIdentifier/random.bin new file mode 100644 index 0000000000000000000000000000000000000000..068a9fa7cd1d7d8402d349517a019d428afec5e6 GIT binary patch literal 1024 zcmV+b1poVTs6;dTh3Q3z7I@NPy8Pe-C1Bpi`%I!U?2kP!DrS=zY}e`Y1SLf(@7Rd3 z@>gITzL=L}E_s_hL;|OS*u03)9$)!ND87_`DE#bhasWlp3xO8iK^Ir@rMSWSV@n$j zoPSfy6I7Iyt!sogv&<781;U02Yo+6_+3X+Vq+g&XUM{a>FM(0?gPe|`tjYVzX#yzi z%h)1HH7TLVbPVuX9wB4Z&!O$}CEeHK$stCpSUm2$Eq<{DrMs)N-EeE7Dqfj_d%Qv$ znFzwYk?Wz)hAY>W>491>Z3xZtDAJlfxw9OK(QMY^(}M!l$63;7hfX>}z3p2jzb%ve zy+oY9nt7`B5xvp9NMTIAuek03y3%*OyKez3)H^;-h+o{Fsf)&0+5i+8o=-{(_>d>= zTii?xP7z6L&5W)05p9|kGuR|GubLG?^K`MNm_-*%88#g`Z8-hVv{WGAO)KXs>Y9WN z=o$Tx!~rITV%OoqzGfi589=hW0JAI!Cp=eNY8&6rnqZxBw^U<-V{~y(W506x9z8V1 zxA2I!bRo+5sLIR>c@t>(@?;*egTcpJ=`hjzBjhycZ8TN@f>Oja7S~XmOEEPo0jVvN zfZdu$d*J0))dtxk2O63hA##~R_qsOeu-}86ePXiw#6=pdqRLL6saKe)^DireK+(%~v z5d0=L?@))`8gJkV^H$vzXCF4PGnLHCEImX?FR-K$;+Qou30ZvI-LnJobq6_U$vjsQ z^VYO9Nr?Bg)|?3y#Wa@;vqt6Ersu0ECH||OnYfetMQrj{=V-s5mW63=aXPah;lJjt z9A=Ec<@4-TS}6Am+IbK%`5a2sO-RfHSDK8a;V;#}8pb&ePJr9u`e-L`^Mk{s$;mO( zyR6UffxJvf;vSv)^yuvy@>-K7t4(8*N6qhL*qH2G6`l3jS<|RwlvNV@6s8PwA2pr_ z%h1#Z&7-ZTO-ck#@F6AP9giVj7kaoSoqH?WjmrgLiw9m{WrV9K5mY--Jwy&rO3Ck+ z?h{#}(Ars!rpF_2nuD9TJ*hsla)8lK78zkmB0TZwK+F=ZQTy6BNuwHK^*O2jItu5* zlG78zNUqqp_9;=#kV-?j%30K<_Y6$X@0pj{VMeXRf7wxKG5@zUjX=E#wV%&AQg_DK z>3rPvsXTdLQfsdD?{xQP1M&NLFp%z5$(pLIh|fXiOWkYTpe4alPe=WErAGj0Oqr`P uMv&6RianG)Xi~J{R41JAvm%O%Wpa+Fz2R@I({9GsM~UugF%py@R@2P_&<8#M literal 0 HcmV?d00001 diff --git a/Tests/Data/FileIdentifier/test_gzip.gz b/Tests/Data/FileIdentifier/test_gzip.gz new file mode 100644 index 0000000000000000000000000000000000000000..6cdf4db7f856f7c1fb9d701adba5a69a9dc3b0f3 GIT binary patch literal 1349 zcmV-L1-kkliwFoCb17E<19W9`bYEwBX>cxdcys`rSJ`gdMi6}0SM&!E`wxbb00wL! zK#;u8lC!%V%t6hu(C53l=OCrLBmoS_B8PocS5=R`$7DV6o~x%Vx|k?)A>(aXXpJG; zwb)VbOD~CH-$K^t=)j30kym@C|Gd*f?iPw$R4LlZ)4w3g;g2t(rof zPl+UyUt`*BLcE;Kg?_ir*-&hLtCoCpHW#*c0VHWloz96Cr^h7?Om0~9LKy_n*`59d;JHU53bc07`U8~Rd!Upyz`(ZLF zMU^g7y6+SyYdyZkW<1xbR78lI8`r169WEL^JvdST!DSXl_MI2HIfq=7$FyO;E2cpVALxpA7TbC7k!3O?OveD z2cuGBqt5&c;Sl7~R3f-cYG0j!IidQXFjhDePTfI|%zjcp6*}nNn$}RDw*Cg2Lr^)T za}a{sco;!zo~;13%14r>eKR=Q_C({cD#43MX7$FI+0^@)5g*NUkAfttfjoe6b4Y#T zOtI1yWbJcliM3P5_v(!FcQTm%PTXy5nQ8xhBYpqB8%{E#5u_vDC~h22-fV9-T?=Wb z{=CBk-0oX*IlYR|hYhNj0gG?bin-ckyFC0nCH~P+?u+Iihw`F*HY(E<_L)s;oKv_b zYBwL0ILm3|$k?~bv%0+&%%W`(1#X$hos6@&+t83SHV`q#NECh9Hq)TgX?m`1b*)ux zdM0noy3nshC_L{0&Ds#IZSua>jT2`;{vW2|giQC|B(D_55C?$6^-f%B8WC&HjvgRt zf`1h?d82U4d4!qKTjRp@Wo}Ek4(UtnWf`0CKnu43J0naq`*<6};aMp{l~eXN7^7mQ zdTl7Rs}0%^C*G@jGHxp;D|FaFDQeB#%s(ubqrbyY*XSYc*m&RQG@M5LzKvo!Ck(A7 zu~osV$yF~Ai*SDK0bOtH<}kNR*8-+D7fKUc?|V21e)i2UH|iwDSS8cPMWv>1%4Sna zx6^`uKYG=@(I$Jd_0Get^o}@oPg|_Ysb<8-1@mbVOY6;si*j~yb4cu;WA0Zd730{&IO5H&I1v zx??wluQf?g;(); var mFileService = new Mock(); @@ -31,8 +33,14 @@ public void AddsDownloadHash() mFileService.Setup(i => i.GetFileHashSha256(downloadFilePath)) .Returns(downloadHashSha256); + + mFileService.Setup(i => i.GetSizeBytes(downloadFilePath)) + .Returns(downloadSize); - var sut = new DownloadHashTransformer(mHttp.Object, mFileService.Object); + mFileService.Setup(i => i.GetMimetype(downloadFilePath)) + .Returns(downloadMimetype); + + var sut = new DownloadAttributeTransformer(mHttp.Object, mFileService.Object); var json = new JObject(); json["spec_version"] = 1; @@ -42,13 +50,19 @@ public void AddsDownloadHash() var result = sut.Transform(new Metadata(json)); var transformedJson = result.Json(); - // Collection Comparison - var download_hashJson = new JObject(); - download_hashJson.SafeAdd("sha1", downloadHashSha1); - download_hashJson.SafeAdd("sha256", downloadHashSha256); - // Assert - CollectionAssert.AreEquivalent(transformedJson["download_hash"],download_hashJson); + Assert.That((string)transformedJson["download_hash"]["sha1"], Is.EqualTo(downloadHashSha1), + "DownloadAttributeTransformer should add a 'sha1' property withing 'download_hash' equal to the sha1 of the file." + ); + Assert.That((string)transformedJson["download_hash"]["sha256"], Is.EqualTo(downloadHashSha256), + "DownloadAttributeTransformer should add a 'sha256' property withing 'download_hash' equal to the sha256 of the file." + ); + Assert.That((long)transformedJson["download_size"], Is.EqualTo(downloadSize), + "DownloadAttributeTransformer should add a download_size property equal to the size of the file in bytes." + ); + Assert.That((string)transformedJson["download_content_type"], Is.EqualTo(downloadMimetype), + "DownloadAttributeTransformer should add a download_content_type property equal to the Mimetype of the file." + ); } [Test] @@ -61,7 +75,7 @@ public void DoesNothingIfFileDoesNotExist() mHttp.Setup(i => i.DownloadPackage(It.IsAny(), It.IsAny())) .Returns((string)null); - var sut = new DownloadHashTransformer(mHttp.Object, mFileService.Object); + var sut = new DownloadAttributeTransformer(mHttp.Object, mFileService.Object); var json = new JObject(); json["spec_version"] = 1; @@ -73,7 +87,7 @@ public void DoesNothingIfFileDoesNotExist() // Assert Assert.That(transformedJson, Is.EqualTo(json), - "DownloadHashTransformer should do nothing if the file does not exist." + "DownloadAttributeTransformer should do nothing if the file does not exist." ); } @@ -84,7 +98,7 @@ public void DoesNothingIfDownloadDoesNotExist() var mHttp = new Mock(); var mFileService = new Mock(); - var sut = new DownloadHashTransformer(mHttp.Object, mFileService.Object); + var sut = new DownloadAttributeTransformer(mHttp.Object, mFileService.Object); var json = new JObject(); json["spec_version"] = 1; @@ -95,7 +109,7 @@ public void DoesNothingIfDownloadDoesNotExist() // Assert Assert.That(transformedJson, Is.EqualTo(json), - "DownloadHashTransformer should do nothing if the download property does not exist." + "DownloadAttributeTransformer should do nothing if the download property does not exist." ); } } diff --git a/Tests/NetKAN/Transformers/DownloadSizeTransformerTests.cs b/Tests/NetKAN/Transformers/DownloadSizeTransformerTests.cs deleted file mode 100644 index 06621112dc..0000000000 --- a/Tests/NetKAN/Transformers/DownloadSizeTransformerTests.cs +++ /dev/null @@ -1,94 +0,0 @@ -using System; -using CKAN.NetKAN.Model; -using CKAN.NetKAN.Services; -using CKAN.NetKAN.Transformers; -using Moq; -using Newtonsoft.Json.Linq; -using NUnit.Framework; - -namespace Tests.NetKAN.Transformers -{ - [TestFixture] - public sealed class DownloadSizeTransformerTests - { - [Test] - public void AddsDownloadSize() - { - // Arrange - const string downloadFilePath = "/DoesNotExist.zip"; - const long downloadSize = 9001; - - var mHttp = new Mock(); - var mFileService = new Mock(); - - mHttp.Setup(i => i.DownloadPackage(It.IsAny(), It.IsAny())) - .Returns(downloadFilePath); - - mFileService.Setup(i => i.GetSizeBytes(downloadFilePath)) - .Returns(downloadSize); - - var sut = new DownloadSizeTransformer(mHttp.Object, mFileService.Object); - - var json = new JObject(); - json["spec_version"] = 1; - json["download"] = "https://awesomemod.example/AwesomeMod.zip"; - - // Act - var result = sut.Transform(new Metadata(json)); - var transformedJson = result.Json(); - - // Assert - Assert.That((long)transformedJson["download_size"], Is.EqualTo(downloadSize), - "DownloadSizeTransformer should add a download_size property equal to the size of the file in bytes." - ); - } - - [Test] - public void DoesNothingIfFileDoesNotExist() - { - // Arrange - var mHttp = new Mock(); - var mFileService = new Mock(); - - mHttp.Setup(i => i.DownloadPackage(It.IsAny(), It.IsAny())) - .Returns((string)null); - - var sut = new DownloadSizeTransformer(mHttp.Object, mFileService.Object); - - var json = new JObject(); - json["spec_version"] = 1; - json["download"] = "https://awesomemod.example/AwesomeMod.zip"; - - // Act - var result = sut.Transform(new Metadata(json)); - var transformedJson = result.Json(); - - // Assert - Assert.That(transformedJson, Is.EqualTo(json), - "DownloadSizeTransformer should do nothing if the file does not exist." - ); - } - - [Test] - public void DoesNothingIfDownloadDoesNotExist() - { - // Arrange - var mHttp = new Mock(); - var mFileService = new Mock(); - - var sut = new DownloadSizeTransformer(mHttp.Object, mFileService.Object); - - var json = new JObject(); - json["spec_version"] = 1; - - // Act - var result = sut.Transform(new Metadata(json)); - var transformedJson = result.Json(); - - // Assert - Assert.That(transformedJson, Is.EqualTo(json), - "DownloadSizeTransformer should do nothing if the download property does not exist." - ); - } - } -} diff --git a/Tests/Tests.csproj b/Tests/Tests.csproj index 06af6fb695..bd66659395 100644 --- a/Tests/Tests.csproj +++ b/Tests/Tests.csproj @@ -148,8 +148,7 @@ - - + From 98014bb7a84585b388b1f457f090634f0c5e01c0 Mon Sep 17 00:00:00 2001 From: Leon Wright Date: Wed, 4 May 2016 13:17:40 +0800 Subject: [PATCH 12/13] Update changelog for 'Add Download Attribute Transformer' --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 38869b0041..f84e9153b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,7 +22,7 @@ All notable changes to this project will be documented in this file. - [NetKAN] Allow specifying when an override is executed (#1684 by: dbent; fixes: #1674) - [NetKAN] Redirects to the download file are now resolved when using HTTP $krefs (#1696 by: dbent, reviewed: techman83) - [NetKAN] Remote AVC files will be used in preference to ones stored in the archive if they have the same version (#1701 by: dbent, reviewed: techman83) -- [NetKAN] Add download hash to Metadata (#1703 by: techman83; addresses: #1682) +- [NetKAN] Add Download Attribute Transformer (#1710 by: techman83; addresses: #1682) ## v1.16.1 From 35b6eb99e11e78321eee2d3c1aac4b3a77ea4488 Mon Sep 17 00:00:00 2001 From: Leon Wright Date: Wed, 4 May 2016 21:57:28 +0800 Subject: [PATCH 13/13] Update spec to match change in download_hash --- Spec.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Spec.md b/Spec.md index c71aec28d6..5409bf0362 100644 --- a/Spec.md +++ b/Spec.md @@ -511,7 +511,7 @@ and not filled in by hand. ##### download_hash -If supplied, `download_hash` is an array of hash digests. Currently +If supplied, `download_hash` is an object of hash digests. Currently SHA1 and SHA256 calculated hashes of the resulting file downloaded. It is recommended that this field is only generated by automated tools (where it is encouraged), and not filled in by hand.