diff --git a/CHANGELOG.md b/CHANGELOG.md
index f52626e4c6..f84e9153b4 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -22,6 +22,7 @@ All notable changes to this project will be documented in this file.
- [NetKAN] Allow specifying when an override is executed (#1684 by: dbent; fixes: #1674)
- [NetKAN] Redirects to the download file are now resolved when using HTTP $krefs (#1696 by: dbent, reviewed: techman83)
- [NetKAN] Remote AVC files will be used in preference to ones stored in the archive if they have the same version (#1701 by: dbent, reviewed: techman83)
+- [NetKAN] Add Download Attribute Transformer (#1710 by: techman83; addresses: #1682)
## v1.16.1
diff --git a/CKAN.schema b/CKAN.schema
index 9f592f3843..5d25f7040c 100644
--- a/CKAN.schema
+++ b/CKAN.schema
@@ -64,6 +64,24 @@
"description" : "The size of the download in bytes",
"type" : "integer"
},
+ "download_hash" : {
+ "description" : "A object of hashes of the downloaded file",
+ "type" : "object",
+ "properties" : {
+ "sha1" : {
+ "description" : "SHA1 hash of the file",
+ "type" : "string"
+ },
+ "sha256" : {
+ "description" : "SHA256 hash of the file",
+ "type" : "string"
+ }
+ }
+ },
+ "download_content_type" : {
+ "description" : "The content type of the download",
+ "type" : "string"
+ },
"license" : {
"description" : "Machine readable license, or array of licenses",
"$ref" : "#/definitions/licenses"
diff --git a/Netkan/CKAN-netkan.csproj b/Netkan/CKAN-netkan.csproj
index 2265d35fec..287e786430 100644
--- a/Netkan/CKAN-netkan.csproj
+++ b/Netkan/CKAN-netkan.csproj
@@ -60,7 +60,7 @@
-
+
diff --git a/Netkan/Services/FileService.cs b/Netkan/Services/FileService.cs
index ba4f8856ff..b26944bb94 100644
--- a/Netkan/Services/FileService.cs
+++ b/Netkan/Services/FileService.cs
@@ -1,4 +1,6 @@
-using System.IO;
+using System;
+using System.IO;
+using System.Security.Cryptography;
namespace CKAN.NetKAN.Services
{
@@ -8,5 +10,58 @@ public long GetSizeBytes(string filePath)
{
return new FileInfo(filePath).Length;
}
+
+ public string GetFileHashSha1(string filePath)
+ {
+ using (FileStream fs = new FileStream(@filePath, FileMode.Open))
+ using (BufferedStream bs = new BufferedStream(fs))
+ using (var sha1 = new SHA1Managed())
+ {
+ byte[] hash = sha1.ComputeHash(bs);
+
+ return BitConverter.ToString(hash).Replace("-", "");
+ }
+ }
+
+ public string GetFileHashSha256(string filePath)
+ {
+ using (FileStream fs = new FileStream(@filePath, FileMode.Open))
+ using (BufferedStream bs = new BufferedStream(fs))
+ using (var sha256 = new SHA256Managed())
+ {
+ byte[] hash = sha256.ComputeHash(bs);
+
+ return BitConverter.ToString(hash).Replace("-", "");
+ }
+ }
+
+ public string GetMimetype(string filePath)
+ {
+ string mimetype;
+
+ switch (FileIdentifier.IdentifyFile(filePath))
+ {
+ case FileType.ASCII:
+ mimetype = "text/plain";
+ break;
+ case FileType.GZip:
+ mimetype = "application/x-gzip";
+ break;
+ case FileType.Tar:
+ mimetype = "application/x-tar";
+ break;
+ case FileType.TarGz:
+ mimetype = "application/x-compressed-tar";
+ break;
+ case FileType.Zip:
+ mimetype = "application/zip";
+ break;
+ default:
+ mimetype = "application/octet-stream";
+ break;
+ }
+
+ return mimetype;
+ }
}
}
diff --git a/Netkan/Services/IFileService.cs b/Netkan/Services/IFileService.cs
index e9ac6f7454..08a21a72cf 100644
--- a/Netkan/Services/IFileService.cs
+++ b/Netkan/Services/IFileService.cs
@@ -3,5 +3,8 @@
internal interface IFileService
{
long GetSizeBytes(string filePath);
+ string GetFileHashSha1(string filePath);
+ string GetFileHashSha256(string filePath);
+ string GetMimetype(string filePath);
}
}
diff --git a/Netkan/Transformers/DownloadSizeTransformer.cs b/Netkan/Transformers/DownloadAttributeTransformer.cs
similarity index 57%
rename from Netkan/Transformers/DownloadSizeTransformer.cs
rename to Netkan/Transformers/DownloadAttributeTransformer.cs
index b7d1217848..aec5c7891c 100644
--- a/Netkan/Transformers/DownloadSizeTransformer.cs
+++ b/Netkan/Transformers/DownloadAttributeTransformer.cs
@@ -1,23 +1,25 @@
using System;
using CKAN.NetKAN.Model;
using CKAN.NetKAN.Services;
+using CKAN.NetKAN.Extensions;
using log4net;
+using Newtonsoft.Json.Linq;
namespace CKAN.NetKAN.Transformers
{
///
/// An that adds the size of the download.
///
- internal sealed class DownloadSizeTransformer : ITransformer
+ internal sealed class DownloadAttributeTransformer : ITransformer
{
- private static readonly ILog Log = LogManager.GetLogger(typeof(DownloadSizeTransformer));
+ private static readonly ILog Log = LogManager.GetLogger(typeof(DownloadAttributeTransformer));
private readonly IHttpService _http;
private readonly IFileService _fileService;
- public string Name { get { return "download_size"; } }
+ public string Name { get { return "download_attributes"; } }
- public DownloadSizeTransformer(IHttpService http, IFileService fileService)
+ public DownloadAttributeTransformer(IHttpService http, IFileService fileService)
{
_http = http;
_fileService = fileService;
@@ -29,7 +31,7 @@ public Metadata Transform(Metadata metadata)
{
var json = metadata.Json();
- Log.InfoFormat("Executing Download Size transformation with {0}", metadata.Kref);
+ Log.InfoFormat("Executing Download Attribute transformation with {0}", metadata.Kref);
Log.DebugFormat("Input metadata:{0}{1}", Environment.NewLine, json);
var file = _http.DownloadPackage(metadata.Download, metadata.Identifier);
@@ -37,6 +39,14 @@ public Metadata Transform(Metadata metadata)
if (file != null)
{
json["download_size"] = _fileService.GetSizeBytes(file);
+
+ json["download_hash"] = new JObject();
+
+ var download_hashJson = (JObject)json["download_hash"];
+ download_hashJson.SafeAdd("sha1", _fileService.GetFileHashSha1(file));
+ download_hashJson.SafeAdd("sha256", _fileService.GetFileHashSha256(file));
+
+ json["download_content_type"] = _fileService.GetMimetype(file);
}
Log.DebugFormat("Transformed metadata:{0}{1}", Environment.NewLine, json);
diff --git a/Netkan/Transformers/NetkanTransformer.cs b/Netkan/Transformers/NetkanTransformer.cs
index 32a684acd5..49e44b307b 100644
--- a/Netkan/Transformers/NetkanTransformer.cs
+++ b/Netkan/Transformers/NetkanTransformer.cs
@@ -39,7 +39,7 @@ bool prerelease
// This is the "default" VersionedOverrideTransformer for compatability with overrides that don't
// specify a before or after property.
new VersionedOverrideTransformer(before: new string[] { null }, after: new string[] { null }),
- new DownloadSizeTransformer(http, fileService),
+ new DownloadAttributeTransformer(http, fileService),
new GeneratedByTransformer(),
new OptimusPrimeTransformer(),
new StripNetkanMetadataTransformer(),
diff --git a/Netkan/Transformers/PropertySortTransformer.cs b/Netkan/Transformers/PropertySortTransformer.cs
index f272a3d9b8..ddda9a946a 100644
--- a/Netkan/Transformers/PropertySortTransformer.cs
+++ b/Netkan/Transformers/PropertySortTransformer.cs
@@ -40,6 +40,8 @@ internal sealed class PropertySortTransformer : ITransformer
{ "install", 22 },
{ "download", 23 },
{ "download_size", 24 },
+ { "download_hash", 25 },
+ { "download_content_type", 26 },
{ "x_generated_by", int.MaxValue }
};
diff --git a/Spec.md b/Spec.md
index e0d20c4033..5409bf0362 100644
--- a/Spec.md
+++ b/Spec.md
@@ -509,6 +509,27 @@ downloading from the `download` URL. It is recommended that this field is
only generated by automated tools (where it is encouraged),
and not filled in by hand.
+##### download_hash
+
+If supplied, `download_hash` is an object of hash digests. Currently
+SHA1 and SHA256 calculated hashes of the resulting file downloaded.
+It is recommended that this field is only generated by automated
+tools (where it is encouraged), and not filled in by hand.
+
+ "download_hash": {
+ "sha1": "1F4B3F21A77D4A302E3417A7C7A24A0B63740FC5",
+ "sha256": "E3B0C44298FC1C149AFBF4C8996FB92427AE41E4649B934CA495991B7852B855"
+ }
+
+##### download_content_type
+
+If supplied, `download_content_type` is the content type of the file
+downloaded from the `download` URL. It is recommended that this field is
+only generated by automated tools (where it is encouraged),
+and not filled in by hand.
+
+ "download_content_type": "application/zip"
+
#### Extensions
Any field starting with `x_` (an x, followed by an underscore) is considered
@@ -547,6 +568,7 @@ When used, the following fields will be auto-filled if not already present:
- `version`
- `download`
- `download_size`
+- `download_hash`
- `resources.homepage`
- `resources.spacedock`
- `resources.repository`
@@ -564,6 +586,7 @@ When used, the following fields will be auto-filled if not already present:
- `version`
- `download`
- `download_size`
+- `download_hash`
- `resources.repository`
Optionally, one asset `:filter_regexp` directive *may* be provided:
@@ -581,6 +604,7 @@ The following fields will be auto-filled if not already present:
- `version`
- `download`
- `download_size`
+- `download_hash`
- `resources.ci`
An `x_netkan_jenkins` field may be provided to customize how the metadata is fetched from the Jenkins server. It is
@@ -621,6 +645,8 @@ When used, the following fields will be auto-filled if not already present:
- `download`
- `download_size`
+- `download_hash`
+- `download_content_type`
This method depends on the existence of an AVC `.version` file in the download file
to determine:
@@ -756,7 +782,7 @@ The possible values of `before` and `after` are:
- `$none`
- `$all`
- `avc`
-- `download_size`
+- `download_attributes`
- `epoch`
- `forced_v`
- `generated_by`
diff --git a/Tests/Data/FileIdentifier/random.bin b/Tests/Data/FileIdentifier/random.bin
new file mode 100644
index 0000000000..068a9fa7cd
Binary files /dev/null and b/Tests/Data/FileIdentifier/random.bin differ
diff --git a/Tests/Data/FileIdentifier/test_gzip.gz b/Tests/Data/FileIdentifier/test_gzip.gz
new file mode 100644
index 0000000000..6cdf4db7f8
Binary files /dev/null and b/Tests/Data/FileIdentifier/test_gzip.gz differ
diff --git a/Tests/NetKAN/Services/FileServiceTests.cs b/Tests/NetKAN/Services/FileServiceTests.cs
index e34581b227..5da8943d5c 100644
--- a/Tests/NetKAN/Services/FileServiceTests.cs
+++ b/Tests/NetKAN/Services/FileServiceTests.cs
@@ -1,4 +1,5 @@
-using CKAN.NetKAN.Services;
+using System.IO;
+using CKAN.NetKAN.Services;
using NUnit.Framework;
using Tests.Data;
@@ -21,5 +22,127 @@ public void GetsFileSizeCorrectly()
"FileService should return the correct file size."
);
}
+
+ [Test]
+ public void GetsFileHashSha1Correctly()
+ {
+ // Arrange
+ var sut = new FileService();
+
+ // Act
+ var result = sut.GetFileHashSha1(TestData.DogeCoinFlagZip());
+
+ // Assert
+ Assert.That(result, Is.EqualTo("47B6ED5F502AD914744882858345BE030A29E1AA"),
+ "FileService should return the correct file SHA1 hash."
+ );
+ }
+
+ [Test]
+ public void GetsFileHashSha256Correctly()
+ {
+ // Arrange
+ var sut = new FileService();
+
+ // Act
+ var result = sut.GetFileHashSha256(TestData.DogeCoinFlagZip());
+
+ // Assert
+ Assert.That(result, Is.EqualTo("EC955DB772FBA8CAA62BF61C180D624C350D792C6F573D35A5EAEE3898DCF7C1"),
+ "FileService should return the correct file SHA256 hash."
+ );
+ }
+
+ [Test]
+ public void GetsAsciiMimeCorrectly()
+ {
+ // Arrange
+ var sut = new FileService();
+
+ // Act
+ var result = sut.GetMimetype(TestData.DataDir("FileIdentifier/test_ascii.txt"));
+
+ // Assert
+ Assert.That(result, Is.EqualTo("text/plain"),
+ "FileService should return the correct mimetype for a text file"
+ );
+ }
+
+ [Test]
+ public void GetsGzipMimeCorrectly()
+ {
+ // Arrange
+ var sut = new FileService();
+
+ // Act
+ var result = sut.GetMimetype(TestData.DataDir("FileIdentifier/test_gzip.gz"));
+
+ // Assert
+ Assert.That(result, Is.EqualTo("application/x-gzip"),
+ "FileService should return the correct mimetype for a gzip file."
+ );
+ }
+
+ [Test]
+ public void GetsTarMimeCorrectly()
+ {
+ // Arrange
+ var sut = new FileService();
+
+ // Act
+ var result = sut.GetMimetype(TestData.DataDir("FileIdentifier/test_tar.tar"));
+
+ // Assert
+ Assert.That(result, Is.EqualTo("application/x-tar"),
+ "FileService should return the correct mimetype for a tar file."
+ );
+ }
+
+ [Test]
+ public void GetsTarGzMimeCorrectly()
+ {
+ // Arrange
+ var sut = new FileService();
+
+ // Act
+ var result = sut.GetMimetype(TestData.DataDir("FileIdentifier/test_targz.tar.gz"));
+
+ // Assert
+ Assert.That(result, Is.EqualTo("application/x-compressed-tar"),
+ "FileService should return the correct mimetype for a tar.gz file."
+ );
+ }
+
+ [Test]
+ public void GetsZipMimeCorrectly()
+ {
+ // Arrange
+ var sut = new FileService();
+
+ // Act
+ var result = sut.GetMimetype(TestData.DogeCoinFlagZip());
+
+ // Assert
+ Assert.That(result, Is.EqualTo("application/zip"),
+ "FileService should return the correct mimetype for a zipfile."
+ );
+ }
+
+ [Test]
+ public void GetsUnknownMimeCorrectly()
+ {
+ // Arrange
+ var sut = new FileService();
+ string random_bin = TestData.DataDir("FileIdentifier/random.bin");
+ Assert.IsTrue(File.Exists(random_bin));
+
+ // Act
+ var result = sut.GetMimetype(random_bin);
+
+ // Assert
+ Assert.That(result, Is.EqualTo("application/octet-stream"),
+ "FileService should return 'application/octet-stream' for all other file types."
+ );
+ }
}
}
diff --git a/Tests/NetKAN/Transformers/DownloadSizeTransformerTests.cs b/Tests/NetKAN/Transformers/DownloadAttributeTransformerTests.cs
similarity index 51%
rename from Tests/NetKAN/Transformers/DownloadSizeTransformerTests.cs
rename to Tests/NetKAN/Transformers/DownloadAttributeTransformerTests.cs
index 06621112dc..13f80d8880 100644
--- a/Tests/NetKAN/Transformers/DownloadSizeTransformerTests.cs
+++ b/Tests/NetKAN/Transformers/DownloadAttributeTransformerTests.cs
@@ -1,6 +1,7 @@
-using System;
+using System;
using CKAN.NetKAN.Model;
using CKAN.NetKAN.Services;
+using CKAN.NetKAN.Extensions;
using CKAN.NetKAN.Transformers;
using Moq;
using Newtonsoft.Json.Linq;
@@ -9,13 +10,16 @@
namespace Tests.NetKAN.Transformers
{
[TestFixture]
- public sealed class DownloadSizeTransformerTests
+ public sealed class DownloadAttributeTransformerTests
{
[Test]
- public void AddsDownloadSize()
+ public void AddsDownloadAttributes()
{
// Arrange
const string downloadFilePath = "/DoesNotExist.zip";
+ const string downloadHashSha1 = "47B6ED5F502AD914744882858345BE030A29E1AA";
+ const string downloadHashSha256 = "EC955DB772FBA8CAA62BF61C180D624C350D792C6F573D35A5EAEE3898DCF7C1";
+ const string downloadMimetype = "application/zip";
const long downloadSize = 9001;
var mHttp = new Mock();
@@ -24,10 +28,19 @@ public void AddsDownloadSize()
mHttp.Setup(i => i.DownloadPackage(It.IsAny(), It.IsAny()))
.Returns(downloadFilePath);
+ mFileService.Setup(i => i.GetFileHashSha1(downloadFilePath))
+ .Returns(downloadHashSha1);
+
+ mFileService.Setup(i => i.GetFileHashSha256(downloadFilePath))
+ .Returns(downloadHashSha256);
+
mFileService.Setup(i => i.GetSizeBytes(downloadFilePath))
.Returns(downloadSize);
- var sut = new DownloadSizeTransformer(mHttp.Object, mFileService.Object);
+ mFileService.Setup(i => i.GetMimetype(downloadFilePath))
+ .Returns(downloadMimetype);
+
+ var sut = new DownloadAttributeTransformer(mHttp.Object, mFileService.Object);
var json = new JObject();
json["spec_version"] = 1;
@@ -38,8 +51,17 @@ public void AddsDownloadSize()
var transformedJson = result.Json();
// Assert
+ Assert.That((string)transformedJson["download_hash"]["sha1"], Is.EqualTo(downloadHashSha1),
+ "DownloadAttributeTransformer should add a 'sha1' property withing 'download_hash' equal to the sha1 of the file."
+ );
+ Assert.That((string)transformedJson["download_hash"]["sha256"], Is.EqualTo(downloadHashSha256),
+ "DownloadAttributeTransformer should add a 'sha256' property withing 'download_hash' equal to the sha256 of the file."
+ );
Assert.That((long)transformedJson["download_size"], Is.EqualTo(downloadSize),
- "DownloadSizeTransformer should add a download_size property equal to the size of the file in bytes."
+ "DownloadAttributeTransformer should add a download_size property equal to the size of the file in bytes."
+ );
+ Assert.That((string)transformedJson["download_content_type"], Is.EqualTo(downloadMimetype),
+ "DownloadAttributeTransformer should add a download_content_type property equal to the Mimetype of the file."
);
}
@@ -53,7 +75,7 @@ public void DoesNothingIfFileDoesNotExist()
mHttp.Setup(i => i.DownloadPackage(It.IsAny(), It.IsAny()))
.Returns((string)null);
- var sut = new DownloadSizeTransformer(mHttp.Object, mFileService.Object);
+ var sut = new DownloadAttributeTransformer(mHttp.Object, mFileService.Object);
var json = new JObject();
json["spec_version"] = 1;
@@ -65,7 +87,7 @@ public void DoesNothingIfFileDoesNotExist()
// Assert
Assert.That(transformedJson, Is.EqualTo(json),
- "DownloadSizeTransformer should do nothing if the file does not exist."
+ "DownloadAttributeTransformer should do nothing if the file does not exist."
);
}
@@ -76,7 +98,7 @@ public void DoesNothingIfDownloadDoesNotExist()
var mHttp = new Mock();
var mFileService = new Mock();
- var sut = new DownloadSizeTransformer(mHttp.Object, mFileService.Object);
+ var sut = new DownloadAttributeTransformer(mHttp.Object, mFileService.Object);
var json = new JObject();
json["spec_version"] = 1;
@@ -87,7 +109,7 @@ public void DoesNothingIfDownloadDoesNotExist()
// Assert
Assert.That(transformedJson, Is.EqualTo(json),
- "DownloadSizeTransformer should do nothing if the download property does not exist."
+ "DownloadAttributeTransformer should do nothing if the download property does not exist."
);
}
}
diff --git a/Tests/Tests.csproj b/Tests/Tests.csproj
index 3eec213fd1..bd66659395 100644
--- a/Tests/Tests.csproj
+++ b/Tests/Tests.csproj
@@ -148,7 +148,7 @@
-
+