Skip to content

Commit

Permalink
Consolidate network download code
Browse files Browse the repository at this point in the history
  • Loading branch information
@dbent
dbent committed Jul 16, 2015
1 parent c259443 commit a7646c0
Show file tree
Hide file tree
Showing 6 changed files with 136 additions and 185 deletions.
58 changes: 50 additions & 8 deletions Core/Net/Curl.cs
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
using System;
using System.IO;
using System.Linq;
using System.Reflection;
using CurlSharp;
using log4net;

Expand All @@ -11,22 +13,22 @@ namespace CKAN
/// </summary>
public static class Curl
{
private static bool init_complete = false;
private static readonly ILog log = LogManager.GetLogger(typeof (Curl));
private static bool _initComplete;
private static readonly ILog Log = LogManager.GetLogger(typeof (Curl));

/// <summary>
/// Has libcurl do all the work it needs to work correctly.
/// NOT THREADSAFE AT ALL. Do this before forking any threads!
/// </summary>
public static void Init()
{
if (init_complete)
if (_initComplete)
{
log.Info("Curl init already performed, not running twice");
Log.Info("Curl init already performed, not running twice");
return;
}
CurlSharp.Curl.GlobalInit(CurlInitFlag.All);
init_complete = true;
_initComplete = true;
}

/// <summary>
Expand All @@ -36,7 +38,7 @@ public static void Init()
public static void CleanUp()
{
CurlSharp.Curl.GlobalCleanup();
init_complete = false;
_initComplete = false;
}


Expand All @@ -50,9 +52,9 @@ public static void CleanUp()
/// Adapted from MultiDemo.cs in the curlsharp repo
public static CurlEasy CreateEasy(string url, CurlWriteCallback wf)
{
if (!init_complete)
if (!_initComplete)
{
log.Warn("Curl environment not pre-initialised, performing non-threadsafe init.");
Log.Warn("Curl environment not pre-initialised, performing non-threadsafe init.");
Init();
}

Expand All @@ -75,6 +77,12 @@ public static CurlEasy CreateEasy(string url, CurlWriteCallback wf)
easy.SslVerifyPeer = false;
}

var caBundle = ResolveCurlCaBundle();
if (caBundle != null)
{
easy.CaInfo = caBundle;
}

return easy;
}

Expand All @@ -98,6 +106,40 @@ public static CurlEasy CreateEasy(Uri url, FileStream stream)
// here.
return CreateEasy(url.OriginalString, stream);
}

/// <summary>
/// Resolves the location of the cURL CA bundle file to use.
/// </summary>
/// <returns>The absolute file path to the bundle file or null if none is found.</returns>
private static string ResolveCurlCaBundle()
{
const string caBundleFileName = "curl-ca-bundle.crt";
const string ckanSubDirectoryName = "CKAN";

var bundle = new[]
{
// Working Directory
Environment.CurrentDirectory,

// Executable Directory
Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location),

// %LOCALAPPDATA%/CKAN
Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), ckanSubDirectoryName),

// %APPDATA%/CKAN
Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), ckanSubDirectoryName),

// %PROGRAMDATA%/CKAN
Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.CommonApplicationData), ckanSubDirectoryName),
}
.Select(i => Path.Combine(i, caBundleFileName))
.FirstOrDefault(File.Exists);

Log.InfoFormat("Using curl-ca bundle: {0}", bundle ?? "(none)");

return bundle;
}
}
}

76 changes: 63 additions & 13 deletions Core/Net/Net.cs
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
using System;
using System.Net;
using System.IO;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using ChinhDo.Transactions;
using log4net;
using CurlSharp;
using log4net;

namespace CKAN
{
Expand All @@ -16,8 +17,8 @@ public class Net
{
public static string UserAgentString = "Mozilla/4.0 (compatible; CKAN)";

private static readonly ILog log = LogManager.GetLogger(typeof (Net));
private static TxFileManager file_transaction = new TxFileManager();
private static readonly ILog Log = LogManager.GetLogger(typeof (Net));
private static readonly TxFileManager FileTransaction = new TxFileManager();

/// <summary>
/// Downloads the specified url, and stores it in the filename given.
Expand All @@ -40,22 +41,20 @@ public static string Download(string url, string filename = null, IUser user = n
// Generate a temporary file if none is provided.
if (filename == null)
{
filename = file_transaction.GetTempFileName();
filename = FileTransaction.GetTempFileName();
}

log.DebugFormat("Downloading {0} to {1}", url, filename);
Log.DebugFormat("Downloading {0} to {1}", url, filename);

var agent = new WebClient();
agent.Headers.Add("user-agent", UserAgentString);
var agent = MakeDefaultHttpClient();

try
{
agent.DownloadFile(url, filename);
}
catch (Exception ex)
{

log.InfoFormat("Download failed, trying with curlsharp...");
Log.InfoFormat("Download failed, trying with curlsharp...");

try
{
Expand All @@ -71,7 +70,7 @@ public static string Download(string url, string filename = null, IUser user = n
}
else
{
log.Debug("curlsharp download successful");
Log.Debug("curlsharp download successful");
}
}

Expand All @@ -88,8 +87,8 @@ public static string Download(string url, string filename = null, IUser user = n
// It's okay if this fails.
try
{
log.DebugFormat("Removing {0} after web error failure", filename);
file_transaction.Delete(filename);
Log.DebugFormat("Removing {0} after web error failure", filename);
FileTransaction.Delete(filename);
}
catch
{
Expand All @@ -108,5 +107,56 @@ public static string Download(string url, string filename = null, IUser user = n

return filename;
}

public static string DownloadText(Uri url)
{
return DownloadText(url.OriginalString);
}

public static string DownloadText(string url)
{
Log.DebugFormat("About to download {0}", url);

var agent = MakeDefaultHttpClient();

try
{
return agent.DownloadString(url);
}
catch (Exception)
{
Log.InfoFormat("Download failed, trying with curlsharp...");

var content = string.Empty;

var client = Curl.CreateEasy(url, delegate(byte[] buf, int size, int nmemb, object extraData)
{
content += Encoding.UTF8.GetString(buf);
return size * nmemb;
});

using (client)
{
var result = client.Perform();

if (result != CurlCode.Ok)
{
throw new Exception("Curl download failed with error " + result);
}

Log.DebugFormat("Download from {0}:\n\n{1}", url, content);

return content;
}
}
}

private static WebClient MakeDefaultHttpClient()
{
var client = new WebClient();
client.Headers.Add("User-Agent", UserAgentString);

return client;
}
}
}
36 changes: 17 additions & 19 deletions Netkan/Program.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,31 +38,29 @@ public static int Main(string[] args)

var moduleService = new ModuleService();
var fileService = new FileService();
var http = new CachingHttpService(FindCache(new KSPManager(new ConsoleUser(false))));

using (var http = new CachingHttpService(FindCache(new KSPManager(new ConsoleUser(false)))))
{
var netkan = ReadNetkan();
Log.Info("Finished reading input");
var netkan = ReadNetkan();
Log.Info("Finished reading input");

new NetkanValidator().Validate(netkan);
Log.Info("Input successfully passed pre-validation");
new NetkanValidator().Validate(netkan);
Log.Info("Input successfully passed pre-validation");

var transformer = new NetkanTransformer(
http,
fileService,
moduleService,
Options.GitHubToken,
Options.PreRelease
);
var transformer = new NetkanTransformer(
http,
fileService,
moduleService,
Options.GitHubToken,
Options.PreRelease
);

var ckan = transformer.Transform(netkan);
Log.Info("Finished transformation");
var ckan = transformer.Transform(netkan);
Log.Info("Finished transformation");

new CkanValidator(netkan, http, moduleService).Validate(ckan);
Log.Info("Output successfully passed post-validation");
new CkanValidator(netkan, http, moduleService).Validate(ckan);
Log.Info("Output successfully passed post-validation");

WriteCkan(ckan);
}
WriteCkan(ckan);
}
else
{
Expand Down
Loading

0 comments on commit a7646c0

Please sign in to comment.