From a7646c081f543c3d05e53b9badead9ae9c91f752 Mon Sep 17 00:00:00 2001 From: Dwayne Bent Date: Wed, 15 Jul 2015 22:08:07 -0400 Subject: [PATCH] Consolidate network download code --- Core/Net/Curl.cs | 58 +++++++-- Core/Net/Net.cs | 76 ++++++++++-- Netkan/Program.cs | 36 +++--- Netkan/Services/CachingHttpService.cs | 123 +------------------ Netkan/Services/IHttpService.cs | 2 +- Netkan/Sources/Kerbalstuff/KerbalstuffApi.cs | 26 +--- 6 files changed, 136 insertions(+), 185 deletions(-) diff --git a/Core/Net/Curl.cs b/Core/Net/Curl.cs index ca3b88eba5..29e337a66a 100644 --- a/Core/Net/Curl.cs +++ b/Core/Net/Curl.cs @@ -1,5 +1,7 @@ using System; using System.IO; +using System.Linq; +using System.Reflection; using CurlSharp; using log4net; @@ -11,8 +13,8 @@ namespace CKAN /// public static class Curl { - private static bool init_complete = false; - private static readonly ILog log = LogManager.GetLogger(typeof (Curl)); + private static bool _initComplete; + private static readonly ILog Log = LogManager.GetLogger(typeof (Curl)); /// /// Has libcurl do all the work it needs to work correctly. @@ -20,13 +22,13 @@ public static class Curl /// public static void Init() { - if (init_complete) + if (_initComplete) { - log.Info("Curl init already performed, not running twice"); + Log.Info("Curl init already performed, not running twice"); return; } CurlSharp.Curl.GlobalInit(CurlInitFlag.All); - init_complete = true; + _initComplete = true; } /// @@ -36,7 +38,7 @@ public static void Init() public static void CleanUp() { CurlSharp.Curl.GlobalCleanup(); - init_complete = false; + _initComplete = false; } @@ -50,9 +52,9 @@ public static void CleanUp() /// Adapted from MultiDemo.cs in the curlsharp repo public static CurlEasy CreateEasy(string url, CurlWriteCallback wf) { - if (!init_complete) + if (!_initComplete) { - log.Warn("Curl environment not pre-initialised, performing non-threadsafe init."); + Log.Warn("Curl environment not pre-initialised, performing non-threadsafe init."); Init(); } @@ -75,6 +77,12 @@ public static CurlEasy CreateEasy(string url, CurlWriteCallback wf) easy.SslVerifyPeer = false; } + var caBundle = ResolveCurlCaBundle(); + if (caBundle != null) + { + easy.CaInfo = caBundle; + } + return easy; } @@ -98,6 +106,40 @@ public static CurlEasy CreateEasy(Uri url, FileStream stream) // here. return CreateEasy(url.OriginalString, stream); } + + /// + /// Resolves the location of the cURL CA bundle file to use. + /// + /// The absolute file path to the bundle file or null if none is found. + private static string ResolveCurlCaBundle() + { + const string caBundleFileName = "curl-ca-bundle.crt"; + const string ckanSubDirectoryName = "CKAN"; + + var bundle = new[] + { + // Working Directory + Environment.CurrentDirectory, + + // Executable Directory + Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), + + // %LOCALAPPDATA%/CKAN + Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), ckanSubDirectoryName), + + // %APPDATA%/CKAN + Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), ckanSubDirectoryName), + + // %PROGRAMDATA%/CKAN + Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.CommonApplicationData), ckanSubDirectoryName), + } + .Select(i => Path.Combine(i, caBundleFileName)) + .FirstOrDefault(File.Exists); + + Log.InfoFormat("Using curl-ca bundle: {0}", bundle ?? "(none)"); + + return bundle; + } } } diff --git a/Core/Net/Net.cs b/Core/Net/Net.cs index 189f656a30..a03ceebca1 100644 --- a/Core/Net/Net.cs +++ b/Core/Net/Net.cs @@ -1,10 +1,11 @@ using System; -using System.Net; using System.IO; +using System.Net; +using System.Text; using System.Text.RegularExpressions; using ChinhDo.Transactions; -using log4net; using CurlSharp; +using log4net; namespace CKAN { @@ -16,8 +17,8 @@ public class Net { public static string UserAgentString = "Mozilla/4.0 (compatible; CKAN)"; - private static readonly ILog log = LogManager.GetLogger(typeof (Net)); - private static TxFileManager file_transaction = new TxFileManager(); + private static readonly ILog Log = LogManager.GetLogger(typeof (Net)); + private static readonly TxFileManager FileTransaction = new TxFileManager(); /// /// Downloads the specified url, and stores it in the filename given. @@ -40,13 +41,12 @@ public static string Download(string url, string filename = null, IUser user = n // Generate a temporary file if none is provided. if (filename == null) { - filename = file_transaction.GetTempFileName(); + filename = FileTransaction.GetTempFileName(); } - log.DebugFormat("Downloading {0} to {1}", url, filename); + Log.DebugFormat("Downloading {0} to {1}", url, filename); - var agent = new WebClient(); - agent.Headers.Add("user-agent", UserAgentString); + var agent = MakeDefaultHttpClient(); try { @@ -54,8 +54,7 @@ public static string Download(string url, string filename = null, IUser user = n } catch (Exception ex) { - - log.InfoFormat("Download failed, trying with curlsharp..."); + Log.InfoFormat("Download failed, trying with curlsharp..."); try { @@ -71,7 +70,7 @@ public static string Download(string url, string filename = null, IUser user = n } else { - log.Debug("curlsharp download successful"); + Log.Debug("curlsharp download successful"); } } @@ -88,8 +87,8 @@ public static string Download(string url, string filename = null, IUser user = n // It's okay if this fails. try { - log.DebugFormat("Removing {0} after web error failure", filename); - file_transaction.Delete(filename); + Log.DebugFormat("Removing {0} after web error failure", filename); + FileTransaction.Delete(filename); } catch { @@ -108,5 +107,56 @@ public static string Download(string url, string filename = null, IUser user = n return filename; } + + public static string DownloadText(Uri url) + { + return DownloadText(url.OriginalString); + } + + public static string DownloadText(string url) + { + Log.DebugFormat("About to download {0}", url); + + var agent = MakeDefaultHttpClient(); + + try + { + return agent.DownloadString(url); + } + catch (Exception) + { + Log.InfoFormat("Download failed, trying with curlsharp..."); + + var content = string.Empty; + + var client = Curl.CreateEasy(url, delegate(byte[] buf, int size, int nmemb, object extraData) + { + content += Encoding.UTF8.GetString(buf); + return size * nmemb; + }); + + using (client) + { + var result = client.Perform(); + + if (result != CurlCode.Ok) + { + throw new Exception("Curl download failed with error " + result); + } + + Log.DebugFormat("Download from {0}:\n\n{1}", url, content); + + return content; + } + } + } + + private static WebClient MakeDefaultHttpClient() + { + var client = new WebClient(); + client.Headers.Add("User-Agent", UserAgentString); + + return client; + } } } \ No newline at end of file diff --git a/Netkan/Program.cs b/Netkan/Program.cs index 270786aa2e..d35b5fafe3 100644 --- a/Netkan/Program.cs +++ b/Netkan/Program.cs @@ -38,31 +38,29 @@ public static int Main(string[] args) var moduleService = new ModuleService(); var fileService = new FileService(); + var http = new CachingHttpService(FindCache(new KSPManager(new ConsoleUser(false)))); - using (var http = new CachingHttpService(FindCache(new KSPManager(new ConsoleUser(false))))) - { - var netkan = ReadNetkan(); - Log.Info("Finished reading input"); + var netkan = ReadNetkan(); + Log.Info("Finished reading input"); - new NetkanValidator().Validate(netkan); - Log.Info("Input successfully passed pre-validation"); + new NetkanValidator().Validate(netkan); + Log.Info("Input successfully passed pre-validation"); - var transformer = new NetkanTransformer( - http, - fileService, - moduleService, - Options.GitHubToken, - Options.PreRelease - ); + var transformer = new NetkanTransformer( + http, + fileService, + moduleService, + Options.GitHubToken, + Options.PreRelease + ); - var ckan = transformer.Transform(netkan); - Log.Info("Finished transformation"); + var ckan = transformer.Transform(netkan); + Log.Info("Finished transformation"); - new CkanValidator(netkan, http, moduleService).Validate(ckan); - Log.Info("Output successfully passed post-validation"); + new CkanValidator(netkan, http, moduleService).Validate(ckan); + Log.Info("Output successfully passed post-validation"); - WriteCkan(ckan); - } + WriteCkan(ckan); } else { diff --git a/Netkan/Services/CachingHttpService.cs b/Netkan/Services/CachingHttpService.cs index 5ca6601c8c..dd9f658512 100644 --- a/Netkan/Services/CachingHttpService.cs +++ b/Netkan/Services/CachingHttpService.cs @@ -1,142 +1,25 @@ using System; -using System.IO; -using System.Linq; -using System.Reflection; -using System.Text; -using CurlSharp; -using log4net; namespace CKAN.NetKAN.Services { - internal sealed partial class CachingHttpService : IHttpService + internal sealed class CachingHttpService : IHttpService { - private static readonly ILog Log = LogManager.GetLogger(typeof(CachingHttpService)); - private readonly NetFileCache _cache; - private CurlEasy _curl; public CachingHttpService(NetFileCache cache) { _cache = cache; - - CurlSharp.Curl.GlobalInit(CurlInitFlag.All); - - _curl = new CurlEasy { UserAgent = CKAN.Net.UserAgentString }; - - var caBundle = ResolveCurlCaBundle(); - if (caBundle != null) - { - _curl.CaInfo = caBundle; - } } public string DownloadPackage(Uri url, string identifier) { - EnsureNotDisposed(); - return _cache.GetCachedFilename(url) ?? - _cache.Store(url, CKAN.Net.Download(url), string.Format("netkan-{0}", identifier), move: true); + _cache.Store(url, Net.Download(url), string.Format("netkan-{0}", identifier), move: true); } public string DownloadText(Uri url) { - EnsureNotDisposed(); - - Log.DebugFormat("About to download {0}", url); - - var content = string.Empty; - - _curl.Url = url.ToString(); - _curl.WriteData = null; - _curl.WriteFunction = delegate(byte[] buf, int size, int nmemb, object extraData) - { - content += Encoding.UTF8.GetString(buf); - return size * nmemb; - }; - - var result = _curl.Perform(); - - if (result != CurlCode.Ok) - { - throw new Exception("Curl download failed with error " + result); - } - - Log.DebugFormat("Download from {0}:\n\n{1}", url, content); - - return content; - } - - /// - /// Resolves the location of the cURL CA bundle file to use. - /// - /// The absolute file path to the bundle file or null if none is found. - private static string ResolveCurlCaBundle() - { - const string caBundleFileName = "curl-ca-bundle.crt"; - const string ckanSubDirectoryName = "CKAN"; - - var bundle = new[] - { - // Working Directory - Environment.CurrentDirectory, - - // Executable Directory - Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), - - // %LOCALAPPDATA%/CKAN - Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), ckanSubDirectoryName), - - // %APPDATA%/CKAN - Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), ckanSubDirectoryName), - - // %PROGRAMDATA%/CKAN - Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.CommonApplicationData), ckanSubDirectoryName), - } - .Select(i => Path.Combine(i, caBundleFileName)) - .FirstOrDefault(File.Exists); - - Log.InfoFormat("Using curl-ca bundle: {0}",bundle ?? "(none)"); - - return bundle; - } - } - - internal sealed partial class CachingHttpService - { - private bool _isDisposed; - - ~CachingHttpService() - { - Dispose(disposing: false); - } - - public void Dispose() - { - Dispose(disposing: true); - GC.SuppressFinalize(this); - } - - private void Dispose(bool disposing) - { - if (disposing) - { - if (_curl != null) - { - CurlSharp.Curl.GlobalCleanup(); - _curl.Dispose(); - _curl = null; - } - } - - _isDisposed = true; - } - - private void EnsureNotDisposed() - { - if (_isDisposed) - { - throw new ObjectDisposedException(typeof(CachingHttpService).FullName); - } + return Net.DownloadText(url); } } } diff --git a/Netkan/Services/IHttpService.cs b/Netkan/Services/IHttpService.cs index 32660999ca..d3cace4ec2 100644 --- a/Netkan/Services/IHttpService.cs +++ b/Netkan/Services/IHttpService.cs @@ -2,7 +2,7 @@ namespace CKAN.NetKAN.Services { - internal interface IHttpService : IDisposable + internal interface IHttpService { string DownloadPackage(Uri url, string identifier); string DownloadText(Uri url); diff --git a/Netkan/Sources/Kerbalstuff/KerbalstuffApi.cs b/Netkan/Sources/Kerbalstuff/KerbalstuffApi.cs index b7fc46834d..fe75eb63dc 100644 --- a/Netkan/Sources/Kerbalstuff/KerbalstuffApi.cs +++ b/Netkan/Sources/Kerbalstuff/KerbalstuffApi.cs @@ -1,5 +1,4 @@ using System; -using System.Net; using System.Text.RegularExpressions; using CKAN.NetKAN.Services; using log4net; @@ -82,29 +81,8 @@ private string Call(string path) var url = KerbalstuffApiBase + path; Log.DebugFormat("Calling {0}", url); - try - { - return _http.DownloadText(new Uri(url)); - } - catch (DllNotFoundException) - { - //Curl is not installed. Curl is a workaround for a mono issue. - //TODO Richard - Once repos are merged go and check all Platform calls to see if they are mono checks - if (!Platform.IsWindows) throw; - //On mircrosft.net so try native code. - using (var web = new WebClient()) - { - try - { - return web.DownloadString(url); - } - catch (WebException webEx) - { - Log.ErrorFormat("WebException while accessing {0}: {1}", url, webEx); - throw; - } - } - } + + return _http.DownloadText(new Uri(url)); } } } \ No newline at end of file