Skip to content

Commit

Permalink
Merge pull request #296 from pjf/netcache
Browse files Browse the repository at this point in the history 
Cache entirely based upon URLs.
  • Loading branch information
@AlexanderDzhoganov
AlexanderDzhoganov committed Nov 9, 2014
2 parents 2c1e139 + 2312968 commit dbe344d
Show file tree
Hide file tree
Showing 15 changed files with 541 additions and 195 deletions.
2 changes: 1 addition & 1 deletion CKAN/CKAN/CKAN.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@
</Reference>
</ItemGroup>
<ItemGroup>
<Compile Include="NetFileCache.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="Module.cs" />
<Compile Include="KSP.cs" />
Expand All @@ -69,7 +70,6 @@
<Compile Include="KSPPathUtils.cs" />
<Compile Include="KSPPathConstants.cs" />
<Compile Include="NetAsyncDownloader.cs" />
<Compile Include="Cache.cs" />
<Compile Include="Registry\AvailableModule.cs" />
<Compile Include="Registry\InstalledModule.cs" />
<Compile Include="Registry\Registry.cs" />
Expand Down
87 changes: 0 additions & 87 deletions CKAN/CKAN/Cache.cs
View file

This file was deleted.

6 changes: 3 additions & 3 deletions CKAN/CKAN/KSP.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@ public class KSP

private string gamedir;
private KSPVersion version;
private Cache _Cache;
public Cache Cache
private NetFileCache _Cache;
public NetFileCache Cache
{
get
{
Expand All @@ -36,7 +36,7 @@ public KSP(string directory)

gamedir = directory;
Init();
_Cache = new Cache(DownloadCacheDir());
_Cache = new NetFileCache(DownloadCacheDir());
}

public string GameDir()
Expand Down
107 changes: 65 additions & 42 deletions CKAN/CKAN/ModuleInstaller.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Security.Cryptography;
using System.Text.RegularExpressions;
using System.Threading;
Expand Down Expand Up @@ -41,7 +42,7 @@ public class ModuleInstaller
private bool installCanceled = false; // Used for inter-thread communication.

// Our own cache is that of the KSP instance we're using.
public Cache Cache
public NetFileCache Cache
{
get
{
Expand Down Expand Up @@ -90,13 +91,13 @@ public string Download(Uri url, string filename)
/// <summary>
/// Downloads the given mod to the cache. Returns the filename it was saved to.
/// </summary>
public static string Download(Uri url, string filename, Cache cache)
public static string Download(Uri url, string filename, NetFileCache cache)
{
log.Info("Downloading " + filename);

string full_path = cache.CachePath(filename);
string tmp_file = Net.Download(url);

return Net.Download(url, full_path);
return cache.Store(url, tmp_file, move: true);
}

/// <summary>
Expand Down Expand Up @@ -130,39 +131,42 @@ public string CachedOrDownload(string identifier, Version version, Uri url, stri
/// If no filename is provided, the module's standard name will be used.
/// Chcecks provided cache first.
/// </summary>
public static string CachedOrDownload(string identifier, Version version, Uri url, Cache cache, string filename = null)
public static string CachedOrDownload(string identifier, Version version, Uri url, NetFileCache cache, string filename = null)
{
if (filename == null)
{
filename = CkanModule.StandardName(identifier, version);
}

string full_path = cache.CachedFile(filename);

if (full_path != null)
string full_path;
if (!cache.IsCached(url, out full_path))
{
log.DebugFormat("Using {0} (cached)", filename);
return full_path;
return Download(url, filename, cache);
}

return Download(url, filename, cache);
log.DebugFormat("Using {0} (cached)", filename);
return full_path;
}

/// <summary>
/// Downloads all the modules specified to the cache.
/// Even if modules share download URLs, they will only be downloaded once.
/// </summary>
public NetAsyncDownloader DownloadAsync(CkanModule[] modules)
{
var urls = new Uri[modules.Length];
var fullPaths = new string[modules.Length];
var unique_downloads = new Dictionary<Uri, CkanModule>();

for (int i = 0; i < modules.Length; i++)
// Walk through all our modules, but only keep the first of each
// one that has a unique download path.
foreach (var module in modules)
{
fullPaths[i] = ksp.Cache.CachePath(modules[i]);
urls[i] = modules[i].download;
if (!unique_downloads.ContainsKey(module.download))
{
unique_downloads[module.download] = module;
}
}

downloader = new NetAsyncDownloader(urls, fullPaths);
downloader = new NetAsyncDownloader(unique_downloads.Keys.ToArray());

if (onReportProgress != null)
{
Expand All @@ -172,21 +176,11 @@ public NetAsyncDownloader DownloadAsync(CkanModule[] modules)
percent);
}

downloader.onCompleted = (_uris, strings, errors) => OnDownloadsComplete(_uris, fullPaths, modules, errors);
downloader.onCompleted = (_uris, paths, errors) => OnDownloadsComplete(_uris, paths, unique_downloads.Values.ToArray(), errors);

return downloader;
}

/// <summary>
/// Downloads all the modules specified to the cache.
/// </summary>
public NetAsyncDownloader DownloadAsync(List<CkanModule> modules)
{
var mod_array = new CkanModule[modules.Count];
modules.CopyTo(mod_array);
return DownloadAsync(mod_array);
}

/// <summary>
/// Installs all modules given a list of identifiers as a transaction. Resolves dependencies.
/// This *will* save the registry at the end of operation.
Expand All @@ -211,14 +205,15 @@ public void InstallList(List<string> modules, RelationshipResolverOptions option

foreach (CkanModule module in modsToInstall)
{
if (Cache.IsCached(module))
string filename;
if (!KSPManager.CurrentInstance.Cache.IsCached(module.download, out filename))
{
User.WriteLine(" * {0} (cached)", module);
User.WriteLine(" * {0}", module);
downloads.Add(module);
}
else
{
User.WriteLine(" * {0}", module);
downloads.Add(module);
User.WriteLine(" * {0} (cached)", module);
}
}

Expand All @@ -243,9 +238,10 @@ public void InstallList(List<string> modules, RelationshipResolverOptions option

if (downloads.Count > 0)
{
downloader = DownloadAsync(downloads);
downloader = DownloadAsync(downloads.ToArray());
downloader.StartDownload();

// Wait for our downloads to finish.
lock (downloader)
{
Monitor.Wait(downloader);
Expand Down Expand Up @@ -303,26 +299,44 @@ public void CancelInstall()
installCanceled = true;
}

/// <summary>
/// Stores all of our files in the cache once done.
/// Called by NetAsyncDownloader.
/// </summary>
private void OnDownloadsComplete(Uri[] urls, string[] filenames, CkanModule[] modules, Exception[] errors)
{
bool noErrors = false;
// XXX: What the hell should we be doing if we are called with nulls?

if (urls != null)
{
noErrors = true;

for (int i = 0; i < errors.Length; i++)
{
if (errors[i] != null)
{
noErrors = false;
User.Error("Failed to download \"{0}\" - error: {1}", urls[i], errors[i].Message);
// XXX: Shouldn't be be throwing an exception about now?
}
else
{
// Even if some of our downloads failed, we want to cache the
// ones which succeeded.
ksp.Cache.Store(urls[i], filenames[i], modules[i].StandardName());

}
}
}

lastDownloadSuccessful = noErrors;
// Finally, remove all our temp files.
// We probably *could* have used Store's integrated move function above, but if we managed
// to somehow get two URLs the same in our download set, that could cause right troubles!

foreach (string tmpfile in filenames)
{
log.DebugFormat("Cleaing up {0}", tmpfile);
file_transaction.Delete(tmpfile);
}

// Signal that we're done.
lock (downloader)
{
Monitor.Pulse(downloader);
Expand All @@ -338,9 +352,8 @@ private void OnDownloadsComplete(Uri[] urls, string[] filenames, CkanModule[] mo

public List<string> GetModuleContentsList(CkanModule module)
{
string filename = Cache.CachedFile(module);

if (filename == null)
string filename;
if (!KSPManager.CurrentInstance.Cache.IsCached(module.download, out filename))
{
return null;
}
Expand Down Expand Up @@ -630,6 +643,16 @@ internal static string TransformOutputName(string file, string outputName, strin
{
string leadingPathToRemove = KSPPathUtils.GetLeadingPathElements(file);

// Special-casing, if stanza.file is just "GameData" or "Ships", strip it.
// TODO: Do we need to do anything special for tutorials or GameRoot?
if (
leadingPathToRemove == string.Empty &&
(file == "GameData" || file == "Ships")
)
{
leadingPathToRemove = file;
}

// If there's a leading path to remove, then we have some extra work that
// needs doing...
if (leadingPathToRemove != string.Empty)
Expand All @@ -646,7 +669,7 @@ internal static string TransformOutputName(string file, string outputName, strin
// Strip off leading path name
outputName = Regex.Replace(outputName, leadingRegEx, "");
}

// Return our snipped, normalised, and ready to go output filename!
return KSPPathUtils.NormalizePath(
Path.Combine(installDir, outputName)
Expand Down
Loading

0 comments on commit dbe344d

Please sign in to comment.