Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Resume failed downloads #3666

Merged
merged 1 commit into from
Sep 13, 2022
Merged

Resume failed downloads #3666

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions Core/Extensions/IOExtensions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,5 +54,28 @@ public static DriveInfo GetDrive(this DirectoryInfo dir)
.OrderByDescending(dr => dr.RootDirectory.FullName.Length)
.FirstOrDefault();

/// <summary>
/// A version of Stream.CopyTo with progress updates.
/// </summary>
/// <param name="src">Stream from which to copy</param>
/// <param name="dest">Stream to which to copy</param>
/// <param name="progress">Callback to notify as we traverse the input, called with count of bytes received</param>
public static void CopyTo(this Stream src, Stream dest, IProgress<long> progress)
{
// CopyTo says its default buffer is 81920, but we want more than 1 update for a 100 KiB file
const int bufSize = 8192;
var buffer = new byte[bufSize];
long total = 0;
while (true)
{
var bytesRead = src.Read(buffer, 0, bufSize);
if (bytesRead == 0)
{
break;
}
dest.Write(buffer, 0, bytesRead);
progress.Report(total += bytesRead);
}
}
}
}
52 changes: 29 additions & 23 deletions Core/ModuleInstaller.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,20 +155,11 @@ public void InstallList(ICollection<CkanModule> modules, RelationshipResolverOpt

foreach (CkanModule module in modsToInstall)
{
User.RaiseMessage(" * {0}", Cache.DescribeAvailability(module));
if (!Cache.IsMaybeCachedZip(module))
{
User.RaiseMessage(" * {0} {1} ({2}, {3})",
module.name,
module.version,
module.download.Host,
CkanModule.FmtSize(module.download_size)
);
downloads.Add(module);
}
else
{
User.RaiseMessage(Properties.Resources.ModuleInstallerModuleCached, module.name, module.version);
}
}

if (ConfirmPrompt && !User.RaiseYesNoDialog("Continue?"))
Expand Down Expand Up @@ -1053,12 +1044,21 @@ public void Upgrade(IEnumerable<CkanModule> modules, IDownloader netAsyncDownloa
{
if (!Cache.IsMaybeCachedZip(module))
{
User.RaiseMessage(Properties.Resources.ModuleInstallerUpgradeInstallingUncached,
module.name,
module.version,
module.download.Host,
CkanModule.FmtSize(module.download_size)
);
var inProgressFile = new FileInfo(Cache.GetInProgressFileName(module));
if (inProgressFile.Exists)
{
User.RaiseMessage(Properties.Resources.ModuleInstallerUpgradeInstallingResuming,
module.name, module.version,
module.download.Host,
CkanModule.FmtSize(module.download_size - inProgressFile.Length));
}
else
{
User.RaiseMessage(Properties.Resources.ModuleInstallerUpgradeInstallingUncached,
module.name, module.version,
module.download.Host,
CkanModule.FmtSize(module.download_size));
}
}
else
{
Expand Down Expand Up @@ -1086,13 +1086,19 @@ public void Upgrade(IEnumerable<CkanModule> modules, IDownloader netAsyncDownloa
{
if (!Cache.IsMaybeCachedZip(module))
{
User.RaiseMessage(Properties.Resources.ModuleInstallerUpgradeUpgradingUncached,
module.name,
installed.version,
module.version,
module.download.Host,
CkanModule.FmtSize(module.download_size)
);
var inProgressFile = new FileInfo(Cache.GetInProgressFileName(module));
if (inProgressFile.Exists)
{
User.RaiseMessage(Properties.Resources.ModuleInstallerUpgradeUpgradingResuming,
module.name, installed.version, module.version,
module.download.Host, CkanModule.FmtSize(module.download_size - inProgressFile.Length));
}
else
{
User.RaiseMessage(Properties.Resources.ModuleInstallerUpgradeUpgradingUncached,
module.name, installed.version, module.version,
module.download.Host, CkanModule.FmtSize(module.download_size));
}
}
else
{
Expand Down
108 changes: 3 additions & 105 deletions Core/Net/Net.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,11 @@
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;

using Autofac;
using ChinhDo.Transactions.FileManager;
using log4net;

using CKAN.Configuration;

namespace CKAN
Expand All @@ -25,7 +27,7 @@ public class Net
private const int MaxRetries = 3;
private const int RetryDelayMilliseconds = 100;

private static readonly ILog log = LogManager.GetLogger(typeof(Net));
private static readonly ILog log = LogManager.GetLogger(typeof(Net));

public static readonly Dictionary<string, Uri> ThrottledHosts = new Dictionary<string, Uri>()
{
Expand Down Expand Up @@ -355,109 +357,5 @@ public static Uri GetRawUri(Uri remoteUri)
return remoteUri;
}
}

/// <summary>
/// A WebClient with some CKAN-sepcific adjustments:
/// - A user agent string (required by GitHub API policy)
/// - Sets the Accept header to a given MIME type (needed to get raw files from GitHub API)
/// - Times out after a specified amount of time in milliseconds, 100 000 milliseconds (=100 seconds) by default (https://stackoverflow.com/a/3052637)
/// - Handles permanent redirects to the same host without clearing the Authorization header (needed to get files from renamed GitHub repositories via API)
/// </summary>
private sealed class RedirectingTimeoutWebClient : WebClient
{
/// <summary>
/// Initialize our special web client
/// </summary>
/// <param name="timeout">Timeout for the request in milliseconds, defaulting to 100 000 (=100 seconds)</param>
/// <param name="mimeType">A mime type sent with the "Accept" header</param>
public RedirectingTimeoutWebClient(int timeout = 100000, string mimeType = "")
{
this.timeout = timeout;
this.mimeType = mimeType;
}

protected override WebRequest GetWebRequest(Uri address)
{
// Set user agent and MIME type for every request. including redirects
Headers.Add("User-Agent", UserAgentString);
if (!string.IsNullOrEmpty(mimeType))
{
log.InfoFormat("Setting MIME type {0}", mimeType);
Headers.Add("Accept", mimeType);
}
if (permanentRedirects.TryGetValue(address, out Uri redirUri))
{
// Obey a previously received permanent redirect
address = redirUri;
}
var request = base.GetWebRequest(address);
if (request is HttpWebRequest hwr)
{
// GitHub API tokens cannot be passed via auto-redirect
hwr.AllowAutoRedirect = false;
hwr.Timeout = timeout;
}
return request;
}

protected override WebResponse GetWebResponse(WebRequest request)
{
if (request == null)
return null;
var response = base.GetWebResponse(request);
if (response == null)
return null;

if (response is HttpWebResponse hwr)
{
int statusCode = (int)hwr.StatusCode;
var location = hwr.Headers["Location"];
if (statusCode >= 300 && statusCode <= 399 && location != null)
{
log.InfoFormat("Redirecting to {0}", location);
hwr.Close();
var redirUri = new Uri(request.RequestUri, location);
if (Headers.AllKeys.Contains("Authorization")
&& request.RequestUri.Host != redirUri.Host)
{
log.InfoFormat("Host mismatch, purging token for redirect");
Headers.Remove("Authorization");
}
// Moved or PermanentRedirect
if (statusCode == 301 || statusCode == 308)
{
permanentRedirects.Add(request.RequestUri, redirUri);
}
return GetWebResponse(GetWebRequest(redirUri));
}
}
return response;
}

private int timeout;
private string mimeType;
private static readonly Dictionary<Uri, Uri> permanentRedirects = new Dictionary<Uri, Uri>();
}

// HACK: The ancient WebClient doesn't support setting the request type to HEAD and WebRequest doesn't support
// setting the User-Agent header.
// Maybe one day we'll be able to use HttpClient (https://msdn.microsoft.com/en-us/library/system.net.http.httpclient%28v=vs.118%29.aspx)
private sealed class RedirectWebClient : WebClient
{
public RedirectWebClient()
{
Headers.Add("User-Agent", UserAgentString);
}

protected override WebRequest GetWebRequest(Uri address)
{
var webRequest = (HttpWebRequest)base.GetWebRequest(address);
webRequest.AllowAutoRedirect = false;

webRequest.Method = "HEAD";

return webRequest;
}
}
}
}
24 changes: 14 additions & 10 deletions Core/Net/NetAsyncDownloader.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,11 +32,14 @@ private class NetAsyncDownloaderDownloadPart
public Exception error;
public int lastProgressUpdateSize;

public event DownloadProgressChangedEventHandler Progress;
/// <summary>
/// Percentage, bytes received, total bytes to receive
/// </summary>
public event Action<int, long, long> Progress;
public event Action<object, AsyncCompletedEventArgs, string> Done;

private string mimeType;
private WebClient agent;
private ResumingWebClient agent;

public NetAsyncDownloaderDownloadPart(Net.DownloadTarget target)
{
Expand All @@ -51,7 +54,7 @@ public NetAsyncDownloaderDownloadPart(Net.DownloadTarget target)
public void Download(Uri url, string path)
{
ResetAgent();
agent.DownloadFileAsync(url, path);
agent.DownloadFileAsyncWithResume(url, path);
}

public void Abort()
Expand All @@ -61,7 +64,7 @@ public void Abort()

private void ResetAgent()
{
agent = new WebClient();
agent = new ResumingWebClient();

agent.Headers.Add("User-Agent", Net.UserAgentString);

Expand All @@ -86,7 +89,11 @@ private void ResetAgent()
// Forward progress and completion events to our listeners
agent.DownloadProgressChanged += (sender, args) =>
{
Progress?.Invoke(sender, args);
Progress?.Invoke(args.ProgressPercentage, args.BytesReceived, args.TotalBytesToReceive);
};
agent.DownloadProgress += (percent, bytesReceived, totalBytesToReceive) =>
{
Progress?.Invoke(percent, bytesReceived, totalBytesToReceive);
};
agent.DownloadFileCompleted += (sender, args) =>
{
Expand Down Expand Up @@ -163,11 +170,8 @@ private void DownloadModule(Net.DownloadTarget target)
dl.target.url.ToString().Replace(" ", "%20"));

// Schedule for us to get back progress reports.
dl.Progress += (sender, args) =>
FileProgressReport(index,
args.ProgressPercentage,
args.BytesReceived,
args.TotalBytesToReceive);
dl.Progress += (ProgressPercentage, BytesReceived, TotalBytesToReceive) =>
FileProgressReport(index, ProgressPercentage, BytesReceived, TotalBytesToReceive);

// And schedule a notification if we're done (or if something goes wrong)
dl.Done += (sender, args, etag) =>
Expand Down
29 changes: 14 additions & 15 deletions Core/Net/NetAsyncModulesDownloader.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
using System.IO;
using System.Linq;

using ChinhDo.Transactions.FileManager;
using log4net;

namespace CKAN
Expand Down Expand Up @@ -62,33 +61,28 @@ public void DownloadModules(IEnumerable<CkanModule> modules)
.Where(group => !currentlyActive.Contains(group.Key))
.ToDictionary(group => group.Key, group => group.First());

// Make sure we have enough space to download this stuff
var downloadSize = unique_downloads.Values.Select(m => m.download_size).Sum();
CKANPathUtils.CheckFreeSpace(new DirectoryInfo(new TxFileManager().GetTempDirectory()),
downloadSize,
Properties.Resources.NotEnoughSpaceToDownload);
// Make sure we have enough space to cache this stuff
cache.CheckFreeSpace(downloadSize);
// Make sure we have enough space to download and cache
cache.CheckFreeSpace(unique_downloads.Values
.Select(m => m.download_size)
.Sum());

this.modules.AddRange(unique_downloads.Values);

try
{
// Start the downloads!
downloader.DownloadAndWait(
unique_downloads.Select(item => new Net.DownloadTarget(
downloader.DownloadAndWait(unique_downloads
.Select(item => new Net.DownloadTarget(
item.Key,
item.Value.InternetArchiveDownload,
// Use a temp file name
null,
cache.GetInProgressFileName(item.Value),
item.Value.download_size,
// Send the MIME type to use for the Accept header
// The GitHub API requires this to include application/octet-stream
string.IsNullOrEmpty(item.Value.download_content_type)
? defaultMimeType
: $"{item.Value.download_content_type};q=1.0,{defaultMimeType};q=0.9"
)).ToList()
);
: $"{item.Value.download_content_type};q=1.0,{defaultMimeType};q=0.9"))
.ToList());
this.modules.Clear();
AllComplete?.Invoke();
}
Expand All @@ -104,8 +98,11 @@ public void DownloadModules(IEnumerable<CkanModule> modules)

private void ModuleDownloadComplete(Uri url, string filename, Exception error, string etag)
{
log.DebugFormat("Received download completion: {0}, {1}, {2}",
url, filename, error?.Message);
if (error != null)
{
// If there was an error in DOWNLOADING, keep the file so we can retry it later
log.Info(error.ToString());
}
else
Expand All @@ -120,6 +117,8 @@ private void ModuleDownloadComplete(Uri url, string filename, Exception error, s
catch (InvalidModuleFileKraken kraken)
{
User.RaiseError(kraken.ToString());
// If there was an error in STORING, delete the file so we can try it from scratch later
File.Delete(filename);
}
catch (FileNotFoundException e)
{
Expand Down
Loading