diff --git a/w32/downloadpackage.task b/w32/downloadpackage.task index 0223e98cb0..6132131ac2 100644 --- a/w32/downloadpackage.task +++ b/w32/downloadpackage.task @@ -38,6 +38,9 @@ using System.Diagnostics; { public string filename; public int progress; + public long lastActivityTicks; + public Exception error; + public bool cancelled; } protected ManualResetEvent TaskCanceled { get; private set; } @@ -70,6 +73,8 @@ using System.Diagnostics; basedir = Path.GetFullPath(@"$(BaseDir)"); TaskCanceled = new ManualResetEvent(false); + ServicePointManager.SecurityProtocol |= SecurityProtocolType.Tls12; + bool result = true; //Log.LogMessage(MessageImportance.High, // "Checking for package \"" + package + "\"."); @@ -96,6 +101,8 @@ using System.Diagnostics; Log.LogMessage(MessageImportance.High, "Start downloading package \"" + package + "\"."); + bool downloaded = true; + using (var client = new System.Net.WebClient()) { Uri uri = new Uri(package); @@ -111,41 +118,12 @@ using System.Diagnostics; } else //if (!File.Exists(output)) // Uncomment to skip download if exists { - var syncObject = new State - { - filename = urifilename, - progress = -1 - }; - lock (syncObject) - { - client.DownloadFileCompleted += new AsyncCompletedEventHandler(DownloadFileCompleted); - client.DownloadProgressChanged += new DownloadProgressChangedEventHandler(DownloadProgressCallback); - client.DownloadFileAsync(uri, output, syncObject); - - while (!Monitor.Wait(syncObject, 1000)) - { - if (TaskCanceled.WaitOne(0)) - { - client.CancelAsync(); - Monitor.Wait(syncObject); - - if (File.Exists(output)) - { - Log.LogMessage(MessageImportance.High, - "Deleting incomplete file " + output + " for package \"" + package + "\"."); - File.Delete(output); - } - - Log.LogMessage(MessageImportance.High, - "Downloading canceled for package \"" + package + "\"."); - - break; - } - } - } + client.DownloadFileCompleted += new AsyncCompletedEventHandler(DownloadFileCompleted); + client.DownloadProgressChanged += new DownloadProgressChangedEventHandler(DownloadProgressCallback); + downloaded = Download(client, uri, output, urifilename); } - if (File.Exists(output)) + if (downloaded && File.Exists(output)) { // Successful download. @@ -181,7 +159,11 @@ using System.Diagnostics; } } - if (!TaskCanceled.WaitOne(0)) + if (!downloaded && !TaskCanceled.WaitOne(0)) + { + result = false; + } + else if (!TaskCanceled.WaitOne(0)) { Log.LogMessage(MessageImportance.High, "Downloading finished for package \"" + package + "\"."); @@ -190,7 +172,129 @@ using System.Diagnostics; m.ReleaseMutex(); - return true; + return result; + } + + // Every failure is retried -- an HTTP status of any kind, 404 and 403 + // included, a dropped or stalled connection, an empty file. The URL a sheet + // names is taken to be right, so a failure is something passing, and a + // missing file on a mirror or a release CDN is often exactly that. Only + // cancelling the build stops the retries early. + private static readonly int[] retryDelaysSeconds = { 2, 5, 10, 20, 30 }; + private const int stallTimeoutSeconds = 60; + + private bool Download(WebClient client, Uri uri, string output, string urifilename) + { + int attempts = retryDelaysSeconds.Length + 1; + string reason = null; + + for (int attempt = 1; attempt <= attempts; attempt++) + { + if (attempt > 1) + { + int delay = retryDelaysSeconds[attempt - 2]; + Log.LogWarning("Downloading package \"" + package + "\" failed (" + reason + "), attempt " + + (attempt - 1) + " of " + attempts + "; retrying in " + delay + " s."); + if (TaskCanceled.WaitOne(delay * 1000)) + { + Log.LogMessage(MessageImportance.High, + "Downloading canceled for package \"" + package + "\"."); + return false; + } + } + + reason = DownloadOnce(client, uri, output, urifilename); + if (reason == null) + { + return true; + } + + if (File.Exists(output)) + { + Log.LogMessage(MessageImportance.High, + "Deleting incomplete file " + output + " for package \"" + package + "\"."); + File.Delete(output); + } + + if (TaskCanceled.WaitOne(0)) + { + Log.LogMessage(MessageImportance.High, + "Downloading canceled for package \"" + package + "\"."); + return false; + } + } + + Log.LogError("Downloading package \"" + package + "\" failed after " + attempts + " attempts: " + reason + "."); + return false; + } + + // One attempt: null when the file arrived, otherwise why it did not. + private string DownloadOnce(WebClient client, Uri uri, string output, string urifilename) + { + var syncObject = new State + { + filename = urifilename, + progress = -1, + lastActivityTicks = DateTime.UtcNow.Ticks + }; + + lock (syncObject) + { + client.DownloadFileAsync(uri, output, syncObject); + + while (!Monitor.Wait(syncObject, 1000)) + { + if (TaskCanceled.WaitOne(0)) + { + client.CancelAsync(); + Monitor.Wait(syncObject); + return "canceled"; + } + + // DownloadFileAsync ignores WebRequest timeouts, so a connection + // that stops delivering would otherwise wait forever. + long idle = DateTime.UtcNow.Ticks - Interlocked.Read(ref syncObject.lastActivityTicks); + if (idle > TimeSpan.FromSeconds(stallTimeoutSeconds).Ticks) + { + client.CancelAsync(); + Monitor.Wait(syncObject); + return "no data for " + stallTimeoutSeconds + " s"; + } + } + } + + if (syncObject.error != null) + { + return Describe(syncObject.error); + } + if (syncObject.cancelled) + { + return "canceled"; + } + if (!File.Exists(output)) + { + return "no file written"; + } + if (new FileInfo(output).Length == 0) + { + return "0 bytes"; + } + return null; + } + + private static string Describe(Exception error) + { + var web = error as WebException; + if (web != null) + { + var http = web.Response as HttpWebResponse; + if (http != null) + { + return "HTTP " + (int)http.StatusCode + " " + http.StatusDescription; + } + return web.Status + ": " + web.Message; + } + return error.GetType().Name + ": " + error.Message; } private void Extract(string filename, string extracttofolder) @@ -244,10 +348,13 @@ using System.Diagnostics; private void DownloadFileCompleted(object sender, AsyncCompletedEventArgs e) { - lock (e.UserState) + var state = (State)e.UserState; + lock (state) { + state.error = e.Error; + state.cancelled = e.Cancelled; //releases blocked thread - Monitor.Pulse(e.UserState); + Monitor.Pulse(state); } } @@ -267,6 +374,8 @@ using System.Diagnostics; private void DownloadProgressCallback(object sender, DownloadProgressChangedEventArgs e) { + Interlocked.Exchange(ref ((State)e.UserState).lastActivityTicks, DateTime.UtcNow.Ticks); + if (((State)e.UserState).progress < e.ProgressPercentage) { ((State)e.UserState).progress = e.ProgressPercentage;