Try #396:

.github/workflows/CI.yml

@@ -25,9 +25,9 @@ jobs:
         arch:
           - x64
         include:
-          # Add a 1.0 job just to make sure we still support it
+          # Add a job using the earliest version of Julia supported by this package
           - os: ubuntu-latest
-            version: 1.0.5
+            version: 1.3
             arch: x64
           # Add a 1.5 job because that's what Invenia actually uses
           - os: ubuntu-latest

Project.toml

@@ -7,6 +7,7 @@ version = "1.53.0"
 Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
 Compat = "34da2185-b29b-5c13-b0c7-acf172513d20"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
+Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
 GitHub = "bc5e4493-9b4d-5f90-b8aa-2b2bcaad7a26"
 HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
 IniFile = "83e8ac13-25f8-5344-8a64-a9f2b223428f"
@@ -32,7 +33,7 @@ OrderedCollections = "1"
 Retry = "0.3, 0.4"
 URIs = "1"
 XMLDict = "0.3, 0.4"
-julia = "1"
+julia = "1.3"
 
 [extras]
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"

src/AWS.jl

@@ -3,6 +3,7 @@ module AWS
 using Compat: Compat, @something
 using Base64
 using Dates
+using Downloads: Downloads, Downloader, Curl
 using HTTP
 using MbedTLS
 using Mocking
@@ -33,6 +34,7 @@ include("AWSMetadata.jl")
 
 include(joinpath("utilities", "request.jl"))
 include(joinpath("utilities", "sign.jl"))
+include(joinpath("utilities", "downloads_backend.jl"))
 
 
 using ..AWSExceptions

src/utilities/downloads_backend.jl

@@ -0,0 +1,88 @@
+struct DownloadsBackend <: AWS.AbstractBackend
+    downloader::Union{Nothing, Downloads.Downloader}
+end
+
+DownloadsBackend() = DownloadsBackend(nothing)
+
+const AWS_DOWNLOADER = Ref{Union{Nothing, Downloader}}(nothing)
+const AWS_DOWNLOAD_LOCK = ReentrantLock()
+
+# Here we mimic Download.jl's own setup for using a global downloader.
+# We do this to have our own downloader (separate from Downloads.jl's global downloader)
+# because we add a hook to avoid redirects in order to try to match the HTTPBackend's
+# implementation, and we don't want to mutate the global downloader from Downloads.jl.
+# https://github.com/JuliaLang/Downloads.jl/blob/84e948c02b8a0625552a764bf90f7d2ee97c949c/src/Downloads.jl#L293-L301
+function get_downloader(downloader=nothing)
+    lock(AWS_DOWNLOAD_LOCK) do
+        yield() # let other downloads finish
+        downloader isa Downloader && return
+        while true
+            downloader = AWS_DOWNLOADER[]
+            downloader isa Downloader && return
+            D = Downloader()
+            D.easy_hook = (easy, info) -> Curl.setopt(easy, Curl.CURLOPT_FOLLOWLOCATION, false)
+            AWS_DOWNLOADER[] = D
+        end
+    end
+    return downloader
+end
+
+
+function AWS._http_request(backend::DownloadsBackend, request)
+    # If we pass `output`, Downloads.jl will expect a message
+    # body in the response. Specifically, it sets
+    # <https://curl.se/libcurl/c/CURLOPT_NOBODY.html>
+    # only when we do not pass the `output` argument.
+    #
+    # When the method is `HEAD`, the response may have a Content-Length
+    # but not send any content back (which appears to be correct,
+    # <https://stackoverflow.com/a/18925736/12486544>).
+    # 
+    # Thus, if we did not set `CURLOPT_NOBODY`, and it gets a Content-Length
+    # back, it will hang waiting for that body.
+    # 
+    # Therefore, we do not pass an `output` when the `request_method` is `HEAD`.
+    if request.request_method != "HEAD"
+        output = IOBuffer()
+        output_arg = (; output=output)
+
+        # We set a callback so later on we know how to get the `body` back.
+        body_arg = () -> (; body = take!(output))
+    else
+        output_arg = NamedTuple()
+        body_arg = () -> NamedTuple()
+    end
+
+    # We pass an `input` only when we have content we wish to send.
+    if !isempty(request.content)
+        input = IOBuffer()
+        write(input, request.content)
+        seekstart(input)
+        input_arg = (; input=input)
+    else
+        input_arg = NamedTuple()
+    end
+
+    @repeat 4 try
+        downloader = @something(backend.downloader, get_downloader())
+        # set the hook so that we don't follow redirects. Only
+        # need to do this on per-request downloaders, because we
+        # set our global one with this hook already.
+        if backend.downloader !== nothing
+            downloader.easy_hook = (easy, info) -> Curl.setopt(easy, Curl.CURLOPT_FOLLOWLOCATION, false)
+        end
+        response = Downloads.request(request.url; input_arg..., output_arg...,
+                                    method = request.request_method,
+                                    request.headers, verbose=false, throw=true,
+                                    downloader)
+        http_response = HTTP.Response(response.status, response.headers; body_arg()..., request=nothing) 
+
+        if HTTP.iserror(http_response)
+            target = HTTP.resource(HTTP.URI(request.url))
+            throw(HTTP.StatusError(http_response.status, request.request_method, target, http_response))
+        end
+        return http_response
+    catch e
+        @delay_retry if ((isa(e, HTTP.StatusError) && AWS._http_status(e) >= 500) || isa(e, Downloads.RequestError)) end
+    end
+end

test/runtests.jl

@@ -34,6 +34,8 @@ function _now_formatted()
     return lowercase(Dates.format(now(Dates.UTC), dateformat"yyyymmdd\THHMMSSsss\Z"))
 end
 
+AWS.DEFAULT_BACKEND[] = AWS.DownloadsBackend()
+
 @testset "AWS.jl" begin
     include("AWS.jl")
     include("AWSCredentials.jl")