Try #396:

Project.toml

@@ -7,6 +7,7 @@ version = "1.48.0"
 Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
 Compat = "34da2185-b29b-5c13-b0c7-acf172513d20"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
+Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
 GitHub = "bc5e4493-9b4d-5f90-b8aa-2b2bcaad7a26"
 HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
 IniFile = "83e8ac13-25f8-5344-8a64-a9f2b223428f"

src/AWS.jl

@@ -3,6 +3,7 @@ module AWS
 using Compat: Compat, @something
 using Base64
 using Dates
+using Downloads: Downloads, Downloader, Curl
 using HTTP
 using MbedTLS
 using Mocking

src/utilities/request.jl

@@ -1,3 +1,9 @@
+abstract type AbstractBackend end
+struct HTTPBackend <: AbstractBackend end
+struct DownloadsBackend <: AbstractBackend end
+
+default_backend() = DownloadsBackend()
+
 Base.@kwdef mutable struct Request
     service::String
     api_version::String
@@ -13,6 +19,80 @@ Base.@kwdef mutable struct Request
     http_options::AbstractDict{Symbol,<:Any}=LittleDict{Symbol,String}()
     return_raw::Bool=false
     response_dict_type::Type{<:AbstractDict}=LittleDict
+    downloader::Union{Nothing, Downloads.Downloader}=nothing
+    backend::AbstractBackend=default_backend()
+end
+
+submit_request(aws::AbstractAWSConfig, request::Request; return_headers::Bool=false) = submit_request(request.backend, aws, request; return_headers)
+
+const AWS_DOWNLOADER = Ref{Union{Nothing, Downloader}}(nothing)
+const AWS_DOWNLOAD_LOCK = ReentrantLock()
+
+# https://github.com/JuliaLang/Downloads.jl/blob/84e948c02b8a0625552a764bf90f7d2ee97c949c/src/Downloads.jl#L293-L301
+function get_downloader(downloader=nothing)
+    lock(AWS_DOWNLOAD_LOCK) do
+        yield() # let other downloads finish
+        downloader isa Downloader && return
+        while true
+            downloader = AWS_DOWNLOADER[]
+            downloader isa Downloader && return
+            AWS_DOWNLOADER[] = Downloader()
+        end
+    end
+    return downloader
+end
+
+function _http_request(::DownloadsBackend, request)
+    # If we pass `output`, Downloads.jl will expect a message
+    # body in the response. Specifically, it sets
+    # <https://curl.se/libcurl/c/CURLOPT_NOBODY.html>
+    # only when we do not pass the `output` argument.
+    #
+    # When the method is `HEAD`, the response may have a Content-Length
+    # but not send any content back (which appears to be correct,
+    # <https://stackoverflow.com/a/18925736/12486544>).
+    # 
+    # Thus, if we did not set `CURLOPT_NOBODY`, and it gets a Content-Length
+    # back, it will hang waiting for that body.
+    # 
+    # Therefore, we do not pass an `output` when the `request_method` is `HEAD`.
+    if request.request_method != "HEAD"
+        output = IOBuffer()
+        output_arg = (; output=output)
+
+        # We set a callback so later on we know how to get the `body` back.
+        body_arg = () -> (; body = take!(output))
+    else
+        output_arg = NamedTuple()
+        body_arg = () -> NamedTuple()
+    end
+
+    # We pass an `input` only when we have content we wish to send.
+    if !isempty(request.content)
+        input = IOBuffer()
+        write(input, request.content)
+        input_arg = (; input=input)
+    else
+        input_arg = NamedTuple()
+    end
+
+    downloader = @something(request.downloader, get_downloader())
+    # set the hook so that we don't follow redirects
+    downloader.easy_hook = (easy, info) -> Curl.setopt(easy, Curl.CURLOPT_FOLLOWLOCATION, false)
+
+
+    response = Downloads.request(request.url; input_arg..., output_arg...,
+                                 method = request.request_method,
+                                 request.headers, verbose=true, throw=true,
+                                 downloader)
+    http_response = HTTP.Response(response.status, response.headers; body_arg()..., request=nothing) 
+
+    if HTTP.iserror(http_response)
+        target = HTTP.resource(HTTP.URI(request.url))
+        throw(HTTP.StatusError(http_response.status, request.request_method, target, http_response))
+    end
+
+    return http_response
 end
 
 
@@ -31,7 +111,7 @@ Submit the request to AWS.
 # Returns
 - `Tuple or Dict`: Tuple if returning_headers, otherwise just return a Dict of the response body
 """
-function submit_request(aws::AbstractAWSConfig, request::Request; return_headers::Bool=false)
+function submit_request(backend::AbstractBackend, aws::AbstractAWSConfig, request::Request; return_headers::Bool=false)
     response = nothing
     TOO_MANY_REQUESTS = 429
     EXPIRED_ERROR_CODES = ["ExpiredToken", "ExpiredTokenException", "RequestExpired"]
@@ -54,7 +134,7 @@ function submit_request(aws::AbstractAWSConfig, request::Request; return_headers
     @repeat 3 try
         credentials(aws) === nothing || sign!(aws, request)
 
-        response = @mock _http_request(request)
+        response = @mock _http_request(backend, request)
 
         if response.status in REDIRECT_ERROR_CODES
             if HTTP.header(response, "Location") != ""
@@ -141,7 +221,7 @@ function submit_request(aws::AbstractAWSConfig, request::Request; return_headers
 end
 
 
-function _http_request(request::Request)
+function _http_request(::HTTPBackend, request::Request)
     @repeat 4 try
         http_stack = HTTP.stack(redirect=false, retry=false, aws_authorization=false)
 

src/utilities/utilities.jl

@@ -68,6 +68,8 @@ function _extract_common_kw_args(service, args)
         headers=LittleDict{String, String}(_pop!(args, "headers", [])),
         http_options=_pop!(args, "http_options", LittleDict{Symbol, String}()),
         response_dict_type=_pop!(args, "response_dict_type", LittleDict),
+        downloader=_pop!(args, "downloader", nothing),
+        backend=_pop!(args, "backend", default_backend()),
     )
 end