diff --git a/google/cloud/internal/curl_handle.cc b/google/cloud/internal/curl_handle.cc index 9d9751e5b05e7..a59005eb39f50 100644 --- a/google/cloud/internal/curl_handle.cc +++ b/google/cloud/internal/curl_handle.cc @@ -370,7 +370,24 @@ Status CurlHandle::AsStatus(CURLcode e, char const* where) { case CURLE_LDAP_INVALID_URL: case CURLE_FILESIZE_EXCEEDED: case CURLE_USE_SSL_FAILED: + code = StatusCode::kUnknown; + break; + case CURLE_SEND_FAIL_REWIND: + // We use curl callbacks to send data in PUT and POST requests. libcurl + // may need to "rewind" the data. The documentation for the error is + // sparse, but the documentation for the "rewind" callbacks goes into + // more detail: + // https://curl.se/libcurl/c/CURLOPT_SEEKFUNCTION.html + // This may happen when doing an HTTP PUT or POST with a multi-pass + // authentication method, or when an existing HTTP connection is + // reused too late and the server closes the connection. + // + // All these cases seem retryable, though it seems more efficient to + // implement the rewind callback. + code = StatusCode::kUnavailable; + break; + case CURLE_SSL_ENGINE_SETFAILED: case CURLE_LOGIN_DENIED: case CURLE_TFTP_NOTFOUND: diff --git a/google/cloud/internal/curl_impl.cc b/google/cloud/internal/curl_impl.cc index a7c4ef55b8f62..0468ea85a3092 100644 --- a/google/cloud/internal/curl_impl.cc +++ b/google/cloud/internal/curl_impl.cc @@ -25,6 +25,8 @@ #include "absl/strings/match.h" #include "absl/strings/strip.h" #include +#include +#include #include #include @@ -89,10 +91,7 @@ Status AsStatus(CURLMcode result, char const* where) { class WriteVector { public: explicit WriteVector(std::vector> v) - : writev_(std::move(v)) { - // Reverse the vector so the first chunk is at the end. - std::reverse(writev_.begin(), writev_.end()); - } + : original_(std::move(v)), writev_(original_.begin(), original_.end()) {} std::size_t size() const { std::size_t size = 0; @@ -105,7 +104,7 @@ class WriteVector { std::size_t MoveTo(absl::Span dst) { auto const avail = dst.size(); while (!writev_.empty()) { - auto& src = writev_.back(); + auto& src = writev_.front(); if (src.size() > dst.size()) { std::copy(src.begin(), src.begin() + dst.size(), dst.begin()); src.remove_prefix(dst.size()); @@ -114,15 +113,74 @@ class WriteVector { } std::copy(src.begin(), src.end(), dst.begin()); dst.remove_prefix(src.size()); - writev_.pop_back(); + writev_.pop_front(); } return avail - dst.size(); } + /// Implements a CURLOPT_SEEKFUNCTION callback. + /// + /// @see https://curl.se/libcurl/c/CURLOPT_SEEKFUNCTION.html + /// @returns true if the seek operation was successful. + bool Seek(std::size_t offset, int origin) { + // libcurl claims to only req + if (origin != SEEK_SET) return false; + writev_.assign(original_.begin(), original_.end()); + // Reverse the vector so the first chunk is at the end. + std::reverse(writev_.begin(), writev_.end()); + while (!writev_.empty()) { + auto& src = writev_.front(); + if (src.size() >= offset) { + src.remove_prefix(offset); + offset = 0; + break; + } + offset -= src.size(); + writev_.pop_front(); + } + return offset == 0; + } + private: - std::vector> writev_; + std::vector> original_; + std::deque> writev_; }; +extern "C" { // libcurl callbacks + +// It would be nice to be able to send data from, and receive data into, +// our own buffers (i.e., without an extra copy). But, there is no such API. + +// Receive response data from peer. +std::size_t WriteFunction(char* ptr, size_t size, size_t nmemb, + void* userdata) { + auto* const request = reinterpret_cast(userdata); + return request->WriteCallback(absl::MakeSpan(ptr, size * nmemb)); +} + +// Receive a response header from peer. +std::size_t HeaderFunction(char* buffer, std::size_t size, std::size_t nitems, + void* userdata) { + auto* const request = reinterpret_cast(userdata); + return request->HeaderCallback(absl::MakeSpan(buffer, size * nitems)); +} + +// Fill buffer to send data to peer (POST/PUT). +std::size_t ReadFunction(char* buffer, std::size_t size, std::size_t nitems, + void* userdata) { + auto* const writev = reinterpret_cast(userdata); + return writev->MoveTo(absl::MakeSpan(buffer, size * nitems)); +} + +int SeekFunction(void* userdata, curl_off_t offset, int origin) { + auto* const writev = reinterpret_cast(userdata); + return writev->Seek(static_cast(offset), origin) + ? CURL_SEEKFUNC_OK + : CURL_SEEKFUNC_FAIL; +} + +} // extern "C" + } // namespace std::size_t SpillBuffer::CopyFrom(absl::Span src) { @@ -166,34 +224,6 @@ std::size_t SpillBuffer::MoveTo(absl::Span dst) { return len; } -extern "C" { // libcurl callbacks - -// It would be nice to be able to send data from, and receive data into, -// our own buffers (i.e., without an extra copy). But, there is no such API. - -// Receive response data from peer. -static std::size_t WriteFunction(char* ptr, size_t size, size_t nmemb, - void* userdata) { - auto* const request = reinterpret_cast(userdata); - return request->WriteCallback(absl::MakeSpan(ptr, size * nmemb)); -} - -// Receive a response header from peer. -static std::size_t HeaderFunction(char* buffer, std::size_t size, - std::size_t nitems, void* userdata) { - auto* const request = reinterpret_cast(userdata); - return request->HeaderCallback(absl::MakeSpan(buffer, size * nitems)); -} - -// Fill buffer to send data to peer (POST/PUT). -static std::size_t ReadFunction(char* buffer, std::size_t size, - std::size_t nitems, void* userdata) { - auto* const writev = reinterpret_cast(userdata); - return writev->MoveTo(absl::MakeSpan(buffer, size * nitems)); -} - -} // extern "C" - CurlImpl::CurlImpl(CurlHandle handle, std::shared_ptr factory, Options const& options) @@ -386,6 +416,10 @@ Status CurlImpl::MakeRequest(HttpMethod method, RestContext& context, if (!status.ok()) return OnTransferError(context, std::move(status)); status = handle_.SetOption(CURLOPT_READDATA, &writev); if (!status.ok()) return OnTransferError(context, std::move(status)); + status = handle_.SetOption(CURLOPT_SEEKFUNCTION, &SeekFunction); + if (!status.ok()) return OnTransferError(context, std::move(status)); + status = handle_.SetOption(CURLOPT_SEEKDATA, &writev); + if (!status.ok()) return OnTransferError(context, std::move(status)); SetHeader("Expect:"); return MakeRequestImpl(context); } @@ -397,6 +431,10 @@ Status CurlImpl::MakeRequest(HttpMethod method, RestContext& context, if (!status.ok()) return OnTransferError(context, std::move(status)); status = handle_.SetOption(CURLOPT_READDATA, &writev); if (!status.ok()) return OnTransferError(context, std::move(status)); + status = handle_.SetOption(CURLOPT_SEEKFUNCTION, &SeekFunction); + if (!status.ok()) return OnTransferError(context, std::move(status)); + status = handle_.SetOption(CURLOPT_SEEKDATA, &writev); + if (!status.ok()) return OnTransferError(context, std::move(status)); status = handle_.SetOption(CURLOPT_UPLOAD, 1L); if (!status.ok()) return OnTransferError(context, std::move(status)); status = handle_.SetOption(CURLOPT_INFILESIZE_LARGE, size);