Skip to content
This repository has been archived by the owner on Jun 2, 2020. It is now read-only.

Commit

Permalink
Allow to fetch from URLs that contain allowed reserved delimiters
Browse files Browse the repository at this point in the history
Go has a bug where it does not include all allowed reserved sub-delimiters
As a workaround, a http.Get is extended to keep the paths un-escaped.

It is expected that URL are previously escaped, so there should be no 
negative side effect

For more information on bug: golang/go#5684
  • Loading branch information
richardiux committed Apr 2, 2015
1 parent 4e8182b commit 10134fc
Show file tree
Hide file tree
Showing 2 changed files with 53 additions and 1 deletion.
52 changes: 52 additions & 0 deletions fetcher/http/client.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
package http

import (
"net/http"
"net/url"
)

// A Client extends http.Client and patches incorrect URL path escaping
// for more information on bug: https://github.com/golang/go/issues/5684
type Client struct {
http.Client
}

// Get issues a GET to the specified URL. If the response is one of the
// following redirect codes, Get follows the redirect after calling the
// Client's CheckRedirect function.
//
// 301 (Moved Permanently)
// 302 (Found)
// 303 (See Other)
// 307 (Temporary Redirect)
//
// An error is returned if the Client's CheckRedirect function fails
// or if there was an HTTP protocol error. A non-2xx response doesn't
// cause an error.
//
// When err is nil, resp always contains a non-nil resp.Body.
// Caller should close resp.Body when done reading from it.
//
func (c *Client) Get(urlStr string) (resp *http.Response, err error) {
fu, err := url.Parse(urlStr)
if err != nil {
return nil, err
}
u := &url.URL{
Scheme: fu.Scheme,
Host: fu.Host,
Opaque: fu.Path,
}

req := &http.Request{
Method: "GET",
URL: u,
Proto: "HTTP/1.1",
ProtoMajor: 1,
ProtoMinor: 1,
Header: make(http.Header),
Host: u.Host,
}

return c.Do(req)
}
2 changes: 1 addition & 1 deletion fetcher/http/http.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ func (f *Fetcher) Fetch(url string, destination string) error {
if _, err := os.Stat(destination); os.IsNotExist(err) {
start := time.Now()

client := &http.Client{Transport: transport}
client := Client{http.Client{Transport: transport}}
resp, err := client.Get(url)

if err != nil {
Expand Down

0 comments on commit 10134fc

Please sign in to comment.