Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improving paths loading from file #483

Merged
merged 2 commits into from
Jan 10, 2022
Merged

Improving paths loading from file #483

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b936278
Improving paths loading from file
Mzack9999 Jan 10, 2022
6fbfaa4
improving invalid uri handling
Mzack9999 Jan 10, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 35 additions & 12 deletions cmd/integration-test/http.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,18 +12,19 @@ import (
)

var httpTestcases = map[string]testutils.TestCase{
"Standard HTTP GET Request": &standardHttpGet{},
"Standard HTTPS GET Request": &standardHttpGet{tls: true},
"Raw HTTP GET Request": &standardHttpGet{unsafe: true},
"Raw request with non standard rfc path via stdin": &standardHttpGet{unsafe: true, stdinPath: "/%invalid"},
"Raw request with non standard rfc path via cli flag": &standardHttpGet{unsafe: true, path: "/%invalid"},
"Regression test for: https://github.com/projectdiscovery/httpx/issues/363": &issue363{}, // infinite redirect
"Regression test for: https://github.com/projectdiscovery/httpx/issues/276": &issue276{}, // full path with port in output
"Regression test for: https://github.com/projectdiscovery/httpx/issues/277": &issue277{}, // scheme://host:port via stdin
"Regression test for: https://github.com/projectdiscovery/httpx/issues/303": &issue303{}, // misconfigured gzip header with uncompressed body
"Regression test for: https://github.com/projectdiscovery/httpx/issues/400": &issue400{}, // post operation with body
"Regression test for: https://github.com/projectdiscovery/httpx/issues/414": &issue414{}, // stream mode with path
"Regression test for: https://github.com/projectdiscovery/httpx/issues/433": &issue433{}, // new line scanning with title flag
"Standard HTTP GET Request": &standardHttpGet{},
"Standard HTTPS GET Request": &standardHttpGet{tls: true},
"Raw HTTP GET Request": &standardHttpGet{unsafe: true},
"Raw request with non standard rfc path via stdin": &standardHttpGet{unsafe: true, stdinPath: "/%invalid"},
"Raw request with non standard rfc path via cli flag": &standardHttpGet{unsafe: true, path: "/%invalid"},
"Regression test for: https://github.com/projectdiscovery/httpx/issues/363": &issue363{}, // infinite redirect
"Regression test for: https://github.com/projectdiscovery/httpx/issues/276": &issue276{}, // full path with port in output
"Regression test for: https://github.com/projectdiscovery/httpx/issues/277": &issue277{}, // scheme://host:port via stdin
"Regression test for: https://github.com/projectdiscovery/httpx/issues/303": &issue303{}, // misconfigured gzip header with uncompressed body
"Regression test for: https://github.com/projectdiscovery/httpx/issues/400": &issue400{}, // post operation with body
"Regression test for: https://github.com/projectdiscovery/httpx/issues/414": &issue414{}, // stream mode with path
"Regression test for: https://github.com/projectdiscovery/httpx/issues/433": &issue433{}, // new line scanning with title flag
"Request URI to existing file - https://github.com/projectdiscovery/httpx/issues/480": &issue480{}, // request uri pointing to existing file
}

type standardHttpGet struct {
Expand Down Expand Up @@ -264,3 +265,25 @@ func (h *issue433) Execute() error {
}
return nil
}

type issue480 struct{}

func (h *issue480) Execute() error {
var ts *httptest.Server
router := httprouter.New()
uriPath := "////////////////../../../../../../../../etc/passwd"
router.GET(uriPath, httprouter.Handle(func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
htmlResponse := "<html><body>ok from uri</body></html>"
fmt.Fprint(w, htmlResponse)
}))
ts = httptest.NewServer(router)
defer ts.Close()
results, err := testutils.RunHttpxAndGetResults(ts.URL, debug, "-path", "////////////////../../../../../../../../etc/passwd")
if err != nil {
return err
}
if !strings.Contains(results[0], uriPath) {
return errIncorrectResultsCount(results)
}
return nil
}
24 changes: 15 additions & 9 deletions runner/runner.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (
"net/url"
"os"
"path"
"path/filepath"
"reflect"
"regexp"
"sort"
Expand Down Expand Up @@ -265,8 +266,10 @@ func New(options *Options) (*Runner, error) {
}

func (r *Runner) prepareInputPaths() {
// most likely, the user would provide the most simplified path to an existing file
isAbsoluteOrRelativePath := filepath.Clean(r.options.RequestURIs) == r.options.RequestURIs
// Check if the user requested multiple paths
if fileutil.FileExists(r.options.RequestURIs) {
if isAbsoluteOrRelativePath && fileutil.FileExists(r.options.RequestURIs) {
r.options.requestURIs = fileutilz.LoadFile(r.options.RequestURIs)
} else if r.options.RequestURIs != "" {
r.options.requestURIs = strings.Split(r.options.RequestURIs, ",")
Expand Down Expand Up @@ -917,14 +920,17 @@ retry:
}
// fix the final output url
fullURL := req.URL.String()
parsedURL, _ := urlutil.Parse(fullURL)
if r.options.Unsafe {
parsedURL.RequestURI = reqURI
// if the full url doesn't end with the custom path we pick the original input value
} else if !stringsutil.HasSuffixAny(fullURL, scanopts.RequestURI) {
parsedURL.RequestURI = scanopts.RequestURI
}
fullURL = parsedURL.String()
if parsedURL, errParse := urlutil.Parse(fullURL); errParse != nil {
return Result{URL: URL.String(), Input: origInput, err: errParse}
} else {
if r.options.Unsafe {
parsedURL.RequestURI = reqURI
// if the full url doesn't end with the custom path we pick the original input value
} else if !stringsutil.HasSuffixAny(fullURL, scanopts.RequestURI) {
parsedURL.RequestURI = scanopts.RequestURI
}
fullURL = parsedURL.String()
}

if r.options.Debug || r.options.DebugRequests {
gologger.Info().Msgf("Dumped HTTP request for %s\n\n", fullURL)
Expand Down