Merge pull request #370 from projectdiscovery/dev

v1.1.2 Release preparation

.github/workflows/build-test.yml

@@ -19,8 +19,11 @@ jobs:
         uses: actions/checkout@v2
 
       - name: Test
-        run: go test .
-        working-directory: cmd/httpx/
+        run: go test ./...
+
+      - name: Integration Tests
+        run: bash run.sh
+        working-directory: integration_tests/
 
       - name: Build
         run: go build .

.gitignore

@@ -1,2 +1,4 @@
 
 cmd/httpx/httpx
+integration_tests/httpx
+integration_tests/integration-test

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.16.7-alpine AS builder
+FROM golang:1.17.0-alpine AS builder
 RUN apk add --no-cache git
 RUN GO111MODULE=on go get -v github.com/projectdiscovery/httpx/cmd/httpx
 

README.md

@@ -80,7 +80,7 @@ This will display help for the tool. Here are all the switches it supports.
   -H value
       Custom Header
   -allow value
-      Allowlist ip/cidr
+      Allow list of IP/CIDR's (file or comma separated)
   -body string
       Content to send in body with HTTP request
   -cdn
@@ -96,7 +96,7 @@ This will display help for the tool. Here are all the switches it supports.
   -debug
       Debug mode
   -deny value
-      Denylist ip/cidr
+      Deny list of IP/CIDR's to process (file or comma separated)
   -exclude-cdn
       Skip full port scans for CDNs (only checks for 80,443)
   -extract-regex string

cmd/integration-test/http.go

@@ -0,0 +1,186 @@
+package main
+
+import (
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+
+	"github.com/julienschmidt/httprouter"
+	"github.com/projectdiscovery/httpx/internal/testutils"
+)
+
+var httpTestcases = map[string]testutils.TestCase{
+	"Standard HTTP GET Request":                                                 &standardHttpGet{},
+	"Standard HTTPS GET Request":                                                &standardHttpGet{tls: true},
+	"Raw HTTP GET Request":                                                      &standardHttpGet{unsafe: true},
+	"Raw request with non standard rfc path via stdin":                          &standardHttpGet{unsafe: true, stdinPath: "/%invalid"},
+	"Raw request with non standard rfc path via cli flag":                       &standardHttpGet{unsafe: true, path: "/%invalid"},
+	"Regression test for: https://github.com/projectdiscovery/httpx/issues/363": &issue363{}, // infinite redirect
+	"Regression test for: https://github.com/projectdiscovery/httpx/issues/276": &issue276{}, // full path with port in output
+	"Regression test for: https://github.com/projectdiscovery/httpx/issues/277": &issue277{}, // scheme://host:port via stdin
+	"Regression test for: https://github.com/projectdiscovery/httpx/issues/303": &issue303{}, // misconfigured gzip header with uncompressed body
+}
+
+type standardHttpGet struct {
+	tls            bool
+	unsafe         bool
+	stdinPath      string
+	path           string
+	expectedOutput string
+}
+
+func (h *standardHttpGet) Execute() error {
+	router := httprouter.New()
+	router.GET("/", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		fmt.Fprintf(w, "This is a test")
+		r.Close = true
+	}))
+	var ts *httptest.Server
+	if h.tls {
+		ts = httptest.NewTLSServer(router)
+	} else {
+		ts = httptest.NewServer(router)
+	}
+	defer ts.Close()
+	var extra []string
+	if h.unsafe {
+		extra = append(extra, "-unsafe")
+	}
+	if h.path != "" {
+		extra = append(extra, "-path", "\""+h.path+"\"")
+	}
+
+	URL := ts.URL
+	if h.stdinPath != "" {
+		URL += h.stdinPath
+	}
+
+	results, err := testutils.RunHttpxAndGetResults(URL, debug, extra...)
+	if err != nil {
+		return err
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+
+	if h.expectedOutput != "" && !strings.EqualFold(results[0], h.expectedOutput) {
+		return errIncorrectResult(results[0], h.expectedOutput)
+	}
+
+	return nil
+}
+
+type issue276 struct{}
+
+func (h *issue276) Execute() error {
+	var ts *httptest.Server
+	router := httprouter.New()
+	router.GET("/redirect", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		w.Header().Add("Location", ts.URL+"/redirect")
+		w.WriteHeader(302)
+		fmt.Fprintf(w, "<html><body><title>Object moved</title></body></html>")
+	}))
+	ts = httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunHttpxAndGetResults(ts.URL+"/redirect", debug, "-status-code", "-title", "-no-color")
+	if err != nil {
+		return err
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	// check if we have all the items on the cli
+	// full url with port
+	// status code
+	// title
+	expected := ts.URL + "/redirect" + " [302] [Object moved]"
+	if !strings.EqualFold(results[0], expected) {
+		return errIncorrectResult(results[0], expected)
+	}
+	return nil
+}
+
+type issue277 struct{}
+
+func (h *issue277) Execute() error {
+	var ts *httptest.Server
+	router := httprouter.New()
+	router.GET("/hpp", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
+		if p.ByName("pp") == `%22%3E%3Ch1%3Easdasd%3C%2Fh1%3E` {
+			w.WriteHeader(http.StatusOK)
+		}
+	}))
+	ts = httptest.NewServer(router)
+	defer ts.Close()
+	uripath := "/hpp/?pp=%22%3E%3Ch1%3Easdasd%3C%2Fh1%3E"
+	results, err := testutils.RunHttpxAndGetResults(ts.URL+uripath, debug)
+	if err != nil {
+		return err
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	// check if we have all the items on the cli
+	// full url with port
+	// status code
+	// title
+	expected := ts.URL + uripath
+	if !strings.EqualFold(results[0], expected) {
+		return errIncorrectResult(results[0], expected)
+	}
+	return nil
+}
+
+type issue303 struct{}
+
+func (h *issue303) Execute() error {
+	var ts *httptest.Server
+	router := httprouter.New()
+	router.GET("/hpp", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
+		// mimic a misconfigured web server behavior declaring gzip body
+		w.Header().Add("Content-Encoding", "gzip")
+		// but sending it uncompressed
+		fmt.Fprint(w, "<html><body>This is a test</body></html>")
+	}))
+	ts = httptest.NewServer(router)
+	defer ts.Close()
+	results, err := testutils.RunHttpxAndGetResults(ts.URL, debug)
+	if err != nil {
+		return err
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	// check if we have all the items on the cli
+	// full url with port
+	expected := ts.URL
+	if !strings.EqualFold(results[0], expected) {
+		return errIncorrectResult(results[0], expected)
+	}
+	return nil
+}
+
+type issue363 struct{}
+
+func (h *issue363) Execute() error {
+	var ts *httptest.Server
+	router := httprouter.New()
+	router.GET("/redirect", httprouter.Handle(func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
+		w.Header().Add("Location", ts.URL+"/redirect")
+		w.WriteHeader(302)
+		fmt.Fprintf(w, "<html><body><title>Object moved</title></body></html>")
+	}))
+	ts = httptest.NewServer(router)
+	defer ts.Close()
+
+	results, err := testutils.RunHttpxAndGetResults(ts.URL+"/redirect", debug, "-no-color", "-follow-redirects")
+	if err != nil {
+		return err
+	}
+	if len(results) != 1 {
+		return errIncorrectResultsCount(results)
+	}
+	return nil
+}

cmd/integration-test/integration-test.go

@@ -0,0 +1,56 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/logrusorgru/aurora"
+	"github.com/projectdiscovery/httpx/internal/testutils"
+)
+
+var (
+	debug      = os.Getenv("DEBUG") == "true"
+	customTest = os.Getenv("TEST")
+	protocol   = os.Getenv("PROTO")
+
+	errored = false
+)
+
+func main() {
+	success := aurora.Green("[✓]").String()
+	failed := aurora.Red("[✘]").String()
+
+	tests := map[string]map[string]testutils.TestCase{
+		"http": httpTestcases,
+	}
+	for proto, tests := range tests {
+		if protocol == "" || protocol == proto {
+			fmt.Printf("Running test cases for \"%s\"\n", aurora.Blue(proto))
+
+			for name, test := range tests {
+				if customTest != "" && !strings.Contains(name, customTest) {
+					continue // only run tests user asked
+				}
+				err := test.Execute()
+				if err != nil {
+					fmt.Fprintf(os.Stderr, "%s Test \"%s\" failed: %s\n", failed, name, err)
+					errored = true
+				} else {
+					fmt.Printf("%s Test \"%s\" passed!\n", success, name)
+				}
+			}
+		}
+	}
+	if errored {
+		os.Exit(1)
+	}
+}
+
+func errIncorrectResultsCount(results []string) error {
+	return fmt.Errorf("incorrect number of results %s", strings.Join(results, "\n\t"))
+}
+
+func errIncorrectResult(expected, got string) error {
+	return fmt.Errorf("incorrect result: expected \"%s\" got \"%s\"", expected, got)
+}

common/customlist/customlist.go

@@ -1,5 +1,7 @@
 package customlist
 
+import "github.com/projectdiscovery/httpx/common/fileutil"
+
 // CustomList for fastdialer
 type CustomList []string
 
@@ -10,6 +12,7 @@ func (c *CustomList) String() string {
 
 // Set a new global header
 func (c *CustomList) Set(value string) error {
-	*c = append(*c, value)
+	values := fileutil.LoadCidrsFromSliceOrFile(value, ",")
+	*c = append(*c, values...)
 	return nil
 }

common/customports/customport.go

@@ -5,6 +5,7 @@ import (
 	"strings"
 
 	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/sliceutil"
 
 	"github.com/projectdiscovery/httpx/common/httpx"
 )
@@ -31,7 +32,7 @@ func (c *CustomPorts) String() string {
 func (c *CustomPorts) Set(value string) error {
 	// ports can be like nmap -p [https|http:]start-end,[https|http:]port1,[https|http:]port2,[https|http:]port3
 	// splits on comma
-	potentialPorts := strings.Split(value, ",")
+	potentialPorts := sliceutil.Dedupe(strings.Split(value, ","))
 
 	// check if port is a single integer value or needs to be expanded further
 	for _, potentialPort := range potentialPorts {
@@ -43,12 +44,20 @@ func (c *CustomPorts) Set(value string) error {
 		} else if strings.HasPrefix(potentialPort, httpx.HTTPS+":") {
 			potentialPort = strings.TrimPrefix(potentialPort, httpx.HTTPS+":")
 			protocol = httpx.HTTPS
+		} else if strings.HasPrefix(potentialPort, httpx.HTTPandHTTPS+":") {
+			potentialPort = strings.TrimPrefix(potentialPort, httpx.HTTPandHTTPS+":")
+			protocol = httpx.HTTPandHTTPS
 		}
 
 		potentialRange := strings.Split(potentialPort, "-")
 		// it's a single port?
 		if len(potentialRange) < portRangeParts {
 			if p, err := strconv.Atoi(potentialPort); err == nil {
+				if existingProtocol, ok := Ports[p]; ok {
+					if existingProtocol == httpx.HTTP && protocol == httpx.HTTPS || existingProtocol == httpx.HTTPS && protocol == httpx.HTTP {
+						protocol = httpx.HTTPandHTTPS
+					}
+				}
 				Ports[p] = protocol
 			} else {
 				gologger.Warning().Msgf("Could not cast port to integer, your value: %s, resulting error %s. Skipping it\n",
@@ -79,6 +88,11 @@ func (c *CustomPorts) Set(value string) error {
 			}
 
 			for i := lowP; i <= highP; i++ {
+				if existingProtocol, ok := Ports[i]; ok {
+					if existingProtocol == httpx.HTTP && protocol == httpx.HTTPS || existingProtocol == httpx.HTTPS && protocol == httpx.HTTP {
+						protocol = httpx.HTTPandHTTPS
+					}
+				}
 				Ports[i] = protocol
 			}
 		}

common/fileutil/fileutil.go

@@ -3,9 +3,14 @@ package fileutil
 import (
 	"bufio"
 	"errors"
+	"io/ioutil"
+	"net"
 	"os"
 	"path/filepath"
 	"regexp"
+
+	"github.com/projectdiscovery/fileutil"
+	"github.com/projectdiscovery/httpx/common/stringz"
 )
 
 // FileExists checks if a file exists and is not a directory
@@ -70,3 +75,21 @@ func FileNameIsGlob(pattern string) bool {
 	_, err := regexp.Compile(pattern)
 	return err == nil
 }
+
+func LoadCidrsFromSliceOrFile(option string, splitchar string) (networkList []string) {
+	items := stringz.SplitByCharAndTrimSpace(option, splitchar)
+	for _, item := range items {
+		// ip
+		if net.ParseIP(item) != nil {
+			networkList = append(networkList, item)
+		} else if _, _, err := net.ParseCIDR(item); err == nil {
+			networkList = append(networkList, item)
+		} else if fileutil.FileExists(item) {
+			if filedata, err := ioutil.ReadFile(item); err == nil && len(filedata) > 0 {
+				networkList = append(networkList, LoadCidrsFromSliceOrFile(string(filedata), "\n")...)
+			}
+		}
+	}
+
+	return networkList
+}