Merge pull request #659 from projectdiscovery/dev

v1.2.2 Release

.github/workflows/build-test.yml

@@ -14,9 +14,9 @@ jobs:
         os: [ubuntu-latest, windows-latest, macOS-latest]
     steps:
       - name: Set up Go
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v3
         with:
-          go-version: 1.17
+          go-version: 1.18
 
       - name: Check out code
         uses: actions/checkout@v3

.github/workflows/codeql-analysis.yml

@@ -28,12 +28,12 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
+      uses: github/codeql-action/init@v2
       with:
         languages: ${{ matrix.language }}
 
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v1
+      uses: github/codeql-action/autobuild@v2
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1
+      uses: github/codeql-action/analyze@v2

.github/workflows/dockerhub-push.yml

@@ -20,19 +20,19 @@ jobs:
           echo "::set-output name=tag::$(curl --silent "https://api.github.com/repos/projectdiscovery/httpx/releases/latest" | jq -r .tag_name)"
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v2
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
 
       - name: Login to DockerHub
-        uses: docker/login-action@v1 
+        uses: docker/login-action@v2 
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_TOKEN }}
 
       - name: Build and push
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v3
         with:
           context: .
           platforms: linux/amd64,linux/arm64,linux/arm

.github/workflows/functional-test.yml

@@ -13,9 +13,9 @@ jobs:
         os: [ubuntu-latest, windows-latest, macOS-latest]
     steps:
       - name: Set up Go
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v3
         with:
-          go-version: 1.17
+          go-version: 1.18
 
       - name: Check out code
         uses: actions/checkout@v3

.github/workflows/lint-test.yml

@@ -10,13 +10,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Set up Go
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v3
         with:
-          go-version: 1.17
+          go-version: 1.18
       - name: Checkout code
         uses: actions/checkout@v3
       - name: Run golangci-lint
-        uses: golangci/golangci-lint-action@v3.1.0
+        uses: golangci/golangci-lint-action@v3.2.0
         with:
           version: latest
           args: --timeout 5m

.github/workflows/release-binary.yml

@@ -15,12 +15,12 @@ jobs:
           fetch-depth: 0
 
       - name: "Set up Go"
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v3
         with: 
-          go-version: 1.17
+          go-version: 1.18
 
       - name: "Create release on GitHub"
-        uses: goreleaser/goreleaser-action@v2
+        uses: goreleaser/goreleaser-action@v3
         with: 
           args: "release --rm-dist"
           version: latest

.github/workflows/sonarcloud.yml

@@ -18,9 +18,9 @@ jobs:
           fetch-depth: 0  # Shallow clones should be disabled for a better relevancy of analysis
 
       - name: "Set up Go"
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v3
         with: 
-          go-version: 1.17
+          go-version: 1.18
 
       - name: Run unit Tests
         run: |

.gitignore

@@ -6,4 +6,4 @@ cmd/functional-test/httpx_dev
 cmd/functional-test/functional-test
 cmd/functional-test/httpx
 cmd/functional-test/*.cfg
-
+.vscode/

Dockerfile

@@ -1,8 +1,8 @@
-FROM golang:1.18.0-alpine AS builder
+FROM golang:1.18.3-alpine AS builder
 RUN apk add --no-cache git
 RUN go install -v github.com/projectdiscovery/httpx/cmd/httpx@latest
 
-FROM alpine:3.15.3
+FROM alpine:3.16.0
 RUN apk -U upgrade --no-cache \
     && apk add --no-cache bind-tools ca-certificates
 COPY --from=builder /go/bin/httpx /usr/local/bin/

Makefile

@@ -3,12 +3,17 @@ GOCMD=go
 GOBUILD=$(GOCMD) build
 GOMOD=$(GOCMD) mod
 GOTEST=$(GOCMD) test
-GOGET=$(GOCMD) get
+GOFLAGS := -v 
+LDFLAGS := -s -w
+
+ifneq ($(shell go env GOOS),darwin)
+LDFLAGS := -extldflags "-static"
+endif
 
 all: build
 build:
-		$(GOBUILD) -v -ldflags="-extldflags=-static" -o "httpx" cmd/httpx/httpx.go
+	$(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "httpx" cmd/httpx/httpx.go
 test: 
-		$(GOTEST) -v ./...
+	$(GOTEST) $(GOFLAGS) ./...
 tidy:
-		$(GOMOD) tidy
+	$(GOMOD) tidy

README.md

@@ -109,25 +109,29 @@ PROBES:
    -probe                display probe status
 
 MATCHERS:
-   -mc, -match-code string         match response with specified status code (-mc 200,302)
-   -ml, -match-length string       match response with specified content length (-ml 100,102)
-   -mlc, -match-line-count string  match response body with specified line count (-mlc 423,532)
-   -mwc, -match-word-count string  match response body with specified word count (-mwc 43,55)
-   -mfc, -match-favicon string[]   match response with specified favicon hash (-mfc 1494302000)
-   -ms, -match-string string       match response with specified string (-ms admin)
-   -mr, -match-regex string        match response with specified regex (-mr admin)
+   -mc, -match-code string            match response with specified status code (-mc 200,302)
+   -ml, -match-length string          match response with specified content length (-ml 100,102)
+   -mlc, -match-line-count string     match response body with specified line count (-mlc 423,532)
+   -mwc, -match-word-count string     match response body with specified word count (-mwc 43,55)
+   -mfc, -match-favicon string[]      match response with specified favicon hash (-mfc 1494302000)
+   -ms, -match-string string          match response with specified string (-ms admin)
+   -mr, -match-regex string           match response with specified regex (-mr admin)
+   -mcdn, -match-cdn string[]         match host with specified cdn provider (google, azure, cloudflare, cloudfront, fastly, incapsula, oracle, akamai, sucuri, leaseweb)
+   -mrt, -match-response-time string  match response with specified response time in seconds (-mrt '< 1')
 
 EXTRACTOR:
    -er, -extract-regex string  display response content for specified regex
 
 FILTERS:
-   -fc, -filter-code string         filter response with specified status code (-fc 403,401)
-   -fl, -filter-length string       filter response with specified content length (-fl 23,33)
-   -flc, -filter-line-count string  filter response body with specified line count (-flc 423,532)
-   -fwc, -filter-word-count string  filter response body with specified word count (-fwc 423,532)
-   -ffc, -filter-favicon string[]   filter response with specified favicon hash (-mfc 1494302000)
-   -fs, -filter-string string       filter response with specified string (-fs admin)
-   -fe, -filter-regex string        filter response with specified regex (-fe admin)
+   -fc, -filter-code string            filter response with specified status code (-fc 403,401)
+   -fl, -filter-length string          filter response with specified content length (-fl 23,33)
+   -flc, -filter-line-count string     filter response body with specified line count (-flc 423,532)
+   -fwc, -filter-word-count string     filter response body with specified word count (-fwc 423,532)
+   -ffc, -filter-favicon string[]      filter response with specified favicon hash (-mfc 1494302000)
+   -fs, -filter-string string          filter response with specified string (-fs admin)
+   -fe, -filter-regex string           filter response with specified regex (-fe admin)
+   -fcdn, -filter-cdn string[]         filter host with specified cdn provider (google, azure, cloudflare, cloudfront, fastly, incapsula, oracle, akamai, sucuri, leaseweb)
+   -frt, -filter-response-time string  filter response with specified response time in seconds (-frt '> 1')
 
 RATE-LIMIT:
    -t, -threads int              number of threads to use (default 50)
@@ -159,6 +163,7 @@ CONFIGURATIONS:
    -r, -resolvers string[]       list of custom resolver (file or comma separated)
    -allow string[]               allowed list of IP/CIDR's to process (file or comma separated)
    -deny string[]                denied list of IP/CIDR's to process (file or comma separated)
+   -sni, -sni-name string        Custom TLS SNI name
    -random-agent                 Enable Random User-Agent to use (default true)
    -H, -header string[]          custom http headers to send with request
    -http-proxy, -proxy string    http proxy to use (eg http://127.0.0.1:8080)
@@ -172,7 +177,7 @@ CONFIGURATIONS:
    -body string                  post body to include in http request
    -s, -stream                   stream mode - start elaborating input targets without sorting
    -sd, -skip-dedupe             disable dedupe input items (only used with stream mode)
-   -ldp, -leave-default-ports    leave default http/https ports in host header (eg. http://host:80 - https//host:443)
+   -ldp, -leave-default-ports    leave default http/https ports in host header (eg. http://host:80 - https//host:443
 
 DEBUG:
    -debug                    display request/response content in cli

cmd/integration-test/http.go

@@ -25,11 +25,16 @@ var httpTestcases = map[string]testutils.TestCase{
 	"Regression test for: https://github.com/projectdiscovery/httpx/issues/414":           &issue414{}, // stream mode with path
 	"Regression test for: https://github.com/projectdiscovery/httpx/issues/433":           &issue433{}, // new line scanning with title flag
 	"Request URI to existing file - https://github.com/projectdiscovery/httpx/issues/480": &issue480{}, // request uri pointing to existing file
+	"Standard HTTP GET Request with match response time":                                  &standardHttpGet{mrt: true, inputValue: "\"<10s\""},
+	"Standard HTTP GET Request with filter response time":                                 &standardHttpGet{frt: true, inputValue: "\">3ms\""},
 }
 
 type standardHttpGet struct {
 	tls            bool
 	unsafe         bool
+	mrt            bool
+	frt            bool
+	inputValue     string
 	stdinPath      string
 	path           string
 	expectedOutput string
@@ -55,7 +60,12 @@ func (h *standardHttpGet) Execute() error {
 	if h.path != "" {
 		extra = append(extra, "-path", "\""+h.path+"\"")
 	}
-
+	if h.mrt {
+		extra = append(extra, "-mrt", h.inputValue)
+	}
+	if h.frt {
+		extra = append(extra, "-frt", h.inputValue)
+	}
 	URL := ts.URL
 	if h.stdinPath != "" {
 		URL += h.stdinPath

common/httpx/httpx.go

@@ -16,6 +16,7 @@ import (
 	"github.com/microcosm-cc/bluemonday"
 	"github.com/projectdiscovery/cdncheck"
 	"github.com/projectdiscovery/fastdialer/fastdialer"
+	"github.com/projectdiscovery/gologger"
 	pdhttputil "github.com/projectdiscovery/httputil"
 	"github.com/projectdiscovery/rawhttp"
 	retryablehttp "github.com/projectdiscovery/retryablehttp-go"
@@ -47,6 +48,7 @@ func New(options *Options) (*HTTPX, error) {
 	if len(options.Resolvers) > 0 {
 		fastdialerOpts.BaseResolvers = options.Resolvers
 	}
+	fastdialerOpts.SNIName = options.SniName
 	dialer, err := fastdialer.NewDialer(fastdialerOpts)
 	if err != nil {
 		return nil, fmt.Errorf("could not create resolver cache: %s", err)
@@ -108,9 +110,13 @@ func New(options *Options) (*HTTPX, error) {
 		MaxIdleConnsPerHost: -1,
 		TLSClientConfig: &tls.Config{
 			InsecureSkipVerify: true,
+			MinVersion:         tls.VersionTLS10,
 		},
 		DisableKeepAlives: true,
 	}
+	if httpx.Options.SniName != "" {
+		transport.TLSClientConfig.ServerName = httpx.Options.SniName
+	}
 
 	if httpx.Options.HTTPProxy != "" {
 		proxyURL, parseErr := url.Parse(httpx.Options.HTTPProxy)
@@ -126,22 +132,27 @@ func New(options *Options) (*HTTPX, error) {
 		CheckRedirect: redirectFunc,
 	}, retryablehttpOptions)
 
-	httpx.client2 = &http.Client{
-		Transport: &http2.Transport{
-			TLSClientConfig: &tls.Config{
-				InsecureSkipVerify: true,
-			},
-			AllowHTTP: true,
+	transport2 := &http2.Transport{
+		TLSClientConfig: &tls.Config{
+			InsecureSkipVerify: true,
+			MinVersion:         tls.VersionTLS10,
 		},
-		Timeout: httpx.Options.Timeout,
+		AllowHTTP: true,
+	}
+	if httpx.Options.SniName != "" {
+		transport2.TLSClientConfig.ServerName = httpx.Options.SniName
+	}
+	httpx.client2 = &http.Client{
+		Transport: transport2,
+		Timeout:   httpx.Options.Timeout,
 	}
 
 	httpx.htmlPolicy = bluemonday.NewPolicy()
 	httpx.CustomHeaders = httpx.Options.CustomHeaders
 	if options.CdnCheck || options.ExcludeCdn {
 		httpx.cdn, err = cdncheck.NewWithCache()
 		if err != nil {
-			return nil, fmt.Errorf("could not create cdn check: %s", err)
+			gologger.Error().Msgf("could not create cdn check: %v", err)
 		}
 	}
 

common/httpx/option.go

@@ -40,6 +40,7 @@ type Options struct {
 	UnsafeURI                 string
 	Resolvers                 []string
 	customCookies             []*http.Cookie
+	SniName                   string
 }
 
 // DefaultOptions contains the default options

common/httpx/types.go

@@ -0,0 +1,8 @@
+package httpx
+
+// Target of the scan with ip|host header customization
+type Target struct {
+	Host       string
+	CustomHost string
+	CustomIP   string
+}