v1.1.1 release preparation

.goreleaser.yml

@@ -1,21 +1,33 @@
+before:
+  hooks:
+    - go mod tidy
+
 builds:
-    - binary: httpx
-      main: cmd/httpx/httpx.go
-      goos:
-        - linux
-        - windows
-        - darwin
-      goarch:
-        - amd64
-        - 386
-        - arm
-        - arm64
-
+- env:
+  - CGO_ENABLED=0
+  goos:
+    - windows
+    - linux
+    - darwin
+  goarch:
+    - amd64
+    - 386
+    - arm
+    - arm64
+
+  ignore:
+    - goos: darwin
+      goarch: '386'
+    - goos: windows
+      goarch: 'arm'
+
+  binary: '{{ .ProjectName }}'
+  main: cmd/httpx/main.go
+
 archives:
-    - id: tgz
-      format: tar.gz
-      replacements:
-          darwin: macOS
-      format_overrides:
-          - goos: windows
-            format: zip
+- format: zip
+  replacements:
+      darwin: macOS
+
+checksum:
+  algorithm: sha256

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.14-alpine AS builder
+FROM golang:1.16.7-alpine AS builder
 RUN apk add --no-cache git
 RUN GO111MODULE=on go get -v github.com/projectdiscovery/httpx/cmd/httpx
 

README.md

@@ -77,14 +77,12 @@ This will display help for the tool. Here are all the switches it supports.
 <summary> 👉 httpx help menu 👈</summary>
 
 ```
-Usage of ./httpx:
-
   -H value
       Custom Header
   -allow value
       Allowlist ip/cidr
   -body string
-      Request Body
+      Content to send in body with HTTP request
   -cdn
       Check if domain's ip belongs to known CDN (akamai, cloudflare, ..)
   -cname
@@ -99,6 +97,8 @@ Usage of ./httpx:
       Debug mode
   -deny value
       Denylist ip/cidr
+  -exclude-cdn
+      Skip full port scans for CDNs (only checks for 80,443)
   -extract-regex string
       Extract Regex
   -fc string
@@ -133,18 +133,18 @@ Usage of ./httpx:
       Match Regex
   -match-string string
       Match string
-  -max-response-body-size int
-      Maximum response body size (default 2147483647)
   -mc string
       Match status code
   -method
-      Output method
+      Display request method
   -ml string
       Match content length
   -no-color
       No Color
   -no-fallback
       If HTTPS on port 443 is successful on default configuration, probes also port 80 for HTTP
+  -no-fallback-scheme
+      The tool will respect and attempt the scheme specified in the url (if HTTPS is specified no HTTP is attempted)
   -o string
       File to write output to (optional)
   -path string
@@ -155,14 +155,24 @@ Usage of ./httpx:
       HTTP1.1 Pipeline
   -ports value
       ports range (nmap syntax: eg 1,2-10,11)
+  -probe
+      Display probe status
   -random-agent
-      Use randomly selected HTTP User-Agent header value
+      Use randomly selected HTTP User-Agent header value (default true)
+  -rate-limit int
+      Maximum requests to send per second (default 150)
   -request string
       File containing raw request
   -response-in-json
       Show Raw HTTP Response In Output (-json only) (deprecated)
+  -response-size-to-read int
+      Max response size to read in bytes (default - unlimited)
+  -response-size-to-save int
+      Max response size to save in bytes (default - unlimited)
   -response-time
       Output the response time
+  -resume
+      Resume scan using resume.cfg
   -retries int
       Number of retries
   -silent
@@ -241,19 +251,31 @@ https://support.hackerone.com
 
 ### Running httpx with file input  
 
-This will run the tool against all the hosts and subdomains in `hosts.txt` and returns URLs running HTTP webserver.
+This will run the tool with the `probe` flag against all of the hosts in **hosts.txt** and return URLs with probed status.
 
 ```sh
-▶ httpx -l hosts.txt -silent
-
-https://docs.hackerone.com
-https://mta-sts.hackerone.com
-https://mta-sts.managed.hackerone.com
-https://mta-sts.forwarding.hackerone.com
-https://www.hackerone.com
-https://resources.hackerone.com
-https://api.hackerone.com
-https://support.hackerone.com
+▶ httpx -l hosts.txt -silent -probe
+
+http://ns.hackerone.com [FAILED]
+https://docs.hackerone.com [SUCCESS]
+https://mta-sts.hackerone.com [SUCCESS]
+https://mta-sts.managed.hackerone.com [SUCCESS]
+http://email.hackerone.com [FAILED]
+https://mta-sts.forwarding.hackerone.com [SUCCESS]
+http://links.hackerone.com [FAILED]
+https://api.hackerone.com [SUCCESS]
+https://www.hackerone.com [SUCCESS]
+http://events.hackerone.com [FAILED]
+https://support.hackerone.com [SUCCESS]
+https://gslink.hackerone.com [SUCCESS]
+http://o1.email.hackerone.com [FAILED]
+http://info.hackerone.com [FAILED]
+https://resources.hackerone.com [SUCCESS]
+http://o2.email.hackerone.com [FAILED]
+http://o3.email.hackerone.com [FAILED]
+http://go.hackerone.com [FAILED]
+http://a.ns.hackerone.com [FAILED]
+http://b.ns.hackerone.com [FAILED]
 ```
 
 ### Running httpx with CIDR input   
@@ -286,7 +308,7 @@ https://173.0.84.34
 
 
 ```sh
-▶ subfinder -d hackerone.com | httpx -title -tech-detect -status-code -follow-redirects
+subfinder -d hackerone.com | httpx -title -tech-detect -status-code
 
     __    __  __       _  __
    / /_  / /_/ /_____ | |/ /

cmd/httpx/httpx.go

@@ -1,6 +1,9 @@
 package main
 
 import (
+	"os"
+	"os/signal"
+
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/httpx/runner"
 )
@@ -9,10 +12,29 @@ func main() {
 	// Parse the command line flags and read config files
 	options := runner.ParseOptions()
 
-	r, err := runner.New(options)
+	httpxRunner, err := runner.New(options)
 	if err != nil {
 		gologger.Fatal().Msgf("Could not create runner: %s\n", err)
 	}
-	r.RunEnumeration()
-	r.Close()
+
+	// Setup graceful exits
+	c := make(chan os.Signal, 1)
+	signal.Notify(c, os.Interrupt)
+	go func() {
+		for range c {
+			gologger.Info().Msgf("CTRL+C pressed: Exiting\n")
+			httpxRunner.Close()
+			if options.ShouldSaveResume() {
+				gologger.Info().Msgf("Creating resume file: %s\n", runner.DefaultResumeFile)
+				err := httpxRunner.SaveResumeConfig()
+				if err != nil {
+					gologger.Error().Msgf("Couldn't create resume file: %s\n", err)
+				}
+			}
+			os.Exit(1)
+		}
+	}()
+
+	httpxRunner.RunEnumeration()
+	httpxRunner.Close()
 }

common/httpx/httpx.go

@@ -3,6 +3,7 @@ package httpx
 import (
 	"crypto/tls"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"net/http"
 	"net/url"
@@ -112,7 +113,7 @@ func New(options *Options) (*HTTPX, error) {
 	httpx.htmlPolicy = bluemonday.NewPolicy()
 	httpx.CustomHeaders = httpx.Options.CustomHeaders
 	httpx.RequestOverride = &options.RequestOverride
-	if options.CdnCheck {
+	if options.CdnCheck || options.ExcludeCdn {
 		httpx.cdn, err = cdncheck.NewWithCache()
 		if err != nil {
 			return nil, fmt.Errorf("could not create cdn check: %s", err)
@@ -156,7 +157,7 @@ get_response:
 	// websockets don't have a readable body
 	if httpresp.StatusCode != http.StatusSwitchingProtocols {
 		var err error
-		respbody, err = ioutil.ReadAll(httpresp.Body)
+		respbody, err = ioutil.ReadAll(io.LimitReader(httpresp.Body, h.Options.MaxResponseBodySizeToRead))
 		if err != nil {
 			return nil, err
 		}