diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 1f45b5496..74ccc3099 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -14,9 +14,9 @@ jobs: os: [ubuntu-latest, windows-latest, macOS-latest] steps: - name: Set up Go - uses: actions/setup-go@v2 + uses: actions/setup-go@v3 with: - go-version: 1.17 + go-version: 1.18 - name: Check out code uses: actions/checkout@v3 diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 7c34372a2..2bbccfd46 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -28,12 +28,12 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v1 + uses: github/codeql-action/init@v2 with: languages: ${{ matrix.language }} - name: Autobuild - uses: github/codeql-action/autobuild@v1 + uses: github/codeql-action/autobuild@v2 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 \ No newline at end of file + uses: github/codeql-action/analyze@v2 \ No newline at end of file diff --git a/.github/workflows/dockerhub-push.yml b/.github/workflows/dockerhub-push.yml index f253b3dda..72c1cfa33 100644 --- a/.github/workflows/dockerhub-push.yml +++ b/.github/workflows/dockerhub-push.yml @@ -20,19 +20,19 @@ jobs: echo "::set-output name=tag::$(curl --silent "https://api.github.com/repos/projectdiscovery/httpx/releases/latest" | jq -r .tag_name)" - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 - name: Login to DockerHub - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_TOKEN }} - name: Build and push - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v3 with: context: . platforms: linux/amd64,linux/arm64,linux/arm diff --git a/.github/workflows/functional-test.yml b/.github/workflows/functional-test.yml index 943e4849c..841226cc8 100644 --- a/.github/workflows/functional-test.yml +++ b/.github/workflows/functional-test.yml @@ -13,9 +13,9 @@ jobs: os: [ubuntu-latest, windows-latest, macOS-latest] steps: - name: Set up Go - uses: actions/setup-go@v2 + uses: actions/setup-go@v3 with: - go-version: 1.17 + go-version: 1.18 - name: Check out code uses: actions/checkout@v3 diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml index ae50778c2..d8b1dbc33 100644 --- a/.github/workflows/lint-test.yml +++ b/.github/workflows/lint-test.yml @@ -10,13 +10,13 @@ jobs: runs-on: ubuntu-latest steps: - name: Set up Go - uses: actions/setup-go@v2 + uses: actions/setup-go@v3 with: - go-version: 1.17 + go-version: 1.18 - name: Checkout code uses: actions/checkout@v3 - name: Run golangci-lint - uses: golangci/golangci-lint-action@v3.1.0 + uses: golangci/golangci-lint-action@v3.2.0 with: version: latest args: --timeout 5m diff --git a/.github/workflows/release-binary.yml b/.github/workflows/release-binary.yml index cf6504a4b..855356fda 100644 --- a/.github/workflows/release-binary.yml +++ b/.github/workflows/release-binary.yml @@ -15,12 +15,12 @@ jobs: fetch-depth: 0 - name: "Set up Go" - uses: actions/setup-go@v2 + uses: actions/setup-go@v3 with: - go-version: 1.17 + go-version: 1.18 - name: "Create release on GitHub" - uses: goreleaser/goreleaser-action@v2 + uses: goreleaser/goreleaser-action@v3 with: args: "release --rm-dist" version: latest diff --git a/.github/workflows/sonarcloud.yml b/.github/workflows/sonarcloud.yml index 122993069..e0df6963d 100644 --- a/.github/workflows/sonarcloud.yml +++ b/.github/workflows/sonarcloud.yml @@ -18,9 +18,9 @@ jobs: fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis - name: "Set up Go" - uses: actions/setup-go@v2 + uses: actions/setup-go@v3 with: - go-version: 1.17 + go-version: 1.18 - name: Run unit Tests run: | diff --git a/.gitignore b/.gitignore index 2390b4e62..b8f8c16ff 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,4 @@ cmd/functional-test/httpx_dev cmd/functional-test/functional-test cmd/functional-test/httpx cmd/functional-test/*.cfg - +.vscode/ diff --git a/Dockerfile b/Dockerfile index 30483e411..b2e1ab5ed 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,8 @@ -FROM golang:1.18.0-alpine AS builder +FROM golang:1.18.3-alpine AS builder RUN apk add --no-cache git RUN go install -v github.com/projectdiscovery/httpx/cmd/httpx@latest -FROM alpine:3.15.3 +FROM alpine:3.16.0 RUN apk -U upgrade --no-cache \ && apk add --no-cache bind-tools ca-certificates COPY --from=builder /go/bin/httpx /usr/local/bin/ diff --git a/Makefile b/Makefile index 595230d9c..102b49df6 100644 --- a/Makefile +++ b/Makefile @@ -3,12 +3,17 @@ GOCMD=go GOBUILD=$(GOCMD) build GOMOD=$(GOCMD) mod GOTEST=$(GOCMD) test -GOGET=$(GOCMD) get +GOFLAGS := -v +LDFLAGS := -s -w + +ifneq ($(shell go env GOOS),darwin) +LDFLAGS := -extldflags "-static" +endif all: build build: - $(GOBUILD) -v -ldflags="-extldflags=-static" -o "httpx" cmd/httpx/httpx.go + $(GOBUILD) $(GOFLAGS) -ldflags '$(LDFLAGS)' -o "httpx" cmd/httpx/httpx.go test: - $(GOTEST) -v ./... + $(GOTEST) $(GOFLAGS) ./... tidy: - $(GOMOD) tidy + $(GOMOD) tidy diff --git a/README.md b/README.md index 40f944cdd..ae5ae2d83 100644 --- a/README.md +++ b/README.md @@ -109,25 +109,29 @@ PROBES: -probe display probe status MATCHERS: - -mc, -match-code string match response with specified status code (-mc 200,302) - -ml, -match-length string match response with specified content length (-ml 100,102) - -mlc, -match-line-count string match response body with specified line count (-mlc 423,532) - -mwc, -match-word-count string match response body with specified word count (-mwc 43,55) - -mfc, -match-favicon string[] match response with specified favicon hash (-mfc 1494302000) - -ms, -match-string string match response with specified string (-ms admin) - -mr, -match-regex string match response with specified regex (-mr admin) + -mc, -match-code string match response with specified status code (-mc 200,302) + -ml, -match-length string match response with specified content length (-ml 100,102) + -mlc, -match-line-count string match response body with specified line count (-mlc 423,532) + -mwc, -match-word-count string match response body with specified word count (-mwc 43,55) + -mfc, -match-favicon string[] match response with specified favicon hash (-mfc 1494302000) + -ms, -match-string string match response with specified string (-ms admin) + -mr, -match-regex string match response with specified regex (-mr admin) + -mcdn, -match-cdn string[] match host with specified cdn provider (google, azure, cloudflare, cloudfront, fastly, incapsula, oracle, akamai, sucuri, leaseweb) + -mrt, -match-response-time string match response with specified response time in seconds (-mrt '< 1') EXTRACTOR: -er, -extract-regex string display response content for specified regex FILTERS: - -fc, -filter-code string filter response with specified status code (-fc 403,401) - -fl, -filter-length string filter response with specified content length (-fl 23,33) - -flc, -filter-line-count string filter response body with specified line count (-flc 423,532) - -fwc, -filter-word-count string filter response body with specified word count (-fwc 423,532) - -ffc, -filter-favicon string[] filter response with specified favicon hash (-mfc 1494302000) - -fs, -filter-string string filter response with specified string (-fs admin) - -fe, -filter-regex string filter response with specified regex (-fe admin) + -fc, -filter-code string filter response with specified status code (-fc 403,401) + -fl, -filter-length string filter response with specified content length (-fl 23,33) + -flc, -filter-line-count string filter response body with specified line count (-flc 423,532) + -fwc, -filter-word-count string filter response body with specified word count (-fwc 423,532) + -ffc, -filter-favicon string[] filter response with specified favicon hash (-mfc 1494302000) + -fs, -filter-string string filter response with specified string (-fs admin) + -fe, -filter-regex string filter response with specified regex (-fe admin) + -fcdn, -filter-cdn string[] filter host with specified cdn provider (google, azure, cloudflare, cloudfront, fastly, incapsula, oracle, akamai, sucuri, leaseweb) + -frt, -filter-response-time string filter response with specified response time in seconds (-frt '> 1') RATE-LIMIT: -t, -threads int number of threads to use (default 50) @@ -159,6 +163,7 @@ CONFIGURATIONS: -r, -resolvers string[] list of custom resolver (file or comma separated) -allow string[] allowed list of IP/CIDR's to process (file or comma separated) -deny string[] denied list of IP/CIDR's to process (file or comma separated) + -sni, -sni-name string Custom TLS SNI name -random-agent Enable Random User-Agent to use (default true) -H, -header string[] custom http headers to send with request -http-proxy, -proxy string http proxy to use (eg http://127.0.0.1:8080) @@ -172,7 +177,7 @@ CONFIGURATIONS: -body string post body to include in http request -s, -stream stream mode - start elaborating input targets without sorting -sd, -skip-dedupe disable dedupe input items (only used with stream mode) - -ldp, -leave-default-ports leave default http/https ports in host header (eg. http://host:80 - https//host:443) + -ldp, -leave-default-ports leave default http/https ports in host header (eg. http://host:80 - https//host:443 DEBUG: -debug display request/response content in cli diff --git a/cmd/integration-test/http.go b/cmd/integration-test/http.go index e7b930e77..bd0f6d8e5 100644 --- a/cmd/integration-test/http.go +++ b/cmd/integration-test/http.go @@ -25,11 +25,16 @@ var httpTestcases = map[string]testutils.TestCase{ "Regression test for: https://github.com/projectdiscovery/httpx/issues/414": &issue414{}, // stream mode with path "Regression test for: https://github.com/projectdiscovery/httpx/issues/433": &issue433{}, // new line scanning with title flag "Request URI to existing file - https://github.com/projectdiscovery/httpx/issues/480": &issue480{}, // request uri pointing to existing file + "Standard HTTP GET Request with match response time": &standardHttpGet{mrt: true, inputValue: "\"<10s\""}, + "Standard HTTP GET Request with filter response time": &standardHttpGet{frt: true, inputValue: "\">3ms\""}, } type standardHttpGet struct { tls bool unsafe bool + mrt bool + frt bool + inputValue string stdinPath string path string expectedOutput string @@ -55,7 +60,12 @@ func (h *standardHttpGet) Execute() error { if h.path != "" { extra = append(extra, "-path", "\""+h.path+"\"") } - + if h.mrt { + extra = append(extra, "-mrt", h.inputValue) + } + if h.frt { + extra = append(extra, "-frt", h.inputValue) + } URL := ts.URL if h.stdinPath != "" { URL += h.stdinPath diff --git a/common/httpx/httpx.go b/common/httpx/httpx.go index 23c04619c..0eb0a70cc 100644 --- a/common/httpx/httpx.go +++ b/common/httpx/httpx.go @@ -16,6 +16,7 @@ import ( "github.com/microcosm-cc/bluemonday" "github.com/projectdiscovery/cdncheck" "github.com/projectdiscovery/fastdialer/fastdialer" + "github.com/projectdiscovery/gologger" pdhttputil "github.com/projectdiscovery/httputil" "github.com/projectdiscovery/rawhttp" retryablehttp "github.com/projectdiscovery/retryablehttp-go" @@ -47,6 +48,7 @@ func New(options *Options) (*HTTPX, error) { if len(options.Resolvers) > 0 { fastdialerOpts.BaseResolvers = options.Resolvers } + fastdialerOpts.SNIName = options.SniName dialer, err := fastdialer.NewDialer(fastdialerOpts) if err != nil { return nil, fmt.Errorf("could not create resolver cache: %s", err) @@ -108,9 +110,13 @@ func New(options *Options) (*HTTPX, error) { MaxIdleConnsPerHost: -1, TLSClientConfig: &tls.Config{ InsecureSkipVerify: true, + MinVersion: tls.VersionTLS10, }, DisableKeepAlives: true, } + if httpx.Options.SniName != "" { + transport.TLSClientConfig.ServerName = httpx.Options.SniName + } if httpx.Options.HTTPProxy != "" { proxyURL, parseErr := url.Parse(httpx.Options.HTTPProxy) @@ -126,14 +132,19 @@ func New(options *Options) (*HTTPX, error) { CheckRedirect: redirectFunc, }, retryablehttpOptions) - httpx.client2 = &http.Client{ - Transport: &http2.Transport{ - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: true, - }, - AllowHTTP: true, + transport2 := &http2.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + MinVersion: tls.VersionTLS10, }, - Timeout: httpx.Options.Timeout, + AllowHTTP: true, + } + if httpx.Options.SniName != "" { + transport2.TLSClientConfig.ServerName = httpx.Options.SniName + } + httpx.client2 = &http.Client{ + Transport: transport2, + Timeout: httpx.Options.Timeout, } httpx.htmlPolicy = bluemonday.NewPolicy() @@ -141,7 +152,7 @@ func New(options *Options) (*HTTPX, error) { if options.CdnCheck || options.ExcludeCdn { httpx.cdn, err = cdncheck.NewWithCache() if err != nil { - return nil, fmt.Errorf("could not create cdn check: %s", err) + gologger.Error().Msgf("could not create cdn check: %v", err) } } diff --git a/common/httpx/option.go b/common/httpx/option.go index 059665a0e..b703bd586 100644 --- a/common/httpx/option.go +++ b/common/httpx/option.go @@ -40,6 +40,7 @@ type Options struct { UnsafeURI string Resolvers []string customCookies []*http.Cookie + SniName string } // DefaultOptions contains the default options diff --git a/common/httpx/types.go b/common/httpx/types.go new file mode 100644 index 000000000..768328284 --- /dev/null +++ b/common/httpx/types.go @@ -0,0 +1,8 @@ +package httpx + +// Target of the scan with ip|host header customization +type Target struct { + Host string + CustomHost string + CustomIP string +} diff --git a/go.mod b/go.mod index f98b28720..447c00b90 100644 --- a/go.mod +++ b/go.mod @@ -5,41 +5,41 @@ go 1.17 require ( github.com/akrylysov/pogreb v0.10.1 // indirect github.com/bluele/gcache v0.0.2 - github.com/corpix/uarand v0.1.1 + github.com/corpix/uarand v0.2.0 github.com/golang/snappy v0.0.4 // indirect github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf github.com/julienschmidt/httprouter v1.3.0 github.com/logrusorgru/aurora v2.0.3+incompatible github.com/microcosm-cc/bluemonday v1.0.18 - github.com/miekg/dns v1.1.46 // indirect + github.com/miekg/dns v1.1.48 // indirect github.com/pkg/errors v0.9.1 - github.com/projectdiscovery/cdncheck v0.0.3 + github.com/projectdiscovery/cdncheck v0.0.4-0.20220413175814-b47bc2d578b1 github.com/projectdiscovery/clistats v0.0.8 github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 - github.com/projectdiscovery/fastdialer v0.0.15 + github.com/projectdiscovery/fastdialer v0.0.16-0.20220515190512-d9a09aab8663 github.com/projectdiscovery/fdmax v0.0.3 - github.com/projectdiscovery/fileutil v0.0.0-20210926202739-6050d0acf73c + github.com/projectdiscovery/fileutil v0.0.0-20220215113056-ba188a0c8abc github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c - github.com/projectdiscovery/goflags v0.0.7 + github.com/projectdiscovery/goflags v0.0.8-0.20220412061559-5119d6086323 github.com/projectdiscovery/gologger v1.1.4 github.com/projectdiscovery/hmap v0.0.2-0.20210917080408-0fd7bd286bfa github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5 github.com/projectdiscovery/iputil v0.0.0-20210804143329-3a30fcde43f3 - github.com/projectdiscovery/mapcidr v0.0.8 + github.com/projectdiscovery/mapcidr v0.0.9 github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e - github.com/projectdiscovery/retryablehttp-go v1.0.2 + github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26d github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea github.com/projectdiscovery/stringsutil v0.0.0-20220208075244-7c05502ca8e9 - github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 - github.com/projectdiscovery/wappalyzergo v0.0.35 + github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 + github.com/projectdiscovery/wappalyzergo v0.0.45 github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/xid v1.4.0 github.com/smartystreets/assertions v1.0.0 // indirect go.etcd.io/bbolt v1.3.6 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/ratelimit v0.2.0 - golang.org/x/net v0.0.0-20210916014120-12bc252f5db8 - golang.org/x/sys v0.0.0-20210915083310-ed5796bab164 // indirect + golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 + golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect golang.org/x/text v0.3.7 ) @@ -47,14 +47,16 @@ require github.com/spaolacci/murmur3 v1.1.0 require ( github.com/PuerkitoBio/goquery v1.8.0 + github.com/RumbleDiscovery/jarm-go v0.0.6 + github.com/ammario/ipisp/v2 v2.0.0 github.com/mfonda/simhash v0.0.0-20151007195837-79f94a1100d6 ) require ( - github.com/RumbleDiscovery/jarm-go v0.0.6 // indirect - github.com/ammario/ipisp/v2 v2.0.0 // indirect + github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 // indirect github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect github.com/andybalholm/cascadia v1.3.1 // indirect + github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect @@ -73,10 +75,11 @@ require ( github.com/yl2chen/cidranger v1.0.2 // indirect github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521 // indirect github.com/zmap/zcrypto v0.0.0-20211005224000-2d0ffdec8a9b // indirect - golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392 // indirect + golang.org/x/crypto v0.0.0-20220210151621-f4118a5b28e2 // indirect golang.org/x/mod v0.4.2 // indirect golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect gopkg.in/ini.v1 v1.62.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) diff --git a/go.sum b/go.sum index 986f7557a..df8187b82 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,5 @@ -github.com/Masterminds/glide v0.13.2/go.mod h1:STyF5vcenH/rUqTEv+/hBXlSTo7KYwg2oc2f4tzPWic= -github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= -github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA= +github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809 h1:ZbFL+BDfBqegi+/Ssh7im5+aQfBRx6it+kHnC7jaDU8= +github.com/Mzack9999/go-http-digest-auth-client v0.6.1-0.20220414142836-eb8883508809/go.mod h1:upgc3Zs45jBDnBT4tVRgRcgm26ABpaP7MoTSdgysca4= github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U= github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI= github.com/RumbleDiscovery/jarm-go v0.0.6 h1:n3JEmOhPyfhmu1aeDEK/10Y2F+GMUYrtGFZmp4Yj0s4= @@ -9,23 +8,22 @@ github.com/RumbleDiscovery/rumble-tools v0.0.0-20201105153123-f2adbb3244d2/go.mo github.com/akrylysov/pogreb v0.10.0/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK4w= github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= -github.com/ammario/ipisp v1.0.0 h1:U4xdVMBFWm0/4sHrQ3hVMC+ygg/Ynm4/vdFdkVAex1o= -github.com/ammario/ipisp v1.0.0/go.mod h1:HM60VFpmEWyU+FisnTTHCeswaU3RW0dCVHihgIGUEGM= github.com/ammario/ipisp/v2 v2.0.0 h1:/aRMp5srZViiBfOUGzl/Esqae4s0MDDzm9buhGcZ0XU= github.com/ammario/ipisp/v2 v2.0.0/go.mod h1:bQ6KAL5LnYYEj6olUn+Bzv/im/4Esa5oGkbv9b+uOjo= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI= github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg= github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= +github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl3/e6D5CLfI0j/7hiIEtvGVFPCZ7Ei2oq8iQ= +github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 h1:ox2F0PSMlrAAiAdknSRMDrAr8mfxPCfSZolH+/qQnyQ= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08/go.mod h1:pCxVEbcm3AMg7ejXyorUXi6HQCzOIBf7zEDVPtw0/U4= -github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA= -github.com/corpix/uarand v0.1.1 h1:RMr1TWc9F4n5jiPDzFHtmaUXLKLNUFK0SgCLo4BhX/U= -github.com/corpix/uarand v0.1.1/go.mod h1:SFKZvkcRoLqVRFZ4u25xPmp6m9ktANfbpXZ7SJ0/FNU= +github.com/corpix/uarand v0.2.0 h1:U98xXwud/AVuCpkpgfPF7J5TQgr7R5tqT8VZP5KWbzE= +github.com/corpix/uarand v0.2.0/go.mod h1:/3Z1QIqWkDIhf6XWn/08/uMHoQ8JUoTIKc2iPchBOmM= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -91,9 +89,9 @@ github.com/microcosm-cc/bluemonday v1.0.18/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2 github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.35/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= -github.com/miekg/dns v1.1.46 h1:uzwpxRtSVxtcIZmz/4Uz6/Rn7G11DvsaslXoy5LxQio= github.com/miekg/dns v1.1.46/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/miekg/dns v1.1.48 h1:Ucfr7IIVyMBz4lRE8qmGUuZ4Wt3/ZGu9hmcMT3Uu4tQ= +github.com/miekg/dns v1.1.48/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -102,7 +100,6 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mreiferson/go-httpclient v0.0.0-20160630210159-31f0106b4474/go.mod h1:OQA4XLvDbMgS8P0CevmM4m9Q3Jq4phKUzcocxuGJ5m8= -github.com/ngdinhtoan/glide-cleanup v0.2.0/go.mod h1:UQzsmiDOb8YV3nOsCxK/c9zPpCZVNoHScRE3EO9pVMM= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= @@ -118,29 +115,31 @@ github.com/onsi/gomega v1.10.5 h1:7n6FEkpFmfCoo2t+YYqXH0evK+a9ICQz0xcAy9dYcaQ= github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7mt48= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e h1:7bwaFH1jvtOo5ndhTQgoA349ozhX+1dc4b6tbaPnBOA= github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e/go.mod h1:/IsapnEYiWG+yEDPXp0e8NWj3npzB9Ccy9lXEUJwMZs= -github.com/projectdiscovery/cdncheck v0.0.3 h1:li2/rUJmhVXSqRFyhJMqi6pdBX6ZxMnwzBfE0Kifj/g= -github.com/projectdiscovery/cdncheck v0.0.3/go.mod h1:EevMeCG1ogBoUJYaa0Mv9R1VUboDm/DiynId7DboKy0= +github.com/projectdiscovery/cdncheck v0.0.4-0.20220413175814-b47bc2d578b1 h1:QtTPPx0uu42AsQJiXT86/wqdHS7/iVcgz1VM38tjv20= +github.com/projectdiscovery/cdncheck v0.0.4-0.20220413175814-b47bc2d578b1/go.mod h1:EevMeCG1ogBoUJYaa0Mv9R1VUboDm/DiynId7DboKy0= github.com/projectdiscovery/clistats v0.0.8 h1:tjmWb15mqsPf/yrQXVHLe2ThZX/5+mgKSfZBKWWLh20= github.com/projectdiscovery/clistats v0.0.8/go.mod h1:lV6jUHAv2bYWqrQstqW8iVIydKJhWlVaLl3Xo9ioVGg= github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345 h1:jT6f/cdOpLkp9GAfRrxk57BUjYfIrR8E+AjMv5H5U4U= github.com/projectdiscovery/cryptoutil v0.0.0-20210805184155-b5d2512f9345/go.mod h1:clhQmPnt35ziJW1AhJRKyu8aygXCSoyWj6dtmZBRjjc= -github.com/projectdiscovery/fastdialer v0.0.15 h1:vOLk8jty+1H9osPRzUT6acD0XBSaglyLS3MlIQaYczk= -github.com/projectdiscovery/fastdialer v0.0.15/go.mod h1:Q28lw9oTpiZHq09uFG6YYYLUsUjsOypZ7PXWwQGBB80= +github.com/projectdiscovery/fastdialer v0.0.16-0.20220515190512-d9a09aab8663 h1:uxOMKHOajVaiz/pMUzc3nvWhICa/euZN+SoVzpqcPiE= +github.com/projectdiscovery/fastdialer v0.0.16-0.20220515190512-d9a09aab8663/go.mod h1:k4iAKJMOJJWpqgYA3tDXtNFRkL40H0DXQ0cmyTg5J5k= github.com/projectdiscovery/fdmax v0.0.3 h1:FM6lv9expZ/rEEBI9tkRh6tx3DV0gtpwzdc0h7bGPqg= github.com/projectdiscovery/fdmax v0.0.3/go.mod h1:NWRcaR7JTO7fC27H4jCl9n7Z+KIredwpgw1fV+4KrKI= -github.com/projectdiscovery/fileutil v0.0.0-20210926202739-6050d0acf73c h1:KDmCXhLLnS/Gc1VDyTxxamRzc8OmHCm1X+f8WQoaTRs= github.com/projectdiscovery/fileutil v0.0.0-20210926202739-6050d0acf73c/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= +github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0= +github.com/projectdiscovery/fileutil v0.0.0-20220215113056-ba188a0c8abc h1:dbDgsj26PW06L3zMo7AT08IbEqMd2u8QQ1BvlfMAY2w= +github.com/projectdiscovery/fileutil v0.0.0-20220215113056-ba188a0c8abc/go.mod h1:Pm0f+MWgDFMSSI9NBedNh48LyYPs8gD3Jd8DXGmp4aQ= github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c h1:1XRSp+44bhWudAWz+2+wHYJBHvDfE8mk9uWpzX+DU9k= github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c/go.mod h1:mBv7GRD5n3WNbFE9blG8ynzXTM5eh9MmwaK6EOyn6Pk= -github.com/projectdiscovery/goflags v0.0.7 h1:aykmRkrOgDyRwcvGrK3qp+9aqcjGfAMs/+LtRmtyxwk= github.com/projectdiscovery/goflags v0.0.7/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY= +github.com/projectdiscovery/goflags v0.0.8-0.20220412061559-5119d6086323 h1:M9DJMe3zMnvHkt5McU1IpoTQbBYc2aB1nsOBLqWlQOs= +github.com/projectdiscovery/goflags v0.0.8-0.20220412061559-5119d6086323/go.mod h1:uN+pHMLsWQoiZHUg/l0tqf/VdbX3+ecKfYz/H7b/+NA= github.com/projectdiscovery/gologger v1.0.1/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE= github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTtMKunQ8O2VI= github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY= @@ -156,8 +155,9 @@ github.com/projectdiscovery/iputil v0.0.0-20210804143329-3a30fcde43f3/go.mod h1: github.com/projectdiscovery/mapcidr v0.0.4/go.mod h1:ALOIj6ptkWujNoX8RdQwB2mZ+kAmKuLJBq9T5gR5wG0= github.com/projectdiscovery/mapcidr v0.0.6/go.mod h1:ZEBhMmBU3laUl3g9QGTrzJku1VJOzjdFwW01f/zVVzM= github.com/projectdiscovery/mapcidr v0.0.7/go.mod h1:7CzdUdjuLVI0s33dQ33lWgjg3vPuLFw2rQzZ0RxkT00= -github.com/projectdiscovery/mapcidr v0.0.8 h1:16U05F2x3o/jSTsxSCY2hCuCs9xOSwVxjo2zlsL4L4E= github.com/projectdiscovery/mapcidr v0.0.8/go.mod h1:7CzdUdjuLVI0s33dQ33lWgjg3vPuLFw2rQzZ0RxkT00= +github.com/projectdiscovery/mapcidr v0.0.9 h1:PIa09fMHdghlmkUeTgHP9bwYnb3k2wXXM2f6LMj26zg= +github.com/projectdiscovery/mapcidr v0.0.9/go.mod h1:zgsrc+UXwcLcBopUNboiI4tpTICbfdTyJZiBi2tx+NI= github.com/projectdiscovery/networkpolicy v0.0.1 h1:RGRuPlxE8WLFF9tdKSjTsYiTIKHNHW20Kl0nGGiRb1I= github.com/projectdiscovery/networkpolicy v0.0.1/go.mod h1:asvdg5wMy3LPVMGALatebKeOYH5n5fV5RCTv6DbxpIs= github.com/projectdiscovery/rawhttp v0.0.8-0.20210814181734-56cca67b6e7e h1:hcpGb5/gSn+kNUmzgodV1+sHDmFybuGhsuhrTqFebQY= @@ -168,8 +168,9 @@ github.com/projectdiscovery/retryabledns v1.0.13-0.20210916165024-76c5b76fd59a/g github.com/projectdiscovery/retryabledns v1.0.13 h1:Ogfv0fl3Nszb+Nq2S2qQmE+PJDlStVxyLkmTotMohZY= github.com/projectdiscovery/retryabledns v1.0.13/go.mod h1:EeqHcAPp0g2GljT4qkxKSAE47Dj0ZrJQ46R9ct3Muhk= github.com/projectdiscovery/retryablehttp-go v1.0.1/go.mod h1:SrN6iLZilNG1X4neq1D+SBxoqfAF4nyzvmevkTkWsek= -github.com/projectdiscovery/retryablehttp-go v1.0.2 h1:LV1/KAQU+yeWhNVlvveaYFsjBYRwXlNEq0PvrezMV0U= github.com/projectdiscovery/retryablehttp-go v1.0.2/go.mod h1:dx//aY9V247qHdsRf0vdWHTBZuBQ2vm6Dq5dagxrDYI= +github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26d h1:VR+tDkedzHIp1pGKIDcfPFt7J8KjcjxGsJvBAP6RXFQ= +github.com/projectdiscovery/retryablehttp-go v1.0.3-0.20220506110515-811d938bd26d/go.mod h1:t4buiLTB0HtI+62iHfGDqQVTv/i+8OhAKwaX93TGsFE= github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea h1:S+DC2tmKG93Om42cnTqrBfIv699pwSIhafqZvip+RIA= github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea/go.mod h1:QHXvznfPfA5f0AZUIBkbLapoUJJlsIDgUlkKva6dOr4= github.com/projectdiscovery/stringsutil v0.0.0-20210524051937-51dabe3b72c0/go.mod h1:TVSdZC0rRQeMIbsNSiGPhbmhyRtxqqtAGA9JiiNp2r4= @@ -178,10 +179,10 @@ github.com/projectdiscovery/stringsutil v0.0.0-20210823090203-2f5f137e8e1d/go.mo github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= github.com/projectdiscovery/stringsutil v0.0.0-20220208075244-7c05502ca8e9 h1:4fvUw6b4sS4GoWbHr60mJo3dI//4mGt3BuLx8Sz9aNw= github.com/projectdiscovery/stringsutil v0.0.0-20220208075244-7c05502ca8e9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I= -github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 h1:9dYmONRtwy+xP8UAGHxEQ0cxO3umc9qiFmnYsoDUps4= -github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1/go.mod h1:oXLErqOpqEAp/ueQlknysFxHO3CUNoSiDNnkiHG+Jpo= -github.com/projectdiscovery/wappalyzergo v0.0.35 h1:UDjCmOygrY0Q25ZH4jz5pEw67wPxO6ilHYJQoxgdfu4= -github.com/projectdiscovery/wappalyzergo v0.0.35/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= +github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2 h1:Es4VvKpJ2+jospbaC8lbRRAEgBtN6fzZh1FjXJQEUxA= +github.com/projectdiscovery/urlutil v0.0.0-20220603144104-f4c60e5107b2/go.mod h1:+4eLbe7wuSIiUiwO0Kq5sWB1SYJAxTWAaPP5bh2VV3o= +github.com/projectdiscovery/wappalyzergo v0.0.45 h1:b2OJ4/FrQg6YBDwuK+xMJj/3+h6u+uHuZYURGYPqw3I= +github.com/projectdiscovery/wappalyzergo v0.0.45/go.mod h1:vS+npIOANv7eKsEtODsyRQt2n1v8VofCwj2gjmq72EM= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= @@ -200,14 +201,14 @@ github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0b github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6 h1:TtyC78WMafNW8QFfv3TeP3yWNDG+uxNkk9vOrnDu6JA= @@ -237,8 +238,9 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392 h1:xYJJ3S178yv++9zXV/hnr29plCAGO9vAFG9dorqaFQc= golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20220210151621-f4118a5b28e2 h1:XdAboW3BNMv9ocSCOk/u1MFioZGzCNkiJZ19v9Oe3Ig= +golang.org/x/crypto v0.0.0-20220210151621-f4118a5b28e2/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= @@ -260,8 +262,10 @@ golang.org/x/net v0.0.0-20210414194228-064579744ee0/go.mod h1:9tjilg8BloeKEkVJvy golang.org/x/net v0.0.0-20210521195947-fe42d452be8f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210916014120-12bc252f5db8 h1:/6y1LfuqNuQdHAm0jjtPtgRcxIxjVZgm5OTu8/QhZvk= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 h1:HVyaeDAYux4pnY+D/SiwmLOR36ewZ4iGQIIrtnuCjFA= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -291,11 +295,13 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210415045647-66c3f260301c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210915083310-ed5796bab164 h1:7ZDGnxgHAMw7thfC5bEos0RDAccZKxioiWBhfIe+tvw= -golang.org/x/sys v0.0.0-20210915083310-ed5796bab164/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= diff --git a/runner/banner.go b/runner/banner.go index 47b86df12..6c09f117d 100644 --- a/runner/banner.go +++ b/runner/banner.go @@ -8,11 +8,11 @@ const banner = ` / __ \/ __/ __/ __ \| / / / / / /_/ /_/ /_/ / | /_/ /_/\__/\__/ .___/_/|_| - /_/ v1.2.1 + /_/ v1.2.2 ` // Version is the current version of httpx -const Version = `v1.2.1` +const Version = `v1.2.2` // showBanner is used to show the banner to the user func showBanner() { diff --git a/runner/filteroperator.go b/runner/filteroperator.go new file mode 100644 index 000000000..69f93cca0 --- /dev/null +++ b/runner/filteroperator.go @@ -0,0 +1,48 @@ +package runner + +import ( + "fmt" + "strings" + "time" +) + +var ( + greaterThan = ">" + lessThan = "<" + equal = "=" + greaterThanEq = ">=" + lessThanEq = "<=" + notEq = "!=" + compareOperators = []string{greaterThanEq, lessThanEq, equal, lessThan, greaterThan, notEq} +) + +type FilterOperator struct { + flag string +} + +// Parse the given value into operator and value pair +func (f FilterOperator) Parse(flagValue string) (string, time.Duration, error) { + var ( + operator string + value time.Duration + err error + ) + for _, op := range compareOperators { + if strings.Contains(flagValue, op) { + splittedFlagValue := strings.SplitAfter(flagValue, op) + operator = strings.Trim(splittedFlagValue[0], " ") + timeVal := strings.Trim(splittedFlagValue[1], " ") + value, err = time.ParseDuration(timeVal) + if err != nil && strings.Contains(err.Error(), "missing unit") { + value, _ = time.ParseDuration(fmt.Sprintf("%ss", timeVal)) + } else if err != nil { + return operator, value, fmt.Errorf("invalid value provided for %s", f.flag) + } + break + } + } + if operator == "" { + return operator, value, fmt.Errorf("invalid operator provided for %s, valid operators are %s", f.flag, strings.Join(compareOperators, ",")) + } + return operator, value, nil +} diff --git a/runner/options.go b/runner/options.go index 509b6bcb4..25407e32a 100644 --- a/runner/options.go +++ b/runner/options.go @@ -1,11 +1,13 @@ package runner import ( + "fmt" "math" "os" "regexp" "strings" - "github.com/projectdiscovery/httpx/common/slice" + + "github.com/projectdiscovery/cdncheck" "github.com/projectdiscovery/fileutil" "github.com/projectdiscovery/goconfig" "github.com/projectdiscovery/goflags" @@ -16,6 +18,7 @@ import ( "github.com/projectdiscovery/httpx/common/customlist" customport "github.com/projectdiscovery/httpx/common/customports" fileutilz "github.com/projectdiscovery/httpx/common/fileutil" + "github.com/projectdiscovery/httpx/common/slice" "github.com/projectdiscovery/httpx/common/stringz" ) @@ -27,6 +30,8 @@ const ( DefaultOutputDirectory = "output" ) +var defaultProviders = strings.Join(cdncheck.GetDefaultProviders(), ", ") + type scanOptions struct { Methods []string StoreResponseDirectory string @@ -229,6 +234,11 @@ type Options struct { Hashes string Jarm bool Asn bool + OutputMatchCdn goflags.NormalizedStringSlice + OutputFilterCdn goflags.NormalizedStringSlice + SniName string + OutputMatchResponseTime string + OutputFilterResponseTime string } // ParseOptions parses the command line options for application @@ -274,6 +284,8 @@ func ParseOptions() *Options { flagSet.NormalizedStringSliceVarP(&options.OutputMatchFavicon, "match-favicon", "mfc", []string{}, "match response with specified favicon hash (-mfc 1494302000)"), flagSet.StringVarP(&options.OutputMatchString, "match-string", "ms", "", "match response with specified string (-ms admin)"), flagSet.StringVarP(&options.OutputMatchRegex, "match-regex", "mr", "", "match response with specified regex (-mr admin)"), + flagSet.NormalizedStringSliceVarP(&options.OutputMatchCdn, "match-cdn", "mcdn", []string{}, fmt.Sprintf("match host with specified cdn provider (%s)", defaultProviders)), + flagSet.StringVarP(&options.OutputMatchResponseTime, "match-response-time", "mrt", "", "match response with specified response time in seconds (-mrt '< 1')"), ) createGroup(flagSet, "extractor", "Extractor", @@ -288,6 +300,8 @@ func ParseOptions() *Options { flagSet.NormalizedStringSliceVarP(&options.OutputFilterFavicon, "filter-favicon", "ffc", []string{}, "filter response with specified favicon hash (-mfc 1494302000)"), flagSet.StringVarP(&options.OutputFilterString, "filter-string", "fs", "", "filter response with specified string (-fs admin)"), flagSet.StringVarP(&options.OutputFilterRegex, "filter-regex", "fe", "", "filter response with specified regex (-fe admin)"), + flagSet.NormalizedStringSliceVarP(&options.OutputFilterCdn, "filter-cdn", "fcdn", []string{}, fmt.Sprintf("filter host with specified cdn provider (%s)", defaultProviders)), + flagSet.StringVarP(&options.OutputFilterResponseTime, "filter-response-time", "frt", "", "filter response with specified response time in seconds (-frt '> 1')"), ) createGroup(flagSet, "rate-limit", "Rate-Limit", @@ -323,6 +337,7 @@ func ParseOptions() *Options { flagSet.NormalizedStringSliceVarP(&options.Resolvers, "resolvers", "r", []string{}, "list of custom resolver (file or comma separated)"), flagSet.Var(&options.Allow, "allow", "allowed list of IP/CIDR's to process (file or comma separated)"), flagSet.Var(&options.Deny, "deny", "denied list of IP/CIDR's to process (file or comma separated)"), + flagSet.StringVarP(&options.SniName, "sni-name", "sni", "", "Custom TLS SNI name"), flagSet.BoolVar(&options.RandomAgent, "random-agent", true, "Enable Random User-Agent to use"), flagSet.VarP(&options.CustomHeaders, "header", "H", "custom http headers to send with request"), flagSet.StringVarP(&options.HTTPProxy, "proxy", "http-proxy", "", "http proxy to use (eg http://127.0.0.1:8080)"), @@ -478,6 +493,9 @@ func (options *Options) validateOptions() { } } } + if len(options.OutputMatchCdn) > 0 || len(options.OutputFilterCdn) > 0 { + options.OutputCDN = true + } } // configureOutput configures the output on the screen @@ -495,6 +513,9 @@ func (options *Options) configureOutput() { if options.Silent { gologger.DefaultLogger.SetMaxLevel(levels.LevelSilent) } + if len(options.OutputMatchResponseTime) > 0 || len(options.OutputFilterResponseTime) > 0 { + options.OutputResponseTime = true + } } func (options *Options) configureResume() error { diff --git a/runner/runner.go b/runner/runner.go index 73c0270d4..610d2f197 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -125,6 +125,7 @@ func New(options *Options) (*Runner, error) { value = strings.TrimSpace(tokens[1]) httpxOptions.CustomHeaders[key] = value } + httpxOptions.SniName = options.SniName runner.hp, err = httpx.New(&httpxOptions) if err != nil { @@ -591,7 +592,76 @@ func (r *Runner) RunEnumeration() { if len(r.options.matchWordsCount) > 0 && !slice.IntSliceContains(r.options.matchWordsCount, resp.Words) { continue } - + if len(r.options.OutputMatchCdn) > 0 && !stringsutil.EqualFoldAny(resp.CDNName, r.options.OutputMatchCdn...) { + continue + } + if len(r.options.OutputFilterCdn) > 0 && stringsutil.EqualFoldAny(resp.CDNName, r.options.OutputFilterCdn...) { + continue + } + if r.options.OutputMatchResponseTime != "" { + filterOps := FilterOperator{flag: "-mrt, -match-response-time"} + operator, value, err := filterOps.Parse(r.options.OutputMatchResponseTime) + if err != nil { + gologger.Fatal().Msg(err.Error()) + } + respTimeTaken, _ := time.ParseDuration(resp.ResponseTime) + switch operator { + // take negation of >= and > + case greaterThanEq, greaterThan: + if respTimeTaken < value { + continue + } + // take negation of <= and < + case lessThanEq, lessThan: + if respTimeTaken > value { + continue + } + // take negation of = + case equal: + if respTimeTaken != value { + continue + } + // take negation of != + case notEq: + if respTimeTaken == value { + continue + } + } + } + if r.options.OutputFilterResponseTime != "" { + filterOps := FilterOperator{flag: "-frt, -filter-response-time"} + operator, value, err := filterOps.Parse(r.options.OutputFilterResponseTime) + if err != nil { + gologger.Fatal().Msg(err.Error()) + } + respTimeTaken, _ := time.ParseDuration(resp.ResponseTime) + switch operator { + case greaterThanEq: + if respTimeTaken >= value { + continue + } + case lessThanEq: + if respTimeTaken <= value { + continue + } + case equal: + if respTimeTaken == value { + continue + } + case lessThan: + if respTimeTaken < value { + continue + } + case greaterThan: + if respTimeTaken > value { + continue + } + case notEq: + if respTimeTaken != value { + continue + } + } + } row := resp.str if r.options.JSONOutput { row = resp.JSON(&r.scanopts) @@ -668,7 +738,7 @@ func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx. for _, method := range scanopts.Methods { for _, prot := range protocols { wg.Add() - go func(target, method, protocol string) { + go func(target httpx.Target, method, protocol string) { defer wg.Done() result := r.analyze(hp, protocol, target, method, t, scanopts) output <- result @@ -709,10 +779,10 @@ func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx. for _, wantedProtocol := range wantedProtocols { for _, method := range scanopts.Methods { wg.Add() - go func(port int, method, protocol string) { + go func(port int, target httpx.Target, method, protocol string) { defer wg.Done() - h, _ := urlutil.ChangePort(target, fmt.Sprint(port)) - result := r.analyze(hp, protocol, h, method, t, scanopts) + target.Host, _ = urlutil.ChangePort(target.Host, fmt.Sprint(port)) + result := r.analyze(hp, protocol, target, method, t, scanopts) output <- result if scanopts.TLSProbe && result.TLSData != nil { scanopts.TLSProbe = false @@ -729,7 +799,7 @@ func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx. r.process(tt, wg, hp, protocol, scanopts, output) } } - }(port, method, wantedProtocol) + }(port, target, method, wantedProtocol) } } } @@ -740,8 +810,8 @@ func (r *Runner) process(t string, wg *sizedwaitgroup.SizedWaitGroup, hp *httpx. } // returns all the targets within a cidr range or the single target -func (r *Runner) targets(hp *httpx.HTTPX, target string) chan string { - results := make(chan string) +func (r *Runner) targets(hp *httpx.HTTPX, target string) chan httpx.Target { + results := make(chan httpx.Target) go func() { defer close(results) @@ -749,7 +819,7 @@ func (r *Runner) targets(hp *httpx.HTTPX, target string) chan string { // * // spaces if strings.ContainsAny(target, "*") || strings.HasPrefix(target, ".") { - // trim * and/or . (prefix) from the target to return the domain instead of wildard + // trim * and/or . (prefix) from the target to return the domain instead of wilcard target = strings.TrimPrefix(strings.Trim(target, "*"), ".") if !r.testAndSet(target) { return @@ -763,54 +833,42 @@ func (r *Runner) targets(hp *httpx.HTTPX, target string) chan string { return } for _, ip := range cidrIps { - results <- ip + results <- httpx.Target{Host: ip} } } else if r.options.ProbeAllIPS { URL, err := urlutil.Parse(target) if err != nil { - results <- target + results <- httpx.Target{Host: target} } ips, _, err := getDNSData(hp, URL.Host) if err != nil || len(ips) == 0 { - results <- target + results <- httpx.Target{Host: target} } for _, ip := range ips { - results <- strings.Join([]string{ip, target}, ",") + results <- httpx.Target{Host: target, CustomIP: ip} } + } else if idxComma := strings.Index(target, ","); idxComma > 0 { + results <- httpx.Target{Host: target[idxComma+1:], CustomHost: target[:idxComma]} } else { - results <- target + results <- httpx.Target{Host: target} } }() return results } -func (r *Runner) analyze(hp *httpx.HTTPX, protocol, domain, method, origInput string, scanopts *scanOptions) Result { +func (r *Runner) analyze(hp *httpx.HTTPX, protocol string, target httpx.Target, method, origInput string, scanopts *scanOptions) Result { origProtocol := protocol if protocol == httpx.HTTPorHTTPS || protocol == httpx.HTTPandHTTPS { protocol = httpx.HTTPS } retried := false retry: - var customHost, customIP string - if scanopts.ProbeAllIPS { - parts := strings.SplitN(domain, ",", 2) - if len(parts) == 2 { - customIP = parts[0] - domain = parts[1] - } - } - if scanopts.VHostInput { - parts := strings.Split(domain, ",") - //nolint:gomnd // not a magic number - if len(parts) != 2 { - return Result{Input: origInput} - } - domain = parts[0] - customHost = parts[1] + if scanopts.VHostInput && target.CustomHost == "" { + return Result{Input: origInput} } - URL, err := urlutil.Parse(domain) + URL, err := urlutil.Parse(target.Host) if err != nil { - return Result{URL: domain, Input: origInput, err: err} + return Result{URL: target.Host, Input: origInput, err: err} } // check if we have to skip the host:port as a result of a previous failure @@ -818,19 +876,19 @@ retry: if r.options.HostMaxErrors >= 0 && r.HostErrorsCache.Has(hostPort) { numberOfErrors, err := r.HostErrorsCache.GetIFPresent(hostPort) if err == nil && numberOfErrors.(int) >= r.options.HostMaxErrors { - return Result{URL: domain, err: errors.New("skipping as previously unresponsive")} + return Result{URL: target.Host, err: errors.New("skipping as previously unresponsive")} } } // check if the combination host:port should be skipped if belonging to a cdn if r.skipCDNPort(URL.Host, URL.Port) { gologger.Debug().Msgf("Skipping cdn target: %s:%s\n", URL.Host, URL.Port) - return Result{URL: domain, Input: origInput, err: errors.New("cdn target only allows ports 80 and 443")} + return Result{URL: target.Host, Input: origInput, err: errors.New("cdn target only allows ports 80 and 443")} } URL.Scheme = protocol - if !strings.Contains(domain, URL.Port) { + if !strings.Contains(target.Host, URL.Port) { URL.Port = "" } @@ -845,9 +903,14 @@ retry: URL.RequestURI += scanopts.RequestURI } var req *retryablehttp.Request - if customIP != "" { - customHost = URL.Host - ctx := context.WithValue(context.Background(), "ip", customIP) //nolint + if target.CustomIP != "" { + var requestIP string + if iputil.IsIPv6(target.CustomIP) { + requestIP = fmt.Sprintf("[%s]", target.CustomIP) + } else { + requestIP = target.CustomIP + } + ctx := context.WithValue(context.Background(), "ip", requestIP) //nolint req, err = hp.NewRequestWithContext(ctx, method, URL.String()) } else { req, err = hp.NewRequest(method, URL.String()) @@ -856,8 +919,8 @@ retry: return Result{URL: URL.String(), Input: origInput, err: err} } - if customHost != "" { - req.Host = customHost + if target.CustomHost != "" { + req.Host = target.CustomHost } if !scanopts.LeaveDefaultPorts { @@ -1128,7 +1191,15 @@ retry: r.stats.IncrementCounter("requests", 1) } } - ip := hp.Dialer.GetDialedIP(URL.Host) + + var ip string + if target.CustomIP != "" { + ip = target.CustomIP + } else { + // hp.Dialer.GetDialedIP would return only the last dialed one + ip = hp.Dialer.GetDialedIP(URL.Host) + } + var asnResponse interface{ String() string } if r.options.Asn { lookupResult, err := ipisp.LookupIP(context.Background(), net.ParseIP(ip)) @@ -1152,15 +1223,12 @@ retry: builder.WriteRune(']') } } - // hp.Dialer.GetDialedIP would return only the last dialed one - if customIP != "" { - ip = customIP - } + if scanopts.OutputIP || scanopts.ProbeAllIPS { builder.WriteString(fmt.Sprintf(" [%s]", ip)) } - ips, cnames, err := getDNSData(hp, domain) + ips, cnames, err := getDNSData(hp, URL.Host) if err != nil { ips = append(ips, ip) } @@ -1293,7 +1361,7 @@ retry: } jarmhash := "" if r.options.Jarm { - jarmhash = hashes.Jarm(fullURL,r.options.Timeout) + jarmhash = hashes.Jarm(fullURL, r.options.Timeout) builder.WriteString(" [") if !scanopts.OutputWithNoColor { builder.WriteString(aurora.Magenta(jarmhash).String()) @@ -1332,7 +1400,7 @@ retry: } writeErr := ioutil.WriteFile(responsePath, []byte(respRaw), 0644) if writeErr != nil { - gologger.Warning().Msgf("Could not write response at path '%s', to disk: %s", responsePath, writeErr) + gologger.Error().Msgf("Could not write response at path '%s', to disk: %s", responsePath, writeErr) } if scanopts.StoreChain && resp.HasChain() { domainFile = strings.ReplaceAll(domainFile, ".txt", ".chain.txt") @@ -1435,49 +1503,49 @@ func (o AsnResponse) String() string { // Result of a scan type Result struct { - Timestamp time.Time `json:"timestamp,omitempty" csv:"timestamp"` - Request string `json:"request,omitempty" csv:"request"` - ResponseHeader string `json:"response-header,omitempty" csv:"response-header"` - Scheme string `json:"scheme,omitempty" csv:"scheme"` - Port string `json:"port,omitempty" csv:"port"` - Path string `json:"path,omitempty" csv:"path"` - A []string `json:"a,omitempty" csv:"a"` - CNAMEs []string `json:"cnames,omitempty" csv:"cnames"` + Timestamp time.Time `json:"timestamp,omitempty" csv:"timestamp"` + ASN interface{} `json:"asn,omitempty" csv:"asn"` + err error + CSPData *httpx.CSPData `json:"csp,omitempty" csv:"csp"` + TLSData *cryptoutil.TLSData `json:"tls-grab,omitempty" csv:"tls-grab"` + Hashes map[string]string `json:"hashes,omitempty" csv:"hashes"` + CDNName string `json:"cdn-name,omitempty" csv:"cdn-name"` + Port string `json:"port,omitempty" csv:"port"` raw string URL string `json:"url,omitempty" csv:"url"` Input string `json:"input,omitempty" csv:"input"` Location string `json:"location,omitempty" csv:"location"` Title string `json:"title,omitempty" csv:"title"` str string - err error - Error string `json:"error,omitempty" csv:"error"` - WebServer string `json:"webserver,omitempty" csv:"webserver"` - ResponseBody string `json:"response-body,omitempty" csv:"response-body"` - ContentType string `json:"content-type,omitempty" csv:"content-type"` - Method string `json:"method,omitempty" csv:"method"` - Host string `json:"host,omitempty" csv:"host"` - ContentLength int `json:"content-length,omitempty" csv:"content-length"` - ChainStatusCodes []int `json:"chain-status-codes,omitempty" csv:"chain-status-codes"` - StatusCode int `json:"status-code,omitempty" csv:"status-code"` - TLSData *cryptoutil.TLSData `json:"tls-grab,omitempty" csv:"tls-grab"` - CSPData *httpx.CSPData `json:"csp,omitempty" csv:"csp"` - VHost bool `json:"vhost,omitempty" csv:"vhost"` - WebSocket bool `json:"websocket,omitempty" csv:"websocket"` - Pipeline bool `json:"pipeline,omitempty" csv:"pipeline"` - HTTP2 bool `json:"http2,omitempty" csv:"http2"` - CDN bool `json:"cdn,omitempty" csv:"cdn"` - CDNName string `json:"cdn-name,omitempty" csv:"cdn-name"` - ResponseTime string `json:"response-time,omitempty" csv:"response-time"` - Technologies []string `json:"technologies,omitempty" csv:"technologies"` - Chain []httpx.ChainItem `json:"chain,omitempty" csv:"chain"` - FinalURL string `json:"final-url,omitempty" csv:"final-url"` - Failed bool `json:"failed" csv:"failed"` - FavIconMMH3 string `json:"favicon-mmh3,omitempty" csv:"favicon-mmh3"` - Hashes map[string]string `json:"hashes,omitempty" csv:"hashes"` - ASN interface{} `json:"asn,omitempty" csv:"asn"` - Lines int `json:"lines" csv:"lines"` - Words int `json:"words" csv:"words"` - Jarm string `json:"jarm,omitempty" csv:"jarm"` + Scheme string `json:"scheme,omitempty" csv:"scheme"` + Error string `json:"error,omitempty" csv:"error"` + WebServer string `json:"webserver,omitempty" csv:"webserver"` + ResponseBody string `json:"response-body,omitempty" csv:"response-body"` + ContentType string `json:"content-type,omitempty" csv:"content-type"` + Method string `json:"method,omitempty" csv:"method"` + Host string `json:"host,omitempty" csv:"host"` + Path string `json:"path,omitempty" csv:"path"` + FavIconMMH3 string `json:"favicon-mmh3,omitempty" csv:"favicon-mmh3"` + FinalURL string `json:"final-url,omitempty" csv:"final-url"` + ResponseHeader string `json:"response-header,omitempty" csv:"response-header"` + Request string `json:"request,omitempty" csv:"request"` + ResponseTime string `json:"response-time,omitempty" csv:"response-time"` + Jarm string `json:"jarm,omitempty" csv:"jarm"` + ChainStatusCodes []int `json:"chain-status-codes,omitempty" csv:"chain-status-codes"` + A []string `json:"a,omitempty" csv:"a"` + CNAMEs []string `json:"cnames,omitempty" csv:"cnames"` + Technologies []string `json:"technologies,omitempty" csv:"technologies"` + Chain []httpx.ChainItem `json:"chain,omitempty" csv:"chain"` + Words int `json:"words" csv:"words"` + Lines int `json:"lines" csv:"lines"` + StatusCode int `json:"status-code,omitempty" csv:"status-code"` + ContentLength int `json:"content-length,omitempty" csv:"content-length"` + Failed bool `json:"failed" csv:"failed"` + VHost bool `json:"vhost,omitempty" csv:"vhost"` + WebSocket bool `json:"websocket,omitempty" csv:"websocket"` + CDN bool `json:"cdn,omitempty" csv:"cdn"` + HTTP2 bool `json:"http2,omitempty" csv:"http2"` + Pipeline bool `json:"pipeline,omitempty" csv:"pipeline"` } // JSON the result