Merge pull request #314 from projectdiscovery/dev

v1.1.0

.github/dependabot.yml

@@ -11,6 +11,7 @@ updates:
     directory: "/"
     schedule:
       interval: "weekly"
+    target-branch: "dev"
     commit-message:
       prefix: "chore"
       include: "scope"
@@ -20,6 +21,7 @@ updates:
     directory: "/"
     schedule:
       interval: "weekly"
+    target-branch: "dev"
     commit-message:
       prefix: "chore"
       include: "scope"
@@ -29,6 +31,7 @@ updates:
     directory: "/"
     schedule:
       interval: "weekly"
+    target-branch: "dev"
     commit-message:
       prefix: "chore"
-      include: "scope"
+      include: "scope"

.github/workflows/build-test.yml

@@ -0,0 +1,27 @@
+name: 🔨 Build Test
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+
+
+jobs:  
+  build:
+    name: Test Builds
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.15
+
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Test
+        run: go test .
+        working-directory: cmd/httpx/
+
+      - name: Build
+        run: go build .
+        working-directory: cmd/httpx/

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,38 @@
+name: 🚨 CodeQL Analysis
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches:
+      - dev
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'go' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.github/workflows/dockerhub-push.yml

@@ -1,17 +1,34 @@
-# dockerhub-push pushes docker build to dockerhub automatically
-# on the creation of a new release
-name: Publish to Dockerhub on creation of a new release
-on: 
+name: 🌥 Docker Push
+
+on:
   release:
     types: [published]
+  workflow_dispatch:
+
 jobs:
-  build:
+  docker:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@master
-    - name: Publish to Dockerhub Registry
-      uses: elgohr/Publish-Docker-Github-Action@master
-      with:
-        name: projectdiscovery/httpx
-        username: ${{ secrets.DOCKER_USERNAME }}
-        password: ${{ secrets.DOCKER_PASSWORD }}
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      -
+        name: Login to DockerHub
+        uses: docker/login-action@v1 
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_TOKEN }}
+      -
+        name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64,linux/arm
+          push: true
+          tags: projectdiscovery/httpx:latest

.github/workflows/lint-test.yml

@@ -0,0 +1,19 @@
+name: 🙏🏻 Lint Test
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  lint:
+    name: Lint Test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Run golangci-lint
+        uses: golangci/golangci-lint-action@v2
+        with:
+          version: latest
+          args: --timeout 5m
+          working-directory: .

.github/workflows/release.yml → .github/workflows/release-binary.yml

@@ -1,8 +1,9 @@
-name: Release
+name: 🎉 Release Binary
 on:
   create:
     tags:
       - v*
+  workflow_dispatch:
 
 jobs: 
   release: 
@@ -17,12 +18,13 @@ jobs:
         name: "Set up Go"
         uses: actions/setup-go@v2
         with: 
-          go-version: 1.14
+          go-version: 1.16
       - 
         env: 
           GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
         name: "Create release on GitHub"
         uses: goreleaser/goreleaser-action@v2
         with: 
           args: "release --rm-dist"
-          version: latest
+          version: latest
+          workdir: .

.golangci.yml

@@ -65,7 +65,6 @@ linters:
     - gocritic
     - gofmt
     - goimports
-    - gomnd
     - goprintffuncname
     - gosimple
     - govet

README.md

@@ -286,7 +286,7 @@ https://173.0.84.34
 
 
 ```sh
-▶ subfinder -d hackerone.com | httpx -title -tech-detect -status-code -title -follow-redirects
+▶ subfinder -d hackerone.com | httpx -title -tech-detect -status-code -follow-redirects
 
     __    __  __       _  __
    / /_  / /_/ /_____ | |/ /

common/httpx/httpx.go

@@ -126,6 +126,8 @@ func New(options *Options) (*HTTPX, error) {
 func (h *HTTPX) Do(req *retryablehttp.Request) (*Response, error) {
 	timeStart := time.Now()
 
+	var gzipRetry bool
+get_response:
 	httpresp, err := h.getResponse(req)
 	if err != nil {
 		return nil, err
@@ -138,6 +140,13 @@ func (h *HTTPX) Do(req *retryablehttp.Request) (*Response, error) {
 	// httputil.DumpResponse does not handle websockets
 	headers, rawResp, err := pdhttputil.DumpResponseHeadersAndRaw(httpresp)
 	if err != nil {
+		// Edge case - some servers respond with gzip encoding header but uncompressed body, in this case the standard library configures the reader as gzip, triggering an error when read.
+		// The bytes slice is not accessible because of abstraction, therefore we need to perform the request again tampering the Accept-Encoding header
+		if !gzipRetry && strings.Contains(err.Error(), "gzip: invalid header") {
+			gzipRetry = true
+			req.Header.Set("Accept-Encoding", "identity")
+			goto get_response
+		}
 		return nil, err
 	}
 	resp.Raw = rawResp

go.mod

@@ -6,26 +6,27 @@ require (
 	github.com/corpix/uarand v0.1.1
 	github.com/hbakhtiyor/strsim v0.0.0-20190107154042-4d2bbb273edf
 	github.com/logrusorgru/aurora v2.0.3+incompatible
-	github.com/microcosm-cc/bluemonday v1.0.9
+	github.com/microcosm-cc/bluemonday v1.0.13
 	github.com/miekg/dns v1.1.42 // indirect
 	github.com/pkg/errors v0.9.1
 	github.com/projectdiscovery/cdncheck v0.0.2
 	github.com/projectdiscovery/clistats v0.0.8
-	github.com/projectdiscovery/fastdialer v0.0.10-0.20210429154054-50371bb58a8d
+	github.com/projectdiscovery/fastdialer v0.0.11-0.20210515173917-52343fce9a26
 	github.com/projectdiscovery/fdmax v0.0.3
 	github.com/projectdiscovery/gologger v1.1.4
 	github.com/projectdiscovery/hmap v0.0.1
 	github.com/projectdiscovery/httputil v0.0.0-20210508183653-2e37c34b438d
 	github.com/projectdiscovery/iputil v0.0.0-20210429152401-c18a5408ca46
 	github.com/projectdiscovery/mapcidr v0.0.7
-	github.com/projectdiscovery/rawhttp v0.0.6
-	github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210524224054-9fbe1f2b0727
-	github.com/projectdiscovery/stringsutil v0.0.0-20210525140246-1de418be6fda // indirect
+	github.com/projectdiscovery/rawhttp v0.0.7
+	github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210526144436-e15804ddc7dc
+	github.com/projectdiscovery/stringsutil v0.0.0-20210525214416-25d8a5c6972b // indirect
 	github.com/projectdiscovery/urlutil v0.0.0-20210525140139-b874f06ad921
-	github.com/projectdiscovery/wappalyzergo v0.0.4
+	github.com/projectdiscovery/wappalyzergo v0.0.7
 	github.com/remeh/sizedwaitgroup v1.0.0
 	github.com/rs/xid v1.3.0
-	golang.org/x/net v0.0.0-20210525063256-abc453219eb5
-	golang.org/x/sys v0.0.0-20210521203332-0cec03c779c1 // indirect
+	go.uber.org/atomic v1.8.0 // indirect
+	golang.org/x/net v0.0.0-20210614182718-04defd469f4e
+	golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 // indirect
 	golang.org/x/text v0.3.6
 )