Skip to content

Commit

Permalink
Merge pull request #568 from projectdiscovery/dev
Browse files Browse the repository at this point in the history
v1.2.1 Release
  • Loading branch information
ehsandeep authored Apr 8, 2022
2 parents 87b9ea9 + 3bb6184 commit 7d28ce4
Show file tree
Hide file tree
Showing 21 changed files with 390 additions and 53 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/build-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ jobs:
go-version: 1.17

- name: Check out code
uses: actions/checkout@v2
uses: actions/checkout@v3

- name: Build
run: go build .
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/codeql-analysis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ jobs:

steps:
- name: Checkout repository
uses: actions/checkout@v2
uses: actions/checkout@v3

# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/dockerhub-push.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v2
uses: actions/checkout@v3

- name: Get Github tag
id: meta
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/functional-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ jobs:
go-version: 1.17

- name: Check out code
uses: actions/checkout@v2
uses: actions/checkout@v3

- name: Functional Tests
run: |
Expand Down
6 changes: 5 additions & 1 deletion .github/workflows/lint-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,12 @@ jobs:
name: Lint Test
runs-on: ubuntu-latest
steps:
- name: Set up Go
uses: actions/setup-go@v2
with:
go-version: 1.17
- name: Checkout code
uses: actions/checkout@v2
uses: actions/checkout@v3
- name: Run golangci-lint
uses: golangci/golangci-lint-action@v3.1.0
with:
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/release-binary.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: "Check out code"
uses: actions/checkout@v2
uses: actions/checkout@v3
with:
fetch-depth: 0

Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/sonarcloud.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ jobs:
name: SonarCloud
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v3
with:
fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis

Expand Down
4 changes: 2 additions & 2 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
FROM golang:1.17.7-alpine AS builder
FROM golang:1.18.0-alpine AS builder
RUN apk add --no-cache git
RUN go install -v github.com/projectdiscovery/httpx/cmd/httpx@latest

FROM alpine:3.15.0
FROM alpine:3.15.3
RUN apk -U upgrade --no-cache \
&& apk add --no-cache bind-tools ca-certificates
COPY --from=builder /go/bin/httpx /usr/local/bin/
Expand Down
58 changes: 57 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ httpx is a fast and multi-purpose HTTP toolkit allow to run multiple probers usi
| Favicon Hash | false | Probe Status | false |
| Body Hash | true | Header Hash | true |
| Redirect chain | false | URL Scheme | true |

| JARM Hash | false | ASN | false |

# Installation Instructions

Expand Down Expand Up @@ -93,6 +93,7 @@ PROBES:
-location display response redirect location
-favicon display mmh3 hash for '/favicon.ico' file
-hash string display response body hash (supported: md5,mmh3,simhash,sha1,sha256,sha512)
-jarm display jarm fingerprint hash
-rt, -response-time display response time
-lc, -line-count display response body line count
-wc, -word-count display response body word count
Expand All @@ -103,6 +104,7 @@ PROBES:
-websocket display server using websocket
-ip display host ip
-cname display host cname
-asn display host asn information
-cdn display cdn in use
-probe display probe status

Expand Down Expand Up @@ -334,6 +336,60 @@ https://mta-sts.hackerone.com/favicon.ico [-1700323260]
https://www.hackerone.com/favicon.ico [778073381]
```

### [JARM Fingerprint](https://github.com/salesforce/jarm)


```console
subfinder -d hackerone.com -silent | httpx -jarm
__ __ __ _ __
/ /_ / /_/ /_____ | |/ /
/ __ \/ __/ __/ __ \| /
/ / / / /_/ /_/ /_/ / |
/_/ /_/\__/\__/ .___/_/|_|
/_/ v1.2.1

projectdiscovery.io

Use with caution. You are responsible for your actions.
Developers assume no liability and are not responsible for any misuse or damage.
https://www.hackerone.com [29d3dd00029d29d00042d43d00041d5de67cc9954cc85372523050f20b5007]
https://mta-sts.hackerone.com [29d29d00029d29d00042d43d00041d2aa5ce6a70de7ba95aef77a77b00a0af]
https://mta-sts.managed.hackerone.com [29d29d00029d29d00042d43d00041d2aa5ce6a70de7ba95aef77a77b00a0af]
https://docs.hackerone.com [29d29d00029d29d00042d43d00041d2aa5ce6a70de7ba95aef77a77b00a0af]
https://support.hackerone.com [29d3dd00029d29d00029d3dd29d29d5a74e95248e58a6162e37847a24849f7]
https://api.hackerone.com [29d3dd00029d29d00042d43d00041d5de67cc9954cc85372523050f20b5007]
https://mta-sts.forwarding.hackerone.com [29d29d00029d29d00042d43d00041d2aa5ce6a70de7ba95aef77a77b00a0af]
https://resources.hackerone.com [2ad2ad0002ad2ad0002ad2ad2ad2ad043bfbd87c13813505a1b60adf4f6ff5]
```

### ASN Fingerprint


```console
subfinder -d hackerone.com -silent | httpx -asn
__ __ __ _ __
/ /_ / /_/ /_____ | |/ /
/ __ \/ __/ __/ __ \| /
/ / / / /_/ /_/ /_/ / |
/_/ /_/\__/\__/ .___/_/|_|
/_/ v1.2.1

projectdiscovery.io

Use with caution. You are responsible for your actions.
Developers assume no liability and are not responsible for any misuse or damage.
https://mta-sts.managed.hackerone.com [AS54113, FASTLY, US, 185.199.108.0/24]
https://gslink.hackerone.com [AS16509, AMAZON-02, US, 13.33.168.0/22]
https://www.hackerone.com [AS13335, CLOUDFLARENET, US, 104.16.96.0/20]
https://mta-sts.forwarding.hackerone.com [AS54113, FASTLY, US, 185.199.108.0/24]
https://resources.hackerone.com [AS16509, AMAZON-02, US, 3.98.0.0/15]
https://support.hackerone.com [AS13335, CLOUDFLARENET, US, 104.16.48.0/20]
https://mta-sts.hackerone.com [AS54113, FASTLY, US, 185.199.111.0/24]
https://docs.hackerone.com [AS54113, FASTLY, US, 185.199.109.0/24]
https://api.hackerone.com [AS13335, CLOUDFLARENET, US, 104.16.96.0/20]
```


### Path Probe


Expand Down
1 change: 1 addition & 0 deletions common/hashes/hashes.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"encoding/base64"
"encoding/hex"
"fmt"

"github.com/mfonda/simhash"
"github.com/spaolacci/murmur3"
)
Expand Down
93 changes: 93 additions & 0 deletions common/hashes/jarmhash.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
package hashes

import (
"fmt"
"net"
"net/url"
"strconv"
"strings"
"time"

"github.com/RumbleDiscovery/jarm-go"
"github.com/projectdiscovery/httpx/common/regexhelper"
"golang.org/x/net/proxy"
)

const defaultPort int = 443

var DefualtBackoff = func(r, m int) time.Duration {
return time.Second
}

type target struct {
Host string
Port int
Retries int
Backoff func(r, m int) time.Duration
}

// fingerprint probes a single host/port
func fingerprint(t target, duration int) string {
timeout := time.Duration(duration) * time.Second
results := []string{}
for _, probe := range jarm.GetProbes(t.Host, t.Port) {
dialer := proxy.FromEnvironmentUsing(&net.Dialer{Timeout: timeout})
addr := net.JoinHostPort(t.Host, fmt.Sprintf("%d", t.Port))
c := net.Conn(nil)
n := 0
for c == nil && n <= t.Retries {
// Ignoring error since error message was already being dropped.
// Also, if theres an error, c == nil.
if c, _ = dialer.Dial("tcp", addr); c != nil || t.Retries == 0 {
break
}
bo := t.Backoff
if bo == nil {
bo = DefualtBackoff
}
time.Sleep(bo(n, t.Retries))
n++
}
if c == nil {
return ""
}
data := jarm.BuildProbe(probe)
_ = c.SetWriteDeadline(time.Now().Add(timeout))
_, err := c.Write(data)
if err != nil {
results = append(results, "")
c.Close()
continue
}
_ = c.SetReadDeadline(time.Now().Add(timeout))
buff := make([]byte, 1484)
_, _ = c.Read(buff)
c.Close()
ans, err := jarm.ParseServerHello(buff, probe)
if err != nil {
results = append(results, "")
continue
}
results = append(results, ans)
}
return jarm.RawHashToFuzzyHash(strings.Join(results, ","))
}
func Jarm(host string, duration int) string {
t := target{}
if u, err := url.Parse(host); err == nil {
if u.Scheme == "http" {
return ""
}
t.Host = u.Hostname()
port, _ := strconv.Atoi(u.Port())
t.Port = port
}
if t.Port == 0 {
t.Port = defaultPort
}
hash := fingerprint(t, duration)
if regexhelper.JarmHashRegex.MatchString(hash) {
return ""
}
return hash
}
45 changes: 32 additions & 13 deletions common/httpx/csp.go
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
package httpx

import (
"net/http"
"bytes"
"strings"

"github.com/PuerkitoBio/goquery"
"github.com/projectdiscovery/httpx/common/slice"
"github.com/projectdiscovery/stringsutil"
)

// CSPHeaders is an incomplete list of most common CSP headers
Expand All @@ -21,22 +23,28 @@ type CSPData struct {
}

// CSPGrab fills the CSPData
func (h *HTTPX) CSPGrab(r *http.Response) *CSPData {
func (h *HTTPX) CSPGrab(r *Response) *CSPData {
domains := make(map[string]struct{})
// extract from headers
for _, cspHeader := range CSPHeaders {
cspRaw := r.Header.Get(cspHeader)
if cspRaw != "" {
rules := strings.Split(cspRaw, ";")
for _, rule := range rules {
// rule is like aa bb domain1 domain2 domain3
tokens := strings.Split(rule, " ")
// we extracts only potential domains
for _, t := range tokens {
if isPotentialDomain(t) {
domains[t] = struct{}{}
if cspValues, ok := r.Headers[cspHeader]; ok {
for _, cspValue := range cspValues {
parsePotentialDomains(domains, cspValue)
}
}
}

// extract from body
if len(r.Data) > 0 {
doc, err := goquery.NewDocumentFromReader(bytes.NewReader(r.Data))
if err == nil {
doc.Find("meta").Each(func(i int, s *goquery.Selection) {
if _, ok := s.Attr("http-equiv"); ok {
if content, ok := s.Attr("content"); ok {
parsePotentialDomains(domains, content)
}
}
}
})
}
}

Expand All @@ -46,6 +54,17 @@ func (h *HTTPX) CSPGrab(r *http.Response) *CSPData {
return nil
}

func parsePotentialDomains(domains map[string]struct{}, data string) {
// rule is like aa bb domain1 domain2 domain3
tokens := stringsutil.SplitAny(data, " ", ";", ",")
// we extracts only potential domains
for _, t := range tokens {
if isPotentialDomain(t) {
domains[t] = struct{}{}
}
}
}

func isPotentialDomain(s string) bool {
return strings.Contains(s, ".") || strings.HasPrefix(s, "http")
}
Loading

0 comments on commit 7d28ce4

Please sign in to comment.