Skip to content

Commit

Permalink
update headers
Browse files Browse the repository at this point in the history
  • Loading branch information
@davemolk
davemolk committed Jan 3, 2023
1 parent 9d3a359 commit 090463a
Show file tree
Hide file tree
Showing 3 changed files with 99 additions and 18 deletions.
5 changes: 5 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,8 @@ Usage of searcher:
Print results as JSON.
-q string
Base search query.
-os string
operating system (used in user agent and header creation)
-t bool
Check Stdin for additional search terms.
-to int
Expand All @@ -79,3 +81,6 @@ Usage of searcher:
-w bool
Write results to file.
```

## Note
Each request gets a randomly assigned user agent corresponding to your os as well as appropriate headers (50/50 chance of chrome or firefox). Go unfortunately doesn't preserve header order, so if that's important to you and what you're up to, you'll have to look elsewhere.
2 changes: 2 additions & 0 deletions cmd/searcher/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ type config struct {
concurrency int
exact bool
json bool
os string
terms bool
timeout int
verbose bool
Expand All @@ -32,6 +33,7 @@ func main() {
flag.IntVar(&config.concurrency, "c", 10, "max number of goroutines to use at any given time")
flag.BoolVar(&config.exact, "e", false, "search for exact match")
flag.BoolVar(&config.json, "j", false, "print results as json")
flag.StringVar(&config.os, "os", "w", "operating system (w or m)")
flag.BoolVar(&config.terms, "t", false, "check stdin for additional search terms")
flag.IntVar(&config.timeout, "to", 5000, "timeout (in ms, default 5000)")
flag.BoolVar(&config.verbose, "v", false, "verbose output")
Expand Down
110 changes: 92 additions & 18 deletions cmd/searcher/requests.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,7 @@ func (s *searcher) makeRequest(url string, timeout int) (*bytes.Buffer, error) {
return nil, fmt.Errorf("unable to create request for %s: %v", url, err)
}

uAgent := s.randomUA()
req.Header.Set("User-Agent", uAgent)
req = s.headers(req)

resp, err := http.DefaultClient.Do(req)
if err != nil {
Expand All @@ -49,25 +48,100 @@ func (s *searcher) makeRequest(url string, timeout int) (*bytes.Buffer, error) {
return buf, nil
}

// randomUA picks a random user agent from the slice and returns it.
func (s *searcher) randomUA() string {
userAgents := s.getUA()
func (s *searcher) headers(r *http.Request) *http.Request {
if rand.Intn(2) == 1 {
return s.ff(r)
}
return s.chrome(r)
}

func (s *searcher) ff(r *http.Request) *http.Request {
uAgent := s.ffUA()
r.Header.Set("Host", r.URL.Host)
r.Header.Set("User-Agent", uAgent)
r.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8")
r.Header.Set("Accept-Language", "en-US,en;q=0.5")
r.Header.Set("Accept-Encoding", "gzip, deflate, br")
r.Header.Set("DNT", "1")
r.Header.Set("Connection", "keep-alive")
r.Header.Set("Upgrade-Insecure-Requests", "1")
r.Header.Set("Sec-Fetch-Dest", "document")
r.Header.Set("Sec-Fetch-Mode", "navigate")
r.Header.Set("Sec-Fetch-Site", "none")
r.Header.Set("Sec-Fetch-User", "?1")
r.Header.Set("Sec-GCP", "1")
return r
}

func (s *searcher) chrome(r *http.Request) *http.Request {
uAgent := s.chromeUA()
r.Header.Set("Host", r.URL.Host)
r.Header.Set("Connection", "keep-alive")
r.Header.Set("Cache-Control", "max-age=0")
r.Header.Set("sec-ch-ua", `" Not A;Brand";v="99", "Chromium";v="99", "Google Chrome";v="99"`)
r.Header.Set("sec-ch-ua-mobile", "?0")
switch s.config.os {
case "m":
r.Header.Set("sec-ch-ua-platform", "Macintosh")
default:
r.Header.Set("sec-ch-ua-platform", "Windows")
}
r.Header.Set("Upgrade-Insecure-Requests", "1")
r.Header.Set("User-Agent", uAgent)
r.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8")
r.Header.Set("Sec-Fetch-Site", "none")
r.Header.Set("Sec-Fetch-Mode", "navigate")
r.Header.Set("Sec-Fetch-User", "?1")
r.Header.Set("Sec-Fetch-Dest", "document")
r.Header.Set("Accept-Encoding", "gzip, deflate, br")
r.Header.Set("Accept-Language", "en-US,en;q=0.5")
return r
}

func (s *searcher) ffUA() string {
var userAgents []string
switch s.config.os {
case "m":
userAgents = []string{
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:107.0) Gecko/20100101 Firefox/107.0",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:106.0) Gecko/20100101 Firefox/106.0",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:104.0) Gecko/20100101 Firefox/104.0",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:102.0) Gecko/20100101 Firefox/102.0",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:101.0) Gecko/20100101 Firefox/101.0",
}
default:
userAgents = []string{
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:99.0) Gecko/20100101 Firefox/99.0",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:108.0) Gecko/20100101 Firefox/108.0",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:102.0) Gecko/20100101 Firefox/102.0",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:106.0) Gecko/20100101 Firefox/106.0",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:104.0) Gecko/20100101 Firefox/104.0",
}
}
random := rand.Intn(len(userAgents))
return userAgents[random]
}

// getUA returns a string slice of ten user agents.
func (s *searcher) getUA() []string {
return []string{
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4692.56 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4889.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/603.3.8 (KHTML, like Gecko)",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/601.7.7 (KHTML, like Gecko) Version/9.1.2 Safari/601.7.7",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.54 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:99.0) Gecko/20100101 Firefox/99.0",
"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.84 Safari/537.36",
func (s *searcher) chromeUA() string {
var userAgents []string
switch s.config.os {
case "m":
userAgents = []string{
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4692.56 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4889.0 Safari/537.36",
}
default:
userAgents = []string{
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.54 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.84 Safari/537.36",
}
}
random := rand.Intn(len(userAgents))
return userAgents[random]
}

0 comments on commit 090463a

Please sign in to comment.