diff --git a/README.md b/README.md index f1e8e57..792713d 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,8 @@ Usage of searcher: Print results as JSON. -q string Base search query. + -os string + operating system (used in user agent and header creation) -t bool Check Stdin for additional search terms. -to int @@ -79,3 +81,6 @@ Usage of searcher: -w bool Write results to file. ``` + +## Note +Each request gets a randomly assigned user agent corresponding to your os as well as appropriate headers (50/50 chance of chrome or firefox). Go unfortunately doesn't preserve header order, so if that's important to you and what you're up to, you'll have to look elsewhere. \ No newline at end of file diff --git a/cmd/searcher/main.go b/cmd/searcher/main.go index f0d18c9..a99cbc2 100644 --- a/cmd/searcher/main.go +++ b/cmd/searcher/main.go @@ -12,6 +12,7 @@ type config struct { concurrency int exact bool json bool + os string terms bool timeout int verbose bool @@ -32,6 +33,7 @@ func main() { flag.IntVar(&config.concurrency, "c", 10, "max number of goroutines to use at any given time") flag.BoolVar(&config.exact, "e", false, "search for exact match") flag.BoolVar(&config.json, "j", false, "print results as json") + flag.StringVar(&config.os, "os", "w", "operating system (w or m)") flag.BoolVar(&config.terms, "t", false, "check stdin for additional search terms") flag.IntVar(&config.timeout, "to", 5000, "timeout (in ms, default 5000)") flag.BoolVar(&config.verbose, "v", false, "verbose output") diff --git a/cmd/searcher/requests.go b/cmd/searcher/requests.go index d8f4f71..9ad7447 100644 --- a/cmd/searcher/requests.go +++ b/cmd/searcher/requests.go @@ -26,8 +26,7 @@ func (s *searcher) makeRequest(url string, timeout int) (*bytes.Buffer, error) { return nil, fmt.Errorf("unable to create request for %s: %v", url, err) } - uAgent := s.randomUA() - req.Header.Set("User-Agent", uAgent) + req = s.headers(req) resp, err := http.DefaultClient.Do(req) if err != nil { @@ -49,25 +48,100 @@ func (s *searcher) makeRequest(url string, timeout int) (*bytes.Buffer, error) { return buf, nil } -// randomUA picks a random user agent from the slice and returns it. -func (s *searcher) randomUA() string { - userAgents := s.getUA() +func (s *searcher) headers(r *http.Request) *http.Request { + if rand.Intn(2) == 1 { + return s.ff(r) + } + return s.chrome(r) +} + +func (s *searcher) ff(r *http.Request) *http.Request { + uAgent := s.ffUA() + r.Header.Set("Host", r.URL.Host) + r.Header.Set("User-Agent", uAgent) + r.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8") + r.Header.Set("Accept-Language", "en-US,en;q=0.5") + r.Header.Set("Accept-Encoding", "gzip, deflate, br") + r.Header.Set("DNT", "1") + r.Header.Set("Connection", "keep-alive") + r.Header.Set("Upgrade-Insecure-Requests", "1") + r.Header.Set("Sec-Fetch-Dest", "document") + r.Header.Set("Sec-Fetch-Mode", "navigate") + r.Header.Set("Sec-Fetch-Site", "none") + r.Header.Set("Sec-Fetch-User", "?1") + r.Header.Set("Sec-GCP", "1") + return r +} + +func (s *searcher) chrome(r *http.Request) *http.Request { + uAgent := s.chromeUA() + r.Header.Set("Host", r.URL.Host) + r.Header.Set("Connection", "keep-alive") + r.Header.Set("Cache-Control", "max-age=0") + r.Header.Set("sec-ch-ua", `" Not A;Brand";v="99", "Chromium";v="99", "Google Chrome";v="99"`) + r.Header.Set("sec-ch-ua-mobile", "?0") + switch s.config.os { + case "m": + r.Header.Set("sec-ch-ua-platform", "Macintosh") + default: + r.Header.Set("sec-ch-ua-platform", "Windows") + } + r.Header.Set("Upgrade-Insecure-Requests", "1") + r.Header.Set("User-Agent", uAgent) + r.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8") + r.Header.Set("Sec-Fetch-Site", "none") + r.Header.Set("Sec-Fetch-Mode", "navigate") + r.Header.Set("Sec-Fetch-User", "?1") + r.Header.Set("Sec-Fetch-Dest", "document") + r.Header.Set("Accept-Encoding", "gzip, deflate, br") + r.Header.Set("Accept-Language", "en-US,en;q=0.5") + return r +} + +func (s *searcher) ffUA() string { + var userAgents []string + switch s.config.os { + case "m": + userAgents = []string{ + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:107.0) Gecko/20100101 Firefox/107.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:106.0) Gecko/20100101 Firefox/106.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:104.0) Gecko/20100101 Firefox/104.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:102.0) Gecko/20100101 Firefox/102.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:101.0) Gecko/20100101 Firefox/101.0", + } + default: + userAgents = []string{ + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:99.0) Gecko/20100101 Firefox/99.0", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:108.0) Gecko/20100101 Firefox/108.0", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:102.0) Gecko/20100101 Firefox/102.0", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:106.0) Gecko/20100101 Firefox/106.0", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:104.0) Gecko/20100101 Firefox/104.0", + } + } random := rand.Intn(len(userAgents)) return userAgents[random] } -// getUA returns a string slice of ten user agents. -func (s *searcher) getUA() []string { - return []string{ - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4692.56 Safari/537.36", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4889.0 Safari/537.36", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/603.3.8 (KHTML, like Gecko)", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/601.7.7 (KHTML, like Gecko) Version/9.1.2 Safari/601.7.7", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.54 Safari/537.36", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:99.0) Gecko/20100101 Firefox/99.0", - "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36", - "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.84 Safari/537.36", +func (s *searcher) chromeUA() string { + var userAgents []string + switch s.config.os { + case "m": + userAgents = []string{ + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4692.56 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4889.0 Safari/537.36", + } + default: + userAgents = []string{ + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.54 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.84 Safari/537.36", + } } + random := rand.Intn(len(userAgents)) + return userAgents[random] }