Skip to content

Commit

Permalink
Merge pull request #468 from projectdiscovery/issue-453-lines-count-m…
Browse files Browse the repository at this point in the history 
…atchers-and-filters

Adding support for lines/words matchers/filters
  • Loading branch information
@ehsandeep
ehsandeep authored Jan 4, 2022
2 parents f36aff9 + 20254fd commit 8d2860c
Show file tree
Hide file tree
Showing 3 changed files with 90 additions and 13 deletions.
31 changes: 19 additions & 12 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,18 +43,19 @@ httpx is a fast and multi-purpose HTTP toolkit allow to run multiple probers usi

### Supported probes:-

| Probes | Default check | Probes | Default check |
|--------------------|-----------------|--------------------|-----------------|
| URL | true | IP | true |
| Title | true | CNAME | true |
| Status Code | true | Raw HTTP | false |
| Content Length | true | HTTP2 | false |
| TLS Certificate | true | HTTP 1.1 Pipeline | false |
| CSP Header | true | Virtual host | false |
| Location Header | true | CDN | false |
| Web Server | true | Path | false |
| Web Socket | true | Ports | false |
| Response Time | true | Request method | false |
| Probes | Default check | Probes | Default check |
| --------------- | ------------- | ----------------- | ------------- |
| URL | true | IP | true |
| Title | true | CNAME | true |
| Status Code | true | Raw HTTP | false |
| Content Length | true | HTTP2 | false |
| Line Count | true | Word Count | true |
| TLS Certificate | true | HTTP 1.1 Pipeline | false |
| CSP Header | true | Virtual host | false |
| Location Header | true | CDN | false |
| Web Server | true | Paths | false |
| Web Socket | true | Ports | false |
| Response Time | true | Request Method | false |


# Installation Instructions
Expand Down Expand Up @@ -87,6 +88,8 @@ PROBES:
-sc, -status-code Display Status Code
-td, -tech-detect Display wappalyzer based technology detection
-cl, -content-length Display Content-Length
-lc, -line-count Display Response body line count
-wc, -word-count Display Response body word count
-server, -web-server Display Server header
-ct, -content-type Display Content-Type header
-rt, -response-time Display the response time
Expand All @@ -105,12 +108,16 @@ MATCHERS:
-ms, -match-string string Match response with given string
-mr, -match-regex string Match response with specific regex
-er, -extract-regex string Display response content with matched regex
-mlc, -match-line-count string Match Response body line count
-mwc, -match-word-count string Match Response body word count

FILTERS:
-fc, -filter-code string Filter response with given status code (-fc 403,401)
-fl, -filter-length string Filter response with given content length (-fl 23,33)
-fs, -filter-string string Filter response with specific string
-fe, -filter-regex string Filter response with specific regex
-flc, -filter-line-count string Filter Response body line count
-fwc, -filter-word-count string Filter Response body word count

RATE-LIMIT:
-t, -threads int Number of threads (default 50)
Expand Down
34 changes: 33 additions & 1 deletion runner/options.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,8 @@ type scanOptions struct {
ExcludeCDN bool
HostMaxErrors int
ProbeAllIPS bool
OutputLinesCount bool
OutputWordsCount bool
}

func (s *scanOptions) Clone() *scanOptions {
Expand Down Expand Up @@ -105,6 +107,8 @@ func (s *scanOptions) Clone() *scanOptions {
MaxResponseBodySizeToSave: s.MaxResponseBodySizeToSave,
MaxResponseBodySizeToRead: s.MaxResponseBodySizeToRead,
HostMaxErrors: s.HostMaxErrors,
OutputLinesCount: s.OutputLinesCount,
OutputWordsCount: s.OutputWordsCount,
}
}

Expand Down Expand Up @@ -199,6 +203,16 @@ type Options struct {
SkipDedupe bool
ProbeAllIPS bool
Resolvers goflags.NormalizedStringSlice
OutputLinesCount bool
OutputMatchLinesCount string
matchLinesCount []int
OutputFilterLinesCount string
filterLinesCount []int
OutputWordsCount bool
OutputMatchWordsCount string
matchWordsCount []int
OutputFilterWordsCount string
filterWordsCount []int
}

// ParseOptions parses the command line options for application
Expand All @@ -219,6 +233,8 @@ func ParseOptions() *Options {
flagSet.BoolVarP(&options.ContentLength, "content-length", "cl", false, "Display Content-Length"),
flagSet.BoolVarP(&options.OutputServerHeader, "web-server", "server", false, "Display Server header"),
flagSet.BoolVarP(&options.OutputContentType, "content-type", "ct", false, "Display Content-Type header"),
flagSet.BoolVarP(&options.OutputLinesCount, "line-count", "lc", false, "Display Response body line count"),
flagSet.BoolVarP(&options.OutputWordsCount, "word-count", "wc", false, "Display Response body word count"),
flagSet.BoolVarP(&options.OutputResponseTime, "response-time", "rt", false, "Display the response time"),
flagSet.BoolVar(&options.ExtractTitle, "title", false, "Display page title"),
flagSet.BoolVar(&options.Location, "location", false, "Display Location header"),
Expand All @@ -228,21 +244,25 @@ func ParseOptions() *Options {
flagSet.BoolVar(&options.OutputCName, "cname", false, "Display Host cname"),
flagSet.BoolVar(&options.OutputCDN, "cdn", false, "Display if CDN in use"),
flagSet.BoolVar(&options.Probe, "probe", false, "Display probe status"),
flagSet.StringVarP(&options.OutputExtractRegex, "extract-regex", "er", "", "Display response content with matched regex"),
)

createGroup(flagSet, "matchers", "Matchers",
flagSet.StringVarP(&options.OutputMatchStatusCode, "match-code", "mc", "", "Match response with given status code (-mc 200,302)"),
flagSet.StringVarP(&options.OutputMatchContentLength, "match-length", "ml", "", "Match response with given content length (-ml 100,102)"),
flagSet.StringVarP(&options.OutputMatchString, "match-string", "ms", "", "Match response with given string"),
flagSet.StringVarP(&options.OutputMatchRegex, "match-regex", "mr", "", "Match response with specific regex"),
flagSet.StringVarP(&options.OutputExtractRegex, "extract-regex", "er", "", "Display response content with matched regex"),
flagSet.StringVarP(&options.OutputMatchLinesCount, "match-line-count", "mlc", "", "Match Response body line count"),
flagSet.StringVarP(&options.OutputMatchWordsCount, "match-word-count", "mwc", "", "Match Response body word count"),
)

createGroup(flagSet, "filters", "Filters",
flagSet.StringVarP(&options.OutputFilterStatusCode, "filter-code", "fc", "", "Filter response with given status code (-fc 403,401)"),
flagSet.StringVarP(&options.OutputFilterContentLength, "filter-length", "fl", "", "Filter response with given content length (-fl 23,33)"),
flagSet.StringVarP(&options.OutputFilterString, "filter-string", "fs", "", "Filter response with specific string"),
flagSet.StringVarP(&options.OutputFilterRegex, "filter-regex", "fe", "", "Filter response with specific regex"),
flagSet.StringVarP(&options.OutputFilterLinesCount, "filter-line-count", "flc", "", "Filter Response body line count"),
flagSet.StringVarP(&options.OutputFilterWordsCount, "filter-word-count", "fwc", "", "Filter Response body word count"),
)

createGroup(flagSet, "rate-limit", "Rate-Limit",
Expand Down Expand Up @@ -373,6 +393,18 @@ func (options *Options) validateOptions() {
gologger.Fatal().Msgf("Invalid value for match regex option: %s\n", err)
}
}
if options.matchLinesCount, err = stringz.StringToSliceInt(options.OutputMatchLinesCount); err != nil {
gologger.Fatal().Msgf("Invalid value for match lines count option: %s\n", err)
}
if options.matchWordsCount, err = stringz.StringToSliceInt(options.OutputMatchWordsCount); err != nil {
gologger.Fatal().Msgf("Invalid value for match words count option: %s\n", err)
}
if options.filterLinesCount, err = stringz.StringToSliceInt(options.OutputFilterLinesCount); err != nil {
gologger.Fatal().Msgf("Invalid value for filter lines count option: %s\n", err)
}
if options.filterWordsCount, err = stringz.StringToSliceInt(options.OutputFilterWordsCount); err != nil {
gologger.Fatal().Msgf("Invalid value for filter words count option: %s\n", err)
}

var resolvers []string
for _, resolver := range options.Resolvers {
Expand Down
38 changes: 38 additions & 0 deletions runner/runner.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,8 @@ func New(options *Options) (*Runner, error) {
scanopts.ExcludeCDN = options.ExcludeCDN
scanopts.HostMaxErrors = options.HostMaxErrors
scanopts.ProbeAllIPS = options.ProbeAllIPS
scanopts.OutputLinesCount = options.OutputLinesCount
scanopts.OutputWordsCount = options.OutputWordsCount
runner.scanopts = scanopts

if options.ShowStatistics {
Expand Down Expand Up @@ -552,6 +554,12 @@ func (r *Runner) RunEnumeration() {
if len(r.options.filterContentLength) > 0 && slice.IntSliceContains(r.options.filterContentLength, resp.ContentLength) {
continue
}
if len(r.options.filterLinesCount) > 0 && slice.IntSliceContains(r.options.filterLinesCount, resp.Lines) {
continue
}
if len(r.options.filterWordsCount) > 0 && slice.IntSliceContains(r.options.filterWordsCount, resp.Words) {
continue
}
if r.options.filterRegex != nil && r.options.filterRegex.MatchString(resp.raw) {
continue
}
Expand All @@ -570,6 +578,12 @@ func (r *Runner) RunEnumeration() {
if r.options.OutputMatchString != "" && !strings.Contains(strings.ToLower(resp.raw), strings.ToLower(r.options.OutputMatchString)) {
continue
}
if len(r.options.matchLinesCount) > 0 && !slice.IntSliceContains(r.options.matchLinesCount, resp.Lines) {
continue
}
if len(r.options.matchWordsCount) > 0 && !slice.IntSliceContains(r.options.matchWordsCount, resp.Words) {
continue
}

row := resp.str
if r.options.JSONOutput {
Expand Down Expand Up @@ -1168,6 +1182,26 @@ retry:
builder.WriteRune(']')
}

if scanopts.OutputLinesCount {
builder.WriteString(" [")
if !scanopts.OutputWithNoColor {
builder.WriteString(aurora.Magenta(resp.Lines).String())
} else {
builder.WriteString(fmt.Sprint(resp.Lines))
}
builder.WriteRune(']')
}

if scanopts.OutputWordsCount {
builder.WriteString(" [")
if !scanopts.OutputWithNoColor {
builder.WriteString(aurora.Magenta(resp.Words).String())
} else {
builder.WriteString(fmt.Sprint(resp.Words))
}
builder.WriteRune(']')
}

// store responses or chain in directory
if scanopts.StoreResponse || scanopts.StoreChain {
domainFile := strings.ReplaceAll(urlutil.TrimScheme(URL.String()), ":", ".")
Expand Down Expand Up @@ -1271,6 +1305,8 @@ retry:
ResponseTime: resp.Duration.String(),
Technologies: technologies,
FinalURL: finalURL,
Lines: resp.Lines,
Words: resp.Words,
}
}

Expand Down Expand Up @@ -1322,6 +1358,8 @@ type Result struct {
Chain []httpx.ChainItem `json:"chain,omitempty" csv:"chain"`
FinalURL string `json:"final-url,omitempty" csv:"final-url"`
Failed bool `json:"failed" csv:"failed"`
Lines int `json:"lines" csv:"lines"`
Words int `json:"words" csv:"words"`
}

// JSON the result
Expand Down

0 comments on commit 8d2860c

Please sign in to comment.