Introducing CanHaveTitleTag Function for MIME Type Validation (#1608)

* chore(deps): bump golang.org/x/crypto from 0.14.0 to 0.17.0 (#1493)

Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.14.0 to 0.17.0.
- [Commits](golang/crypto@v0.14.0...v0.17.0)

---
updated-dependencies:
- dependency-name: golang.org/x/crypto
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

* chore(deps): bump github.com/cloudflare/circl from 1.3.3 to 1.3.7 (#1526)

Bumps [github.com/cloudflare/circl](https://github.com/cloudflare/circl) from 1.3.3 to 1.3.7.
- [Release notes](https://github.com/cloudflare/circl/releases)
- [Commits](cloudflare/circl@v1.3.3...v1.3.7)

---
updated-dependencies:
- dependency-name: github.com/cloudflare/circl
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

* chore(deps): bump github.com/quic-go/quic-go from 0.37.4 to 0.37.7 (#1531)

Bumps [github.com/quic-go/quic-go](https://github.com/quic-go/quic-go) from 0.37.4 to 0.37.7.
- [Release notes](https://github.com/quic-go/quic-go/releases)
- [Changelog](https://github.com/quic-go/quic-go/blob/master/Changelog.md)
- [Commits](quic-go/quic-go@v0.37.4...v0.37.7)

---
updated-dependencies:
- dependency-name: github.com/quic-go/quic-go
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

* Update Title Extraction

* Make Requsted Changes

* Import slices

* optional asn

---------

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com>
Co-authored-by: Sandeep Singh <sandeep@projectdiscovery.io>
Co-authored-by: mzack <marco.rivoli.nvh@gmail.com>

common/httpx/title.go

@@ -9,12 +9,22 @@ import (
 
 	stringsutil "github.com/projectdiscovery/utils/strings"
 	"golang.org/x/net/html"
+	"slices"
 )
 
 var (
-	cutset        = "\n\t\v\f\r"
-	reTitle       = regexp.MustCompile(`(?im)<\s*title.*>(.*?)<\s*/\s*title>`)
-	reContentType = regexp.MustCompile(`(?im)\s*charset="(.*?)"|charset=(.*?)"\s*`)
+	cutset                  = "\n\t\v\f\r"
+	reTitle                 = regexp.MustCompile(`(?im)<\s*title.*>(.*?)<\s*/\s*title>`)
+	reContentType           = regexp.MustCompile(`(?im)\s*charset="(.*?)"|charset=(.*?)"\s*`)
+	supportedTitleMimeTypes = []string{
+		"text/html",
+		"application/xhtml+xml",
+		"application/xml",
+		"application/rss+xml",
+		"application/atom+xml",
+		"application/xhtml+xml",
+		"application/vnd.wap.xhtml+xml",
+	}
 )
 
 // ExtractTitle from a response
@@ -40,6 +50,10 @@ func ExtractTitle(r *Response) (title string) {
 	return title
 }
 
+func CanHaveTitleTag(mimeType string) bool {  
+    return slices.Contains(supportedTitleMimeTypes, mimeType)  
+}  
+
 func getTitleWithDom(r *Response) (*html.Node, error) {
 	var title *html.Node
 	var crawler func(*html.Node)

runner/runner.go

@@ -1590,8 +1590,12 @@ retry:
 		builder.WriteRune(']')
 	}
 
-	title := httpx.ExtractTitle(resp)
-	if scanopts.OutputTitle {
+	var title string
+	if httpx.CanHaveTitleTag(resp.GetHeaderPart("Content-Type", ";")) {
+		title = httpx.ExtractTitle(resp)
+	}
+
+	if scanopts.OutputTitle && title != "" {
 		builder.WriteString(" [")
 		if !scanopts.OutputWithNoColor {
 			builder.WriteString(aurora.Cyan(title).String())

runner/runner_test.go

@@ -89,6 +89,10 @@ func TestRunner_cidr_targets(t *testing.T) {
 }
 
 func TestRunner_asn_targets(t *testing.T) {
+	if os.Getenv("PDCP_API_KEY") == "" {
+		return
+	}
+
 	options := &Options{}
 	r, err := New(options)
 	require.Nil(t, err, "could not create httpx runner")
@@ -131,10 +135,12 @@ func TestRunner_countTargetFromRawTarget(t *testing.T) {
 	got = r.countTargetFromRawTarget(input)
 	require.Equal(t, expected, got, "got wrong output")
 
-	input = "AS14421"
-	expected = 256
-	got = r.countTargetFromRawTarget(input)
-	require.Equal(t, expected, got, "got wrong output")
+	if os.Getenv("PDCP_API_KEY") != "" {
+		input = "AS14421"
+		expected = 256
+		got = r.countTargetFromRawTarget(input)
+		require.Equal(t, expected, got, "got wrong output")
+	}
 
 	input = "173.0.84.0/24"
 	expected = 256