Merge pull request #420 from projectdiscovery/issue-416-deny-infinite…

…-recursion

Adding max recursion level for self-dos on recursive load function

common/customlist/customlist.go

@@ -2,6 +2,8 @@ package customlist
 
 import "github.com/projectdiscovery/httpx/common/fileutil"
 
+const maxRecursion = 10
+
 // CustomList for fastdialer
 type CustomList []string
 
@@ -12,7 +14,7 @@ func (c *CustomList) String() string {
 
 // Set a new global header
 func (c *CustomList) Set(value string) error {
-	values := fileutil.LoadCidrsFromSliceOrFile(value, ",")
+	values := fileutil.LoadCidrsFromSliceOrFileWithMaxRecursion(value, ",", maxRecursion)
 	*c = append(*c, values...)
 	return nil
 }

common/fileutil/fileutil.go

@@ -13,21 +13,6 @@ import (
 	"github.com/projectdiscovery/httpx/common/stringz"
 )
 
-// FileExists checks if a file exists and is not a directory
-func FileExists(filename string) bool {
-	info, err := os.Stat(filename)
-	if os.IsNotExist(err) || err != nil || info == nil {
-		return false
-	}
-	return !info.IsDir()
-}
-
-// FolderExists checks if a folder exists
-func FolderExists(folderpath string) bool {
-	_, err := os.Stat(folderpath)
-	return !os.IsNotExist(err)
-}
-
 // HasStdin determines if the user has piped input
 func HasStdin() bool {
 	stat, err := os.Stdin.Stat()
@@ -76,20 +61,22 @@ func FileNameIsGlob(pattern string) bool {
 	return err == nil
 }
 
-func LoadCidrsFromSliceOrFile(option string, splitchar string) (networkList []string) {
+func LoadCidrsFromSliceOrFileWithMaxRecursion(option string, splitchar string, maxRecursion int) (networkList []string) {
+	if maxRecursion < 0 {
+		return
+	}
 	items := stringz.SplitByCharAndTrimSpace(option, splitchar)
 	for _, item := range items {
-		// ip
 		if net.ParseIP(item) != nil {
 			networkList = append(networkList, item)
 		} else if _, _, err := net.ParseCIDR(item); err == nil {
 			networkList = append(networkList, item)
 		} else if fileutil.FileExists(item) {
 			if filedata, err := ioutil.ReadFile(item); err == nil && len(filedata) > 0 {
-				networkList = append(networkList, LoadCidrsFromSliceOrFile(string(filedata), "\n")...)
+				networkList = append(networkList, LoadCidrsFromSliceOrFileWithMaxRecursion(string(filedata), "\n", maxRecursion-1)...)
 			}
 		}
 	}
 
-	return networkList
+	return
 }

go.mod

@@ -22,6 +22,7 @@ require (
 	github.com/projectdiscovery/fdmax v0.0.3
 	github.com/projectdiscovery/fileutil v0.0.0-20210926044607-04f32490aa21
 	github.com/projectdiscovery/goconfig v0.0.0-20210804090219-f893ccd0c69c
+	github.com/projectdiscovery/goflags v0.0.7
 	github.com/projectdiscovery/gologger v1.1.4
 	github.com/projectdiscovery/hmap v0.0.2-0.20210630092648-6c0a1b362caa
 	github.com/projectdiscovery/httputil v0.0.0-20210816170244-86fd46bc09f5
@@ -62,7 +63,6 @@ require (
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.1 // indirect
 	github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e // indirect
-	github.com/projectdiscovery/goflags v0.0.7 // indirect
 	github.com/projectdiscovery/networkpolicy v0.0.1 // indirect
 	github.com/projectdiscovery/reflectutil v0.0.0-20210804085554-4d90952bf92f // indirect
 	github.com/syndtr/goleveldb v1.0.0 // indirect

go.sum

@@ -94,9 +94,11 @@ github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8
 github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw=
 github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
 github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8=
@@ -187,6 +189,7 @@ github.com/projectdiscovery/retryablehttp-go v1.0.2-0.20210526144436-e15804ddc7d
 github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea h1:S+DC2tmKG93Om42cnTqrBfIv699pwSIhafqZvip+RIA=
 github.com/projectdiscovery/sliceutil v0.0.0-20210804143453-61f3e7fd43ea/go.mod h1:QHXvznfPfA5f0AZUIBkbLapoUJJlsIDgUlkKva6dOr4=
 github.com/projectdiscovery/stringsutil v0.0.0-20210524051937-51dabe3b72c0/go.mod h1:TVSdZC0rRQeMIbsNSiGPhbmhyRtxqqtAGA9JiiNp2r4=
+github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I=
 github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 h1:xbL1/7h0k6HE3RzPdYk9W/8pUxESrGWewTaZdIB5Pes=
 github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9/go.mod h1:oTRc18WBv9t6BpaN9XBY+QmG28PUpsyDzRht56Qf49I=
 github.com/projectdiscovery/urlutil v0.0.0-20210805190935-3d83726391c1 h1:9dYmONRtwy+xP8UAGHxEQ0cxO3umc9qiFmnYsoDUps4=
@@ -314,6 +317,7 @@ google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+Rur
 google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU=