Fix: improvements to allow grouped scans - helpful for reporting

go.sum

@@ -62,11 +62,14 @@ github.com/adedayo/tcpscan v0.5.5/go.mod h1:6v2onzEsetkP35RqQnKhB5B5MLVV1w+eepgC
 github.com/adedayo/tcpscan v0.5.6 h1:EzNBxn2nIJ5WzuAFb5tmxUGX83Yc0Xg6YN1hn3bLMao=
 github.com/adedayo/tcpscan v0.5.6/go.mod h1:6v2onzEsetkP35RqQnKhB5B5MLVV1w+eepgCkXWQzwc=
 github.com/adedayo/tlsaudit v0.2.2/go.mod h1:qlilsuzRz/YHkPXQvhnKxLr0ROWQTEvLEihXq7nHAdk=
+github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6 h1:G1bPvciwNyF7IUmKXNt9Ak3m6u9DE1rF+RmtIkBpVdA=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/carlescere/scheduler v0.0.0-20170109141437-ee74d2f83d82 h1:9bAydALqAjBfPHd/eAiJBHnMZUYov8m2PkXVr+YGQeI=
 github.com/carlescere/scheduler v0.0.0-20170109141437-ee74d2f83d82/go.mod h1:tyA14J0sA3Hph4dt+AfCjPrYR13+vVodshQSM7km9qw=
+github.com/coreos/etcd v3.3.10+incompatible h1:jFneRYjIvLMLhDLCzuTuU4rSJUjRplcJQ7pD7MnhC04=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
+github.com/coreos/go-semver v0.2.0 h1:3Jm3tLmsgAYcjC+4Up7hJrFBPr+n7rAqYeSw/SZazuY=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dgraph-io/badger v1.5.4 h1:gVTrpUTbbr/T24uvoCaqY2KSHfNLVGm0w+hbee2HMeg=
@@ -135,8 +138,10 @@ github.com/spf13/viper v1.3.1/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DM
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
+github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8 h1:3SVOIvH7Ae1KRYyQWRjXWJEA9sS/c/pjvH++55Gr648=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
+github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77 h1:ESFSdwYZvkeru3RtdrYueztKhOBCSAAzS4Gf+k0tEow=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/ratelimit v0.0.0-20180316092928-c15da0234277 h1:d9qaMM+ODpCq+9We41//fu/sHsTnXcrqd1en3x+GKy4=
@@ -167,6 +172,7 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/urfave/cli.v2 v2.0.0-20180128182452-d3ae77c26ac8 h1:Ggy3mWN4l3PUFPfSG0YB3n5fVYggzysUmiUQ89SnX6Y=
 gopkg.in/urfave/cli.v2 v2.0.0-20180128182452-d3ae77c26ac8/go.mod h1:cKXr3E0k4aosgycml1b5z33BVV6hai1Kh7uDgFOkbcs=

pkg/model/models.go

@@ -600,20 +600,43 @@ func GetCipherConfig(cipher uint16) (config CipherConfig, err error) {
 }
 
 //ScanRequest is a model to describe a given TLS Audit scan
-type ScanRequest struct {
-	CIDRs  []string
+// type ScanRequest struct {
+// 	CIDRs  []string
+// 	Config ScanConfig
+// 	//Next two fields will be automatically set once scan starts
+// 	Day    string //Date the scan was run in the format yyyy-mm-dd
+// 	ScanID string //Non-empty ScanID means this is a ScanRequest to resume an existing, possibly incomplete, scan
+// }
+
+//ScanGroup is a grouping of CIDR ranges to be scanned with descriptions, useful for reporting
+type ScanGroup struct {
+	Description string   `yaml:"description"` //Freeform text used in reporting
+	CIDRRanges  []string `yaml:"cidrRanges"`
+}
+
+//AdvancedScanRequest is a model to describe a given TLS Audit scan
+type AdvancedScanRequest struct {
 	Config ScanConfig
-	Day    string //Date the scan was run in the format yyyy-mm-dd
-	ScanID string //Non-empty ScanID means this is a ScanRequest to resume an existing, possibly incomplete, scan
+	//Next two fields will be automatically set once scan starts
+	Day        string //Date the scan was run in the format yyyy-mm-dd
+	ScanID     string //Non-empty ScanID means this is a ScanRequest to resume an existing, possibly incomplete, scan
+	ScanGroups []ScanGroup
+}
+
+//GroupedHost exploded hosts from an associated ScanGroup
+type GroupedHost struct {
+	ScanGroup ScanGroup
+	Hosts     []string
 }
 
 //PersistedScanRequest persisted version of ScanRequest
 type PersistedScanRequest struct {
-	Request   ScanRequest
-	Hosts     []string
-	ScanStart time.Time
-	ScanEnd   time.Time
-	Progress  int
+	Request      AdvancedScanRequest
+	GroupedHosts []GroupedHost
+	ScanStart    time.Time
+	ScanEnd      time.Time
+	Progress     int
+	HostCount    int
 }
 
 // //ServerResultSummary is a mini report of scan result
@@ -626,7 +649,7 @@ type PersistedScanRequest struct {
 
 //ScanResultSummary is the summary of a scan result session
 type ScanResultSummary struct {
-	Request          ScanRequest
+	Request          AdvancedScanRequest
 	ScanStart        time.Time
 	ScanEnd          time.Time
 	Progress         int
@@ -1443,12 +1466,12 @@ func scoreCipher(cipher, protocol uint16, scan ScanResult) (score string) {
 
 //TLSAuditConfig is the configuration of the nmap runner
 type TLSAuditConfig struct {
-	DailySchedules   []string `yaml:"dailySchedules"` // in the format 13:45, 01:20 etc
-	ServicePort      int      `yaml:"servicePort"`
-	IsProduction     bool     `yaml:"isProduction"`
-	PacketsPerSecond int      `yaml:"packetsPerSecond"`
-	Timeout          int      `yaml:"timeout"`
-	CIDRRanges       []string `yaml:"cidrRanges"`
+	DailySchedules   []string    `yaml:"dailySchedules"` // in the format 13:45, 01:20 etc
+	ServicePort      int         `yaml:"servicePort"`
+	IsProduction     bool        `yaml:"isProduction"`
+	PacketsPerSecond int         `yaml:"packetsPerSecond"`
+	Timeout          int         `yaml:"timeout"`
+	ScanGroups       []ScanGroup `yaml:"scanGroups"`
 }
 
 //TLSAuditSnapshot a snapshot representing the results of a given scan session

pkg/persistence.go

@@ -56,7 +56,7 @@ func GetScanData(date, scanID string) []tlsmodel.HumanScanResult {
 }
 
 //ListScans returns the ScanID list of  persisted scans
-func ListScans(rewindDays int, completed bool) (result []tlsmodel.ScanRequest) {
+func ListScans(rewindDays int, completed bool) (result []tlsmodel.AdvancedScanRequest) {
 	if rewindDays < 0 {
 		log.Print("The number of days in the past must be non-negative.")
 		return
@@ -91,7 +91,7 @@ func ListScans(rewindDays int, completed bool) (result []tlsmodel.ScanRequest) {
 		for _, sID := range dirs {
 			scanID := sID.Name()
 			//LoadScanRequest retrieves persisted scan request from folder following a layout pattern
-			if psr, err := LoadScanRequest(d, scanID); err == nil && (len(psr.Hosts) == psr.Progress) == completed {
+			if psr, err := LoadScanRequest(d, scanID); err == nil && (psr.HostCount == psr.Progress) == completed {
 				result = append(result, psr.Request)
 			}
 		}
@@ -140,7 +140,11 @@ func streamExistingResult(psr tlsmodel.PersistedScanRequest,
 	defer db.Close()
 
 	hostResults := make(map[string][]tlsmodel.ScanResult)
-	total := len(psr.Hosts)
+	total := 0
+	for _, gs := range psr.GroupedHosts {
+		total += len(gs.Hosts)
+	}
+
 	position := 0
 
 	db.View(func(txn *badger.Txn) error {
@@ -317,7 +321,7 @@ func LoadScanRequest(dir, scanID string) (psr tlsmodel.PersistedScanRequest, e e
 		return psr, outErr
 	}
 	psr, e = tlsmodel.UnmasharlPersistedScanRequest(data)
-	if e == nil && len(psr.Hosts) == psr.Progress {
+	if e == nil && psr.HostCount == psr.Progress {
 		lock.Lock()
 		psrCache[fmt.Sprintf("%s:%s", dir, scanID)] = psr
 		lock.Unlock()

pkg/scan-service.go

@@ -1,7 +1,6 @@
 package tlsaudit
 
 import (
-	"bufio"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
@@ -59,7 +58,7 @@ func init() {
 
 //AddTLSAuditRoutes adds TLSAudit service's routes to an existing router setup
 func AddTLSAuditRoutes(r *mux.Router) {
-	r.HandleFunc("/scan", RealtimeScan).Methods("GET")
+	r.HandleFunc("/scan", RealtimeAdvancedScan).Methods("GET")
 	r.HandleFunc("/listtlsscan/{rewind}/{completed}", getTLSAuditScanRequests).Methods("GET")
 	r.HandleFunc("/getscandata/{date}/{scanID}", getTLSAuditScanData).Methods("GET")
 	r.HandleFunc("/getprotocols/{date}/{scanID}", getTLSProtocols).Methods("GET")
@@ -188,55 +187,62 @@ func ipResolver(ip string) string {
 	}
 	return ""
 }
-func getIPsFromConfig() []string {
+func getIPsFromConfig() []tlsmodel.GroupedHost {
 	config, err := loadTLSConfig(TLSAuditConfigPath)
 	if err != nil {
-		return []string{}
+		return []tlsmodel.GroupedHost{}
 	}
 	ips := getIPsToScan(config)
 	return ips
 }
 
-func getIPsToScan(config tlsmodel.TLSAuditConfig) []string {
-	data := make(map[string]string)
-	ips := []string{}
-	for _, c := range config.CIDRRanges {
-		ports := ""
-		if strings.Contains(c, ":") {
-			cc, p, err := extractPorts(c)
-			if err != nil {
-				continue
+func getIPsToScan(config tlsmodel.TLSAuditConfig) []tlsmodel.GroupedHost {
+
+	groupedHosts := []tlsmodel.GroupedHost{}
+	for _, sg := range config.ScanGroups {
+		data := make(map[string]string)
+		ips := []string{}
+		for _, c := range sg.CIDRRanges {
+			ports := ""
+			if strings.Contains(c, ":") {
+				cc, p, err := extractPorts(c)
+				if err != nil {
+					continue
+				}
+				c = cc
+				ports = p
 			}
-			c = cc
-			ports = p
-		}
-		for _, ip := range cidr.Expand(c) {
-			ip = fmt.Sprintf("%s/32", ip)
-			if ps, present := data[ip]; present {
-				if ps == "" {
+			for _, ip := range cidr.Expand(c) {
+				ip = fmt.Sprintf("%s/32", ip)
+				if ps, present := data[ip]; present {
+					if ps == "" {
+						data[ip] = ports
+					} else if ports != "" {
+						data[ip] = fmt.Sprintf("%s,%s", ps, ports)
+					}
+				} else {
 					data[ip] = ports
-				} else if ports != "" {
-					data[ip] = fmt.Sprintf("%s,%s", ps, ports)
 				}
-			} else {
-				data[ip] = ports
 			}
 		}
-	}
-	for ip, ports := range data {
-		x := ip
-
-		if ports != "" {
-			z := strings.Split(ip, "/")
-			if len(z) != 2 {
-				continue
+		for ip, ports := range data {
+			x := ip
+			if ports != "" {
+				z := strings.Split(ip, "/")
+				if len(z) != 2 {
+					continue
+				}
+				x = fmt.Sprintf("%s:%s/%s", z[0], ports, z[1])
+				println(x)
 			}
-			x = fmt.Sprintf("%s:%s/%s", z[0], ports, z[1])
-			println(x)
+			ips = append(ips, x)
 		}
-		ips = append(ips, x)
+		groupedHosts = append(groupedHosts, tlsmodel.GroupedHost{
+			ScanGroup: sg,
+			Hosts:     ips,
+		})
 	}
-	return ips
+	return groupedHosts
 }
 
 func extractPorts(cidrX string) (string, string, error) {
@@ -256,7 +262,7 @@ func extractPorts(cidrX string) (string, string, error) {
 }
 
 //ScheduleTLSAudit runs TLSAudit scan
-func ScheduleTLSAudit(ipSource func() []string, resolver func(string) string) {
+func ScheduleTLSAudit(ipSource func() []tlsmodel.GroupedHost, resolver func(string) string) {
 
 	//a restart schould clear the lock file
 	if _, err := os.Stat(runFlag2); !os.IsNotExist(err) { // there is a runlock
@@ -286,7 +292,7 @@ func ScheduleTLSAudit(ipSource func() []string, resolver func(string) string) {
 }
 
 //RunTLSScan accepts generator of IP addresses to scan and a function to map the IPs to hostnames (if any) - function to allow the hostname resolution happen in parallel if necessary
-func runTLSScan(ipSource func() []string, ipToHostnameResolver func(string) string) {
+func runTLSScan(ipSource func() []tlsmodel.GroupedHost, ipToHostnameResolver func(string) string) {
 	//create a directory, if not exist, for tlsaudit to keep temporary file
 	path := filepath.Join("data", "tlsaudit", "scan")
 	if _, err := os.Stat(path); os.IsNotExist(err) {
@@ -302,8 +308,6 @@ func runTLSScan(ipSource func() []string, ipToHostnameResolver func(string) stri
 	}
 	psr := tlsmodel.PersistedScanRequest{}
 
-	hosts := []string{}
-	// ipHostNameMapping := ipToHostnameResolver
 	if _, err := os.Stat(workList); !os.IsNotExist(err) { // there is a worklist (due to a previous crash!)
 		//load the list of IPs from there
 		println("Resuming due to a worklist")
@@ -314,10 +318,10 @@ func runTLSScan(ipSource func() []string, ipToHostnameResolver func(string) stri
 		}
 		defer file.Close()
 
-		scanner := bufio.NewScanner(file)
-		for scanner.Scan() {
-			hosts = append(hosts, scanner.Text())
-		}
+		// scanner := bufio.NewScanner(file)
+		// for scanner.Scan() {
+		// 	hosts = append(hosts, scanner.Text())
+		// }
 
 		day, err := ioutil.ReadFile(runFlag)
 		if err != nil {
@@ -343,20 +347,24 @@ func runTLSScan(ipSource func() []string, ipToHostnameResolver func(string) stri
 		}
 		fmt.Printf("Will be scanning with PSR %#v", psr)
 	} else { // starting a fresh scan
-		hosts = ipSource()
-		//shuffle hosts randomly
-		rand.Shuffle(len(hosts), func(i, j int) {
-			hosts[i], hosts[j] = hosts[j], hosts[i]
-		})
+		shuffledHosts := []string{}
+		for _, gh := range ipSource() {
+			//shuffle hosts randomly
+			rand.Shuffle(len(gh.Hosts), func(i, j int) {
+				gh.Hosts[i], gh.Hosts[j] = gh.Hosts[j], gh.Hosts[i]
+			})
+			psr.GroupedHosts = append(psr.GroupedHosts, gh)
+			shuffledHosts = append(shuffledHosts, gh.Hosts...)
+		}
 
-		//write shuffled hosts into worklist file
-		if err := ioutil.WriteFile(workList, []byte(strings.Join(hosts, "\n")+"\n"), 0644); err != nil {
+		//write shuffled hosts into worklist file - we no longer rely on this for scanning, kept for debugging
+		if err := ioutil.WriteFile(workList, []byte(strings.Join(shuffledHosts, "\n")+"\n"), 0644); err != nil {
 			log.Error(err)
 			return
 		}
 
 		//track progress in the progress file
-		if err := ioutil.WriteFile(progress, []byte(fmt.Sprintf("-1,%d", len(hosts))), 0644); err != nil {
+		if err := ioutil.WriteFile(progress, []byte(fmt.Sprintf("-1,%d", len(shuffledHosts))), 0644); err != nil {
 			log.Error(err)
 			return
 		}
@@ -372,8 +380,11 @@ func runTLSScan(ipSource func() []string, ipToHostnameResolver func(string) stri
 			log.Error(err)
 			return
 		}
-		psr.Hosts = hosts
-		request := tlsmodel.ScanRequest{}
+		psr.HostCount = len(shuffledHosts)
+		request := tlsmodel.AdvancedScanRequest{}
+		for _, gh := range psr.GroupedHosts {
+			request.ScanGroups = append(request.ScanGroups, gh.ScanGroup)
+		}
 		request.Day = today
 		request.ScanID = GetNextScanID()
 		config, _ := loadTLSConfig(TLSAuditConfigPath)
@@ -385,7 +396,11 @@ func runTLSScan(ipSource func() []string, ipToHostnameResolver func(string) stri
 		psr.Request = request
 	}
 
-	go resolveIPs(hosts)
+	//we've got the psr. Now use it as the basis of the scans
+	hosts := []string{} // hosts to scan
+	for _, gh := range psr.GroupedHosts {
+		hosts = append(hosts, gh.Hosts...)
+	}
 
 	//get ready to scan
 	//get where we "stopped" last time possibly after a crash
@@ -402,6 +417,11 @@ func runTLSScan(ipSource func() []string, ipToHostnameResolver func(string) stri
 	}
 	psr.Progress = stopped
 
+	//only resolve IPs for the hosts that have not been scanned
+	if stopped < len(hosts) {
+		go resolveIPs(hosts[stopped:])
+	}
+
 	PersistScanRequest(psr)
 
 	count := len(hosts)