Display metasploit module information for each detected CVE-IDs (#1011)

* add metasploit

* fix go deps

* fix msf report

* fix msfdb server port number

* delete non-unique msfdb url from fulltext report

* fix(report): validate msfdb config on report (#1)

* fix(msfdb): update deps (go-msfdb)

* version up go-msfdb v0.1.0

Co-authored-by: Kota Kanbe <kotakanbe@gmail.com>

commands/discover.go

@@ -95,6 +95,11 @@ type        = "sqlite3"
 sqlite3Path = "/path/to/go-exploitdb.sqlite3"
 #url        = ""
 
+[metasploit]
+type        = "sqlite3"
+sqlite3Path = "/path/to/go-msfdb.sqlite3"
+#url        = ""
+
 # https://vuls.io/docs/en/usage-settings.html#slack-section
 #[slack]
 #hookURL      = "https://hooks.slack.com/services/abc123/defghijklmnopqrstuvwxyz"

commands/report.go

@@ -11,6 +11,7 @@ import (
 	"github.com/future-architect/vuls/exploit"
 	"github.com/future-architect/vuls/gost"
 	"github.com/future-architect/vuls/models"
+	"github.com/future-architect/vuls/msf"
 	"github.com/future-architect/vuls/oval"
 	"github.com/future-architect/vuls/report"
 	"github.com/future-architect/vuls/util"
@@ -21,12 +22,13 @@ import (
 
 // ReportCmd is subcommand for reporting
 type ReportCmd struct {
-	configPath  string
-	cveDict     c.GoCveDictConf
-	ovalDict    c.GovalDictConf
-	gostConf    c.GostConf
-	exploitConf c.ExploitConf
-	httpConf    c.HTTPConf
+	configPath     string
+	cveDict        c.GoCveDictConf
+	ovalDict       c.GovalDictConf
+	gostConf       c.GostConf
+	exploitConf    c.ExploitConf
+	metasploitConf c.MetasploitConf
+	httpConf       c.HTTPConf
 }
 
 // Name return subcommand name
@@ -87,6 +89,9 @@ func (*ReportCmd) Usage() string {
 		[-exploitdb-type=sqlite3|mysql|redis|http]
 		[-exploitdb-sqlite3-path=/path/to/exploitdb.sqlite3]
 		[-exploitdb-url=http://127.0.0.1:1326 or DB connection string]
+		[-msfdb-type=sqlite3|mysql|redis|http]
+		[-msfdb-sqlite3-path=/path/to/msfdb.sqlite3]
+		[-msfdb-url=http://127.0.0.1:1327 or DB connection string]
 		[-http="http://vuls-report-server"]
 		[-trivy-cachedb-dir=/path/to/dir]
 
@@ -192,6 +197,12 @@ func (p *ReportCmd) SetFlags(f *flag.FlagSet) {
 	f.StringVar(&p.exploitConf.URL, "exploitdb-url", "",
 		"http://exploit.com:1326 or DB connection string")
 
+	f.StringVar(&p.metasploitConf.Type, "msfdb-type", "",
+		"DB type of msf (sqlite3, mysql, postgres, redis or http)")
+	f.StringVar(&p.metasploitConf.SQLite3Path, "msfdb-sqlite3-path", "", "/path/to/sqlite3")
+	f.StringVar(&p.metasploitConf.URL, "msfdb-url", "",
+		"http://metasploit.com:1327 or DB connection string")
+
 	f.StringVar(&p.httpConf.URL, "http", "", "-to-http http://vuls-report")
 
 	f.StringVar(&c.Conf.TrivyCacheDBDir, "trivy-cachedb-dir",
@@ -212,6 +223,7 @@ func (p *ReportCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}
 	c.Conf.OvalDict.Overwrite(p.ovalDict)
 	c.Conf.Gost.Overwrite(p.gostConf)
 	c.Conf.Exploit.Overwrite(p.exploitConf)
+	c.Conf.Metasploit.Overwrite(p.metasploitConf)
 	c.Conf.HTTP.Overwrite(p.httpConf)
 
 	var dir string
@@ -395,12 +407,22 @@ func (p *ReportCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}
 				return subcommands.ExitFailure
 			}
 		}
+
+		if c.Conf.Metasploit.URL != "" {
+			err := msf.CheckHTTPHealth()
+			if err != nil {
+				util.Log.Errorf("metasploit HTTP server is not running. err: %+v", err)
+				util.Log.Errorf("Run go-msfdb as server mode before reporting")
+				return subcommands.ExitFailure
+			}
+		}
 		dbclient, locked, err := report.NewDBClient(report.DBClientConf{
-			CveDictCnf:  c.Conf.CveDict,
-			OvalDictCnf: c.Conf.OvalDict,
-			GostCnf:     c.Conf.Gost,
-			ExploitCnf:  c.Conf.Exploit,
-			DebugSQL:    c.Conf.DebugSQL,
+			CveDictCnf:    c.Conf.CveDict,
+			OvalDictCnf:   c.Conf.OvalDict,
+			GostCnf:       c.Conf.Gost,
+			ExploitCnf:    c.Conf.Exploit,
+			MetasploitCnf: c.Conf.Metasploit,
+			DebugSQL:      c.Conf.DebugSQL,
 		})
 		if locked {
 			util.Log.Errorf("SQLite3 is locked. Close other DB connections and try again. err: %+v", err)

commands/server.go

@@ -13,6 +13,7 @@ import (
 	c "github.com/future-architect/vuls/config"
 	"github.com/future-architect/vuls/exploit"
 	"github.com/future-architect/vuls/gost"
+	"github.com/future-architect/vuls/msf"
 	"github.com/future-architect/vuls/oval"
 	"github.com/future-architect/vuls/report"
 	"github.com/future-architect/vuls/server"
@@ -23,12 +24,13 @@ import (
 
 // ServerCmd is subcommand for server
 type ServerCmd struct {
-	configPath  string
-	listen      string
-	cveDict     c.GoCveDictConf
-	ovalDict    c.GovalDictConf
-	gostConf    c.GostConf
-	exploitConf c.ExploitConf
+	configPath     string
+	listen         string
+	cveDict        c.GoCveDictConf
+	ovalDict       c.GovalDictConf
+	gostConf       c.GostConf
+	exploitConf    c.ExploitConf
+	metasploitConf c.MetasploitConf
 }
 
 // Name return subcommand name
@@ -65,6 +67,9 @@ func (*ServerCmd) Usage() string {
 		[-exploitdb-type=sqlite3|mysql|redis|http]
 		[-exploitdb-sqlite3-path=/path/to/exploitdb.sqlite3]
 		[-exploitdb-url=http://127.0.0.1:1326 or DB connection string]
+		[-msfdb-type=sqlite3|mysql|redis|http]
+		[-msfdb-sqlite3-path=/path/to/msfdb.sqlite3]
+		[-msfdb-url=http://127.0.0.1:1327 or DB connection string]
 
 		[RFC3339 datetime format under results dir]
 `
@@ -126,6 +131,12 @@ func (p *ServerCmd) SetFlags(f *flag.FlagSet) {
 	f.StringVar(&p.exploitConf.SQLite3Path, "exploitdb-sqlite3-path", "", "/path/to/sqlite3")
 	f.StringVar(&p.exploitConf.URL, "exploitdb-url", "",
 		"http://exploit.com:1326 or DB connection string")
+
+	f.StringVar(&p.metasploitConf.Type, "msfdb-type", "",
+		"DB type of msf (sqlite3, mysql, postgres, redis or http)")
+	f.StringVar(&p.metasploitConf.SQLite3Path, "msfdb-sqlite3-path", "", "/path/to/sqlite3")
+	f.StringVar(&p.metasploitConf.URL, "msfdb-url", "",
+		"http://metasploit.com:1327 or DB connection string")
 }
 
 // Execute execute
@@ -144,6 +155,7 @@ func (p *ServerCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}
 	c.Conf.OvalDict.Overwrite(p.ovalDict)
 	c.Conf.Gost.Overwrite(p.gostConf)
 	c.Conf.Exploit.Overwrite(p.exploitConf)
+	c.Conf.Metasploit.Overwrite(p.metasploitConf)
 
 	util.Log.Info("Validating config...")
 	if !c.Conf.ValidateOnReport() {
@@ -191,12 +203,21 @@ func (p *ServerCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}
 		}
 	}
 
+	if c.Conf.Metasploit.URL != "" {
+		err := msf.CheckHTTPHealth()
+		if err != nil {
+			util.Log.Errorf("metasploit HTTP server is not running. err: %+v", err)
+			util.Log.Errorf("Run go-msfdb as server mode before reporting")
+			return subcommands.ExitFailure
+		}
+	}
 	dbclient, locked, err := report.NewDBClient(report.DBClientConf{
-		CveDictCnf:  c.Conf.CveDict,
-		OvalDictCnf: c.Conf.OvalDict,
-		GostCnf:     c.Conf.Gost,
-		ExploitCnf:  c.Conf.Exploit,
-		DebugSQL:    c.Conf.DebugSQL,
+		CveDictCnf:    c.Conf.CveDict,
+		OvalDictCnf:   c.Conf.OvalDict,
+		GostCnf:       c.Conf.Gost,
+		ExploitCnf:    c.Conf.Exploit,
+		MetasploitCnf: c.Conf.Metasploit,
+		DebugSQL:      c.Conf.DebugSQL,
 	})
 	if locked {
 		util.Log.Errorf("SQLite3 is locked. Close other DB connections and try again: %+v", err)

commands/tui.go

@@ -11,6 +11,7 @@ import (
 	"github.com/future-architect/vuls/exploit"
 	"github.com/future-architect/vuls/gost"
 	"github.com/future-architect/vuls/models"
+	"github.com/future-architect/vuls/msf"
 	"github.com/future-architect/vuls/oval"
 	"github.com/future-architect/vuls/report"
 	"github.com/future-architect/vuls/util"
@@ -20,11 +21,12 @@ import (
 
 // TuiCmd is Subcommand of host discovery mode
 type TuiCmd struct {
-	configPath  string
-	cveDict     c.GoCveDictConf
-	ovalDict    c.GovalDictConf
-	gostConf    c.GostConf
-	exploitConf c.ExploitConf
+	configPath     string
+	cveDict        c.GoCveDictConf
+	ovalDict       c.GovalDictConf
+	gostConf       c.GostConf
+	exploitConf    c.ExploitConf
+	metasploitConf c.MetasploitConf
 }
 
 // Name return subcommand name
@@ -62,6 +64,9 @@ func (*TuiCmd) Usage() string {
 		[-exploitdb-type=sqlite3|mysql|redis|http]
 		[-exploitdb-sqlite3-path=/path/to/exploitdb.sqlite3]
 		[-exploitdb-url=http://127.0.0.1:1326 or DB connection string]
+		[-msfdb-type=sqlite3|mysql|redis|http]
+		[-msfdb-sqlite3-path=/path/to/msfdb.sqlite3]
+		[-msfdb-url=http://127.0.0.1:1327 or DB connection string]
 		[-trivy-cachedb-dir=/path/to/dir]
 
 `
@@ -127,6 +132,12 @@ func (p *TuiCmd) SetFlags(f *flag.FlagSet) {
 	f.StringVar(&p.exploitConf.URL, "exploitdb-url", "",
 		"http://exploit.com:1326 or DB connection string")
 
+	f.StringVar(&p.metasploitConf.Type, "msfdb-type", "",
+		"DB type of msf (sqlite3, mysql, postgres, redis or http)")
+	f.StringVar(&p.metasploitConf.SQLite3Path, "msfdb-sqlite3-path", "", "/path/to/sqlite3")
+	f.StringVar(&p.metasploitConf.URL, "msfdb-url", "",
+		"http://metasploit.com:1327 or DB connection string")
+
 	f.StringVar(&c.Conf.TrivyCacheDBDir, "trivy-cachedb-dir",
 		utils.DefaultCacheDir(), "/path/to/dir")
 }
@@ -148,6 +159,7 @@ func (p *TuiCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) s
 	c.Conf.OvalDict.Overwrite(p.ovalDict)
 	c.Conf.Gost.Overwrite(p.gostConf)
 	c.Conf.Exploit.Overwrite(p.exploitConf)
+	c.Conf.Metasploit.Overwrite(p.metasploitConf)
 
 	var dir string
 	var err error
@@ -213,12 +225,22 @@ func (p *TuiCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) s
 			return subcommands.ExitFailure
 		}
 	}
+
+	if c.Conf.Metasploit.URL != "" {
+		err := msf.CheckHTTPHealth()
+		if err != nil {
+			util.Log.Errorf("metasploit HTTP server is not running. err: %+v", err)
+			util.Log.Errorf("Run go-msfdb as server mode before reporting")
+			return subcommands.ExitFailure
+		}
+	}
 	dbclient, locked, err := report.NewDBClient(report.DBClientConf{
-		CveDictCnf:  c.Conf.CveDict,
-		OvalDictCnf: c.Conf.OvalDict,
-		GostCnf:     c.Conf.Gost,
-		ExploitCnf:  c.Conf.Exploit,
-		DebugSQL:    c.Conf.DebugSQL,
+		CveDictCnf:    c.Conf.CveDict,
+		OvalDictCnf:   c.Conf.OvalDict,
+		GostCnf:       c.Conf.Gost,
+		ExploitCnf:    c.Conf.Exploit,
+		MetasploitCnf: c.Conf.Metasploit,
+		DebugSQL:      c.Conf.DebugSQL,
 	})
 	if locked {
 		util.Log.Errorf("SQLite3 is locked. Close other DB connections and try again: %+v", err)

config/config.go

@@ -114,10 +114,11 @@ type Config struct {
 	UUID       bool `json:"uuid,omitempty"`
 	DetectIPS  bool `json:"detectIps,omitempty"`
 
-	CveDict  GoCveDictConf `json:"cveDict,omitempty"`
-	OvalDict GovalDictConf `json:"ovalDict,omitempty"`
-	Gost     GostConf      `json:"gost,omitempty"`
-	Exploit  ExploitConf   `json:"exploit,omitempty"`
+	CveDict    GoCveDictConf  `json:"cveDict,omitempty"`
+	OvalDict   GovalDictConf  `json:"ovalDict,omitempty"`
+	Gost       GostConf       `json:"gost,omitempty"`
+	Exploit    ExploitConf    `json:"exploit,omitempty"`
+	Metasploit MetasploitConf `json:"metasploit,omitempty"`
 
 	Slack    SlackConf    `json:"-"`
 	EMail    SMTPConf     `json:"-"`
@@ -245,6 +246,10 @@ func (c Config) ValidateOnReportDB() bool {
 		errs = append(errs, err)
 	}
 
+	if err := validateDB("msfdb", c.Metasploit.Type, c.Metasploit.SQLite3Path, c.Metasploit.URL); err != nil {
+		errs = append(errs, err)
+	}
+
 	for _, err := range errs {
 		log.Error(err)
 	}
@@ -1000,6 +1005,64 @@ func (cnf *ExploitConf) IsFetchViaHTTP() bool {
 	return Conf.Exploit.Type == "http"
 }
 
+// MetasploitConf is metasploit config
+type MetasploitConf struct {
+	// DB type for metasploit dictionary (sqlite3, mysql, postgres or redis)
+	Type string
+
+	// http://metasploit-dictionary.com:1324 or DB connection string
+	URL string `json:"-"`
+
+	// /path/to/metasploit.sqlite3
+	SQLite3Path string `json:"-"`
+}
+
+func (cnf *MetasploitConf) setDefault() {
+	if cnf.Type == "" {
+		cnf.Type = "sqlite3"
+	}
+	if cnf.URL == "" && cnf.SQLite3Path == "" {
+		wd, _ := os.Getwd()
+		cnf.SQLite3Path = filepath.Join(wd, "go-msfdb.sqlite3")
+	}
+}
+
+const metasploitDBType = "METASPLOITDB_TYPE"
+const metasploitDBURL = "METASPLOITDB_URL"
+const metasploitDBPATH = "METASPLOITDB_SQLITE3_PATH"
+
+// Overwrite set options with the following priority.
+// 1. Command line option
+// 2. Environment variable
+// 3. config.toml
+func (cnf *MetasploitConf) Overwrite(cmdOpt MetasploitConf) {
+	if os.Getenv(metasploitDBType) != "" {
+		cnf.Type = os.Getenv(metasploitDBType)
+	}
+	if os.Getenv(metasploitDBURL) != "" {
+		cnf.URL = os.Getenv(metasploitDBURL)
+	}
+	if os.Getenv(metasploitDBPATH) != "" {
+		cnf.SQLite3Path = os.Getenv(metasploitDBPATH)
+	}
+
+	if cmdOpt.Type != "" {
+		cnf.Type = cmdOpt.Type
+	}
+	if cmdOpt.URL != "" {
+		cnf.URL = cmdOpt.URL
+	}
+	if cmdOpt.SQLite3Path != "" {
+		cnf.SQLite3Path = cmdOpt.SQLite3Path
+	}
+	cnf.setDefault()
+}
+
+// IsFetchViaHTTP returns wether fetch via http
+func (cnf *MetasploitConf) IsFetchViaHTTP() bool {
+	return Conf.Metasploit.Type == "http"
+}
+
 // AWS is aws config
 type AWS struct {
 	// AWS profile to use

config/tomlloader.go

@@ -35,6 +35,7 @@ func (c TOMLLoader) Load(pathToToml, keyPass string) error {
 	Conf.OvalDict = conf.OvalDict
 	Conf.Gost = conf.Gost
 	Conf.Exploit = conf.Exploit
+	Conf.Metasploit = conf.Metasploit
 
 	d := conf.Default
 	Conf.Default = d

go.mod

@@ -46,8 +46,10 @@ require (
 	github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5
 	github.com/sirupsen/logrus v1.6.0
 	github.com/spf13/afero v1.3.0
-	github.com/spf13/cobra v0.0.5
+	github.com/spf13/cobra v1.0.0
+	github.com/takuzoo3868/go-msfdb v0.1.0
 	golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9
+	golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
 	golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543
 	k8s.io/utils v0.0.0-20200619165400-6e3d28b6ed19