Merge pull request #152 from sadayuki-matsuno/delete_sqlite

delete sqlite3

Makefile

@@ -11,7 +11,7 @@
 	clean
 
 SRCS = $(shell git ls-files '*.go')
-PKGS = ./. ./db ./config ./models ./report ./cveapi ./scan ./util ./commands
+PKGS = ./. ./config ./models ./report ./cveapi ./scan ./util ./commands
 
 all: test
 

README.ja.md

@@ -601,8 +601,8 @@ scan:
         scan
                 [-lang=en|ja]
                 [-config=/path/to/config.toml]
-                [-dbpath=/path/to/vuls.sqlite3]
-                [--cve-dictionary-dbpath=/path/to/cve.sqlite3]
+                [-results-dir=/path/to/results]
+                [-cve-dictionary-dbpath=/path/to/cve.sqlite3]
                 [-cve-dictionary-url=http://127.0.0.1:1323]
                 [-cvss-over=7]
                 [-ignore-unscored-cves]
@@ -649,8 +649,8 @@ scan:
         http://CVE.Dictionary (default "http://127.0.0.1:1323")
   -cvss-over float
         -cvss-over=6.5 means reporting CVSS Score 6.5 and over (default: 0 (means report all))
-  -dbpath string
-        /path/to/sqlite3 (default "$PWD/vuls.sqlite3")
+  -results-dir string
+        /path/to/results (default "$PWD/results")
   -debug
         debug mode
   -debug-sql
@@ -892,10 +892,10 @@ Vulsは、DockerホストにSSHで接続し、`docker exec`でDockerコンテナ
 ```
 $ vuls tui -h
 tui:
-	tui [-dbpath=/path/to/vuls.sqlite3]
+	tui [-results-dir=/path/to/results]
 
-  -dbpath string
-        /path/to/sqlite3 (default "$PWD/vuls.sqlite3")
+  -results-dir string
+        /path/to/results (default "$PWD/results")
   -debug-sql
     	debug SQL
 

README.md

@@ -599,8 +599,8 @@ scan:
         scan
                 [-lang=en|ja]
                 [-config=/path/to/config.toml]
-                [-dbpath=/path/to/vuls.sqlite3]
-                [--cve-dictionary-dbpath=/path/to/cve.sqlite3]
+                [-results-dir=/path/to/results]
+                [-cve-dictionary-dbpath=/path/to/cve.sqlite3]
                 [-cve-dictionary-url=http://127.0.0.1:1323]
                 [-cvss-over=7]
                 [-ignore-unscored-cves]
@@ -646,8 +646,8 @@ scan:
         http://CVE.Dictionary (default "http://127.0.0.1:1323")
   -cvss-over float
         -cvss-over=6.5 means reporting CVSS Score 6.5 and over (default: 0 (means report all))
-  -dbpath string
-        /path/to/sqlite3 (default "$PWD/vuls.sqlite3")
+  -results-dir string
+        /path/to/results (default "$PWD/results")
   -debug
         debug mode
   -debug-sql
@@ -882,10 +882,10 @@ For more details, see [Architecture section](https://github.com/future-architect
 ```
 $ vuls tui -h
 tui:
-	tui [-dbpath=/path/to/vuls.sqlite3]
+	tui [-results-dir=/path/to/results]
 
-  -dbpath string
-        /path/to/sqlite3 (default "$PWD/vuls.sqlite3")
+  -results-dir string
+        /path/to/results (default "$PWD/results")
   -debug-sql
     	debug SQL
 

commands/history.go

@@ -20,25 +20,22 @@ package commands
 import (
 	"flag"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
 
-	"golang.org/x/net/context"
-
-	"github.com/Sirupsen/logrus"
 	c "github.com/future-architect/vuls/config"
-	"github.com/future-architect/vuls/db"
-	"github.com/future-architect/vuls/models"
+	"github.com/future-architect/vuls/report"
 	"github.com/google/subcommands"
+	"golang.org/x/net/context"
 )
 
 // HistoryCmd is Subcommand of list scanned results
 type HistoryCmd struct {
-	debug    bool
-	debugSQL bool
-
-	dbpath string
+	debug       bool
+	debugSQL    bool
+	jsonBaseDir string
 }
 
 // Name return subcommand name
@@ -53,7 +50,7 @@ func (*HistoryCmd) Synopsis() string {
 func (*HistoryCmd) Usage() string {
 	return `history:
 	history
-		[-dbpath=/path/to/vuls.sqlite3]
+		[-results-dir=/path/to/results]
 	`
 }
 
@@ -62,47 +59,45 @@ func (p *HistoryCmd) SetFlags(f *flag.FlagSet) {
 	f.BoolVar(&p.debugSQL, "debug-sql", false, "SQL debug mode")
 
 	wd, _ := os.Getwd()
-	defaultDBPath := filepath.Join(wd, "vuls.sqlite3")
-	f.StringVar(&p.dbpath, "dbpath", defaultDBPath, "/path/to/sqlite3")
+	defaultJSONBaseDir := filepath.Join(wd, "results")
+	f.StringVar(&p.jsonBaseDir, "results-dir", defaultJSONBaseDir, "/path/to/results")
 }
 
 // Execute execute
 func (p *HistoryCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus {
 
 	c.Conf.DebugSQL = p.debugSQL
-	c.Conf.DBPath = p.dbpath
+	c.Conf.JSONBaseDir = p.jsonBaseDir
 
-	//  _, err := scanHistories()
-	histories, err := scanHistories()
-	if err != nil {
-		logrus.Error("Failed to select scan histories: ", err)
+	var err error
+	var jsonDirs report.JSONDirs
+	if jsonDirs, err = report.GetValidJSONDirs(); err != nil {
 		return subcommands.ExitFailure
 	}
-	const timeLayout = "2006-01-02 15:04"
-	for _, history := range histories {
-		names := []string{}
-		for _, result := range history.ScanResults {
-			if 0 < len(result.Container.ContainerID) {
-				names = append(names, result.Container.Name)
-			} else {
-				names = append(names, result.ServerName)
+	for _, d := range jsonDirs {
+		var files []os.FileInfo
+		if files, err = ioutil.ReadDir(d); err != nil {
+			return subcommands.ExitFailure
+		}
+		var hosts []string
+		for _, f := range files {
+			// TODO this "if block" will be deleted in a future release
+			if f.Name() == "all.json" {
+				continue
 			}
+			if filepath.Ext(f.Name()) != ".json" {
+				continue
+			}
+			fileBase := strings.TrimSuffix(f.Name(), filepath.Ext(f.Name()))
+			hosts = append(hosts, fileBase)
 		}
-		fmt.Printf("%-3d %s scanned %d servers: %s\n",
-			history.ID,
-			history.ScannedAt.Format(timeLayout),
-			len(history.ScanResults),
-			strings.Join(names, ", "),
+		splitPath := strings.Split(d, string(os.PathSeparator))
+		timeStr := splitPath[len(splitPath)-1]
+		fmt.Printf("%s scanned %d servers: %s\n",
+			timeStr,
+			len(hosts),
+			strings.Join(hosts, ", "),
 		)
 	}
 	return subcommands.ExitSuccess
 }
-
-func scanHistories() (histories []models.ScanHistory, err error) {
-	if err := db.OpenDB(); err != nil {
-		return histories, fmt.Errorf(
-			"Failed to open DB. datafile: %s, err: %s", c.Conf.DBPath, err)
-	}
-	histories, err = db.SelectScanHistories()
-	return
-}

commands/scan.go

@@ -24,11 +24,11 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/Sirupsen/logrus"
 	c "github.com/future-architect/vuls/config"
 	"github.com/future-architect/vuls/cveapi"
-	"github.com/future-architect/vuls/db"
 	"github.com/future-architect/vuls/report"
 	"github.com/future-architect/vuls/scan"
 	"github.com/future-architect/vuls/util"
@@ -44,7 +44,7 @@ type ScanCmd struct {
 
 	configPath string
 
-	dbpath           string
+	jsonBaseDir      string
 	cvedbpath        string
 	cveDictionaryURL string
 
@@ -86,7 +86,7 @@ func (*ScanCmd) Usage() string {
 	scan
 		[-lang=en|ja]
 		[-config=/path/to/config.toml]
-		[-dbpath=/path/to/vuls.sqlite3]
+		[-results-dir=/path/to/results]
 		[-cve-dictionary-dbpath=/path/to/cve.sqlite3]
 		[-cve-dictionary-url=http://127.0.0.1:1323]
 		[-cvss-over=7]
@@ -124,8 +124,8 @@ func (p *ScanCmd) SetFlags(f *flag.FlagSet) {
 	defaultConfPath := filepath.Join(wd, "config.toml")
 	f.StringVar(&p.configPath, "config", defaultConfPath, "/path/to/toml")
 
-	defaultDBPath := filepath.Join(wd, "vuls.sqlite3")
-	f.StringVar(&p.dbpath, "dbpath", defaultDBPath, "/path/to/sqlite3")
+	defaultJSONBaseDir := filepath.Join(wd, "results")
+	f.StringVar(&p.jsonBaseDir, "results-dir", defaultJSONBaseDir, "/path/to/results")
 
 	f.StringVar(
 		&p.cvedbpath,
@@ -284,6 +284,7 @@ func (p *ScanCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{})
 
 	// logger
 	Log := util.NewCustomLogger(c.ServerInfo{})
+	scannedAt := time.Now()
 
 	// report
 	reports := []report.ResultWriter{
@@ -297,10 +298,10 @@ func (p *ScanCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{})
 		reports = append(reports, report.MailWriter{})
 	}
 	if p.reportJSON {
-		reports = append(reports, report.JSONWriter{})
+		reports = append(reports, report.JSONWriter{ScannedAt: scannedAt})
 	}
 	if p.reportText {
-		reports = append(reports, report.TextFileWriter{})
+		reports = append(reports, report.TextFileWriter{ScannedAt: scannedAt})
 	}
 	if p.reportS3 {
 		c.Conf.AwsRegion = p.awsRegion
@@ -315,17 +316,17 @@ func (p *ScanCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{})
 	}
 	if p.reportAzureBlob {
 		c.Conf.AzureAccount = p.azureAccount
-		if c.Conf.AzureAccount == "" {
+		if len(c.Conf.AzureAccount) == 0 {
 			c.Conf.AzureAccount = os.Getenv("AZURE_STORAGE_ACCOUNT")
 		}
 
 		c.Conf.AzureKey = p.azureKey
-		if c.Conf.AzureKey == "" {
+		if len(c.Conf.AzureKey) == 0 {
 			c.Conf.AzureKey = os.Getenv("AZURE_STORAGE_ACCESS_KEY")
 		}
 
 		c.Conf.AzureContainer = p.azureContainer
-		if c.Conf.AzureContainer == "" {
+		if len(c.Conf.AzureContainer) == 0 {
 			Log.Error("Azure storage container name is requied with --azure-container option")
 			return subcommands.ExitUsageError
 		}
@@ -337,7 +338,7 @@ func (p *ScanCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{})
 		reports = append(reports, report.AzureBlobWriter{})
 	}
 
-	c.Conf.DBPath = p.dbpath
+	c.Conf.JSONBaseDir = p.jsonBaseDir
 	c.Conf.CveDBPath = p.cvedbpath
 	c.Conf.CveDictionaryURL = p.cveDictionaryURL
 	c.Conf.CvssScoreOver = p.cvssScoreOver
@@ -382,21 +383,6 @@ func (p *ScanCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{})
 		return subcommands.ExitFailure
 	}
 
-	Log.Info("Insert to DB...")
-	if err := db.OpenDB(); err != nil {
-		Log.Errorf("Failed to open DB. datafile: %s, err: %s", c.Conf.DBPath, err)
-		return subcommands.ExitFailure
-	}
-	if err := db.MigrateDB(); err != nil {
-		Log.Errorf("Failed to migrate. err: %s", err)
-		return subcommands.ExitFailure
-	}
-
-	if err := db.Insert(scanResults); err != nil {
-		Log.Fatalf("Failed to insert. dbpath: %s, err: %s", c.Conf.DBPath, err)
-		return subcommands.ExitFailure
-	}
-
 	Log.Info("Reporting...")
 	filtered := scanResults.FilterByCvssOver()
 	for _, w := range reports {