Fixes for V2 for alpine, rhel, nvd, and ubuntu

Signed-off-by: nycnewman <edward@digitalasset.com>

ext/vulnsrc/alpine/alpine.go

@@ -17,25 +17,29 @@
 package alpine
 
 import (
+	"crypto/sha256"
+	"encoding/hex"
 	"io"
 	"io/ioutil"
+	"net/http"
 	"os"
-	"os/exec"
 	"path/filepath"
 	"strings"
 
 	log "github.com/sirupsen/logrus"
 	"gopkg.in/yaml.v2"
 
+	"github.com/PuerkitoBio/goquery"
+
 	"github.com/quay/clair/v2/database"
 	"github.com/quay/clair/v2/ext/versionfmt"
 	"github.com/quay/clair/v2/ext/versionfmt/dpkg"
 	"github.com/quay/clair/v2/ext/vulnsrc"
-	"github.com/quay/clair/v2/pkg/commonerr"
 )
 
 const (
-	secdbGitURL  = "https://github.com/alpinelinux/alpine-secdb"
+	// secdbGitURL  = "https://github.com/alpinelinux/alpine-secdb"
+	baseURL  = "https://secdb.alpinelinux.org/"
 	updaterFlag  = "alpine-secdbUpdater"
 	nvdURLPrefix = "https://cve.mitre.org/cgi-bin/cvename.cgi?name="
 )
@@ -46,15 +50,136 @@ func init() {
 
 type updater struct {
 	repositoryLocalPath string
+	currentDir string
+	hash_slice [][32] byte
+}
+
+func (u *updater) processFile(filename string) {
+	nameParts := strings.Split(filename, ".")
+	if nameParts[1] == "json" {
+		return
+	}
+
+	response, err := http.Get(baseURL + u.currentDir + filename)
+	if err != nil {
+		//log.WithError(err).WithField("package", "Alpine").Error("Failed to get vuln file")
+		return
+	}
+	defer response.Body.Close()
+
+	file, err := os.Create(filepath.Join(filepath.Join(u.repositoryLocalPath, u.currentDir), filename))
+	if err != nil {
+		log.WithField("package", "Alpine").Fatal(err)
+		return
+	}
+	defer file.Close()
+
+	// find hash of file contents as part of checking for changes
+	file_hasher := sha256.New()
+	fileContents, err := ioutil.ReadAll(response.Body)
+	file_hasher.Write([] byte(fileContents[:]))
+
+	// Must be a better way to achieve this...
+	var file_hash [32]byte
+	copy(file_hash[:], file_hasher.Sum(nil))
+	u.hash_slice = append(u.hash_slice, file_hash)
+
+	file.WriteString(string(fileContents[:]))
+	return
+}
+
+func (u *updater) processFiles(index int, element *goquery.Selection) {
+	href, exists := element.Attr("href")
+	if exists {
+		if href != "../" {
+			u.processFile(href)
+		}
+	}
+}
+
+func (u *updater) processVersionDir(versionDir string) {
+	response, err := http.Get(baseURL + versionDir)
+	if err != nil {
+		log.WithError(err).WithField("package", "Alpine").Error("Failed to get version")
+	}
+	defer response.Body.Close()
+
+	document, err := goquery.NewDocumentFromReader(response.Body)
+	if err != nil {
+		log.Fatal("Error loading HTTP response body. ", err)
+	}
+	document.Find("a").Each(u.processFiles)
+}
+
+func (u *updater) processVersions(index int, element *goquery.Selection) {
+	href, exists := element.Attr("href")
+	if exists {
+		if href != "../" {
+			// create Version directory
+			os.Mkdir(filepath.Join(u.repositoryLocalPath, href),0700)
+			u.currentDir = href
+			u.processVersionDir(href)
+		}
+	}
+}
+
+func sliceXOR (a, b [32]byte) (result [32]byte) {
+	var tmpval [32]byte
+	for i:=0; i<32; i++ {
+		tmpval[i] = a[i] ^ b[i]
+	}
+	result = tmpval
+	return
+}
+
+func (u *updater) getVulnFiles(repoPath, tempDirPrefix string) (commit string, err error) {
+	log.WithField("package", "alpine").Debug("Getting vulnerability data...")
+
+	// Set up temporary location for downlaods
+	if repoPath == "" {
+		u.repositoryLocalPath, err = ioutil.TempDir(os.TempDir(), tempDirPrefix)
+		if err != nil {
+			return
+		}
+	} else {
+		u.repositoryLocalPath = repoPath
+	}
+
+	u.hash_slice = nil
+	u.currentDir = ""
+
+	// Get root directory of web server
+	response, err := http.Get(baseURL)
+	if err != nil {
+		return
+	}
+	defer response.Body.Close()
+
+	document, err := goquery.NewDocumentFromReader(response.Body)
+	if err != nil {
+		log.WithError(err).WithField("package", "Alpine").Fatal("Error loading HTTP response body. ")
+		return
+	}
+	document.Find("a").Each(u.processVersions)
+
+	// Find XOR of all file hash values to use as commit hash replacement. Used to detect for changes to source files
+	var tmp_commit [32]byte
+	for i:=0; i < len(u.hash_slice); i++ {
+		tmp_commit = sliceXOR(tmp_commit, u.hash_slice[i])
+	}
+	commit = hex.EncodeToString(tmp_commit[:])
+
+	return
 }
 
 func (u *updater) Update(db database.Datastore) (resp vulnsrc.UpdateResponse, err error) {
 	log.WithField("package", "Alpine").Info("Start fetching vulnerabilities")
 
 	// Pull the master branch.
 	var commit string
-	commit, err = u.pullRepository()
-	if err != nil {
+
+	if commit, err = u.getVulnFiles(u.repositoryLocalPath, updaterFlag); err != nil {
+		log.WithField("package", "alpine").Debug("no file updates, skip")
 		return
 	}
 
@@ -172,40 +297,6 @@ func parseVulnsFromNamespace(repositoryPath, namespace string) (vulns []database
 	return
 }
 
-func (u *updater) pullRepository() (commit string, err error) {
-	// If the repository doesn't exist, clone it.
-	if _, pathExists := os.Stat(u.repositoryLocalPath); u.repositoryLocalPath == "" || os.IsNotExist(pathExists) {
-		if u.repositoryLocalPath, err = ioutil.TempDir(os.TempDir(), "alpine-secdb"); err != nil {
-			return "", vulnsrc.ErrFilesystem
-		}
-
-		cmd := exec.Command("git", "clone", secdbGitURL, ".")
-		cmd.Dir = u.repositoryLocalPath
-		if out, err := cmd.CombinedOutput(); err != nil {
-			u.Clean()
-			log.WithError(err).WithField("output", string(out)).Error("could not pull alpine-secdb repository")
-			return "", commonerr.ErrCouldNotDownload
-		}
-	} else {
-		// The repository already exists and it needs to be refreshed via a pull.
-		cmd := exec.Command("git", "pull")
-		cmd.Dir = u.repositoryLocalPath
-		if _, err := cmd.CombinedOutput(); err != nil {
-			return "", vulnsrc.ErrGitFailure
-		}
-	}
-
-	cmd := exec.Command("git", "rev-parse", "HEAD")
-	cmd.Dir = u.repositoryLocalPath
-	out, err := cmd.CombinedOutput()
-	if err != nil {
-		return "", vulnsrc.ErrGitFailure
-	}
-
-	commit = strings.TrimSpace(string(out))
-	return
-}
-
 type secDBFile struct {
 	Distro   string `yaml:"distroversion"`
 	Packages []struct {

ext/vulnsrc/rhel/rhel.go

@@ -339,6 +339,8 @@ func link(def definition) (link string) {
 
 func severity(def definition) database.Severity {
 	switch strings.TrimSpace(def.Title[strings.LastIndex(def.Title, "(")+1 : len(def.Title)-1]) {
+	case "None":
+		return database.NegligibleSeverity
 	case "Low":
 		return database.LowSeverity
 	case "Moderate":

ext/vulnsrc/ubuntu/ubuntu.go

@@ -307,37 +307,56 @@ func parseUbuntuCVE(fileContent io.Reader) (vulnerability database.Vulnerability
 					continue
 				}
 
-				var version string
+				// var version string
+				var versions []string
+
+				versions = nil
+
 				if md["status"] == "released" {
 					if md["note"] != "" {
-						var err error
-						err = versionfmt.Valid(dpkg.ParserName, md["note"])
-						if err != nil {
-							log.WithError(err).WithField("version", md["note"]).Warning("could not parse package version. skipping")
+						if strings.Contains(md["note"], ",")  { //contains a comma
+							var tmp_strings []string
+							tmp_strings = strings.Split(md["note"], ",")
+							for _,s := range tmp_strings {
+								var err error
+								err = versionfmt.Valid(dpkg.ParserName, s)
+								if err != nil {
+									log.WithError(err).WithField("version", md["note"]).Warning("could not parse package version. skipping")
+								}
+								versions = append(versions, s)
+							}
+						} else {
+							var err error
+							err = versionfmt.Valid(dpkg.ParserName, md["note"])
+							if err != nil {
+								log.WithError(err).WithField("version", md["note"]).Warning("could not parse package version. skipping")
+							}
+							versions = append(versions, md["note"])
 						}
-						version = md["note"]
 					}
 				} else if md["status"] == "not-affected" {
-					version = versionfmt.MinVersion
+					versions = append(versions, versionfmt.MinVersion)
 				} else {
-					version = versionfmt.MaxVersion
+					versions = append(versions, versionfmt.MaxVersion)
 				}
-				if version == "" {
+				if versions == nil {
 					continue
 				}
 
 				// Create and add the new package.
-				featureVersion := database.FeatureVersion{
-					Feature: database.Feature{
-						Namespace: database.Namespace{
-							Name:          "ubuntu:" + database.UbuntuReleasesMapping[md["release"]],
-							VersionFormat: dpkg.ParserName,
+				for _, v := range versions {
+					featureVersion := database.FeatureVersion{
+						Feature: database.Feature{
+							Namespace: database.Namespace{
+								Name:          "ubuntu:" + database.UbuntuReleasesMapping[md["release"]],
+								VersionFormat: dpkg.ParserName,
+							},
+							Name: md["package"],
 						},
-						Name: md["package"],
-					},
-					Version: version,
+						Version: v,
+					}
+					vulnerability.FixedIn = append(vulnerability.FixedIn, featureVersion)
 				}
-				vulnerability.FixedIn = append(vulnerability.FixedIn, featureVersion)
 			}
 		}
 	}

go.mod

@@ -4,6 +4,7 @@ go 1.12
 
 require (
 	github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a
+	github.com/coreos/clair v2.1.0+incompatible // indirect
 	github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf
 	github.com/davecgh/go-spew v1.0.1-0.20160907170601-6d212800a42e
 	github.com/fernet/fernet-go v0.0.0-20151007213151-1b2437bc582b

go.sum

@@ -1,5 +1,8 @@
 github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a h1:BtpsbiV638WQZwhA98cEZw2BsbnQJrbd0BI7tsy0W1c=
 github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/coreos/clair v1.2.6 h1:hhgkFHuEAx9l1iIwFfemrUZJc1lo1mVTs3MYZoDYiVQ=
+github.com/coreos/clair v2.1.0+incompatible h1:lY0fTAGneYxXfq0j2vM+Xxip30XBzSn2tzSolAwkMnc=
+github.com/coreos/clair v2.1.0+incompatible/go.mod h1:uXhHPWAoRqw0jJc2f8RrPCwRhIo9otQ8OEWUFtpCiwA=
 github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf h1:CAKfRE2YtTUIjjh1bkBtyYFaUT/WmOqsJjgtihT0vMI=
 github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/davecgh/go-spew v1.0.1-0.20160907170601-6d212800a42e h1:9EoM2C6YAkhnxTxG3LrAos2/KaALZdSNG5HTGPEEedE=
@@ -38,6 +41,7 @@ github.com/stretchr/testify v1.1.4 h1:ToftOQTytwshuOSj6bDSolVUa3GINfJP/fg3OkkOzQ
 github.com/stretchr/testify v1.1.4/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/tylerb/graceful v1.2.15 h1:B0x01Y8fsJpogzZTkDg6BDi6eMf03s01lEKGdrv83oA=
 github.com/tylerb/graceful v1.2.15/go.mod h1:LPYTbOYmUTdabwRt0TGhLllQ0MUNbs0Y5q1WXJOI9II=
+golang.org/x/sys v0.0.0-20170427041856-9ccfe848b9db h1:znurcNjtwV7XblDOBERYCP1TUjpwbp8bi3Szx8gbNBE=
 golang.org/x/sys v0.0.0-20170427041856-9ccfe848b9db/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 gopkg.in/yaml.v2 v2.0.0-20170407172122-cd8b52f8269e h1:o/mfNjxpTLivuKEfxzzwrJ8PmulH2wEp7t713uMwKAA=
 gopkg.in/yaml.v2 v2.0.0-20170407172122-cd8b52f8269e/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=