implement nodejs fetcher

Signed-off-by: liang chenye <liangchenye@huawei.com>

cmd/clair/main.go

@@ -29,6 +29,7 @@ import (
 	_ "github.com/coreos/clair/notifier/notifiers"
 
 	_ "github.com/coreos/clair/updater/fetchers/debian"
+	_ "github.com/coreos/clair/updater/fetchers/nodejs"
 	_ "github.com/coreos/clair/updater/fetchers/rhel"
 	_ "github.com/coreos/clair/updater/fetchers/ubuntu"
 	_ "github.com/coreos/clair/updater/metadata_fetchers/nvd"

updater/fetchers/nodejs/nodejs.go

@@ -0,0 +1,199 @@
+// Copyright 2016 clair authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package nodejs
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"strings"
+
+	"github.com/coreos/clair/database"
+	"github.com/coreos/clair/updater"
+	cerrors "github.com/coreos/clair/utils/errors"
+	"github.com/coreos/clair/utils/types"
+	"github.com/coreos/pkg/capnslog"
+)
+
+const (
+	url                  = "https://api.nodesecurity.io/advisories"
+	cveURLPrefix         = "http://cve.mitre.org/cgi-bin/cvename.cgi?name="
+	updaterFlag          = "nodejsUpdater"
+	defaultNodejsVersion = "all"
+	//FIXME: Add a suffix when an advisory is fixed `after` a certain version.
+	defaultVersionSuffix = "-1"
+)
+
+var log = capnslog.NewPackageLogger("github.com/coreos/clair", "updater/fetchers/nodejs")
+
+type nodejsAdvisory struct {
+	ID                 int      `json:"id"`
+	CreatedAt          string   `json:"created_at"`
+	UpdatedAt          string   `json:"updated_at"`
+	PublishDate        string   `json:"publish_date"`
+	Title              string   `json:"title"`
+	Author             string   `json:"author"`
+	ModuleName         string   `json:"module_name"`
+	CVES               []string `json:"cves"`
+	VulnerableVersions string   `json:"vulnerable_versions"`
+	PatchedVersions    string   `json:"patched_versions"`
+	Slug               string   `json:"slug"`
+	Overview           string   `json:"overview"`
+	Recommandation     string   `json:"recommandation"`
+	References         string   `json:"references"`
+	LegacySlug         string   `json:"legacy_slug"`
+	AllowedScopes      []string `json:"allowed_scopes"`
+	CvesVector         string   `json:"cves_vector"`
+	CvssScore          float32  `json:"cvss_score"`
+}
+
+type nodejsAdvisories struct {
+	Total   int              `json:"total"`
+	Count   int              `json:"count"`
+	Offset  int              `json:"offset"`
+	Results []nodejsAdvisory `json:"results"`
+}
+
+// NodejsFetcher implements updater.Fetcher for the Node Security Project
+// (https://nodesecurity.io).
+type NodejsFetcher struct{}
+
+func init() {
+	updater.RegisterFetcher("nodejs", &NodejsFetcher{})
+}
+
+// FetchUpdate fetches vulnerability updates from the Node Security Project.
+func (fetcher *NodejsFetcher) FetchUpdate(datastore database.Datastore) (resp updater.FetcherResponse, err error) {
+	log.Info("fetching Nodejs vulnerabilities")
+
+	// Download JSON.
+	r, err := http.Get(url)
+	if err != nil {
+		log.Errorf("could not download Nodejs's update: %s", err)
+		return resp, cerrors.ErrCouldNotDownload
+	}
+
+	defer r.Body.Close()
+
+	// Get the latest date of the latest update's JSON data
+	latestUpdate, err := datastore.GetKeyValue(updaterFlag)
+
+	fmt.Println("get lasted update <", latestUpdate, ">")
+	if err != nil {
+		return resp, err
+	}
+
+	// Unmarshal JSON.
+	var advisories nodejsAdvisories
+	err = json.NewDecoder(r.Body).Decode(&advisories)
+	fmt.Println("get lasted update len <", len(advisories.Results), ">")
+	if err != nil {
+		log.Errorf("could not unmarshal Nodejs's JSON: %s", err)
+		return resp, cerrors.ErrCouldNotParse
+	}
+
+	resp.Vulnerabilities, resp.FlagValue = parseNodejsAdvisories(advisories.Results, latestUpdate)
+	resp.FlagName = updaterFlag
+	fmt.Println(resp.FlagValue, resp.FlagName, len(resp.Vulnerabilities))
+
+	return resp, nil
+}
+
+func parseNodejsAdvisories(advisories []nodejsAdvisory, latestUpdate string) (vulnerabilities []database.Vulnerability, newUpdated string) {
+	mvulnerabilities := make(map[string]*database.Vulnerability)
+
+	for _, advisory := range advisories {
+		for _, vulnName := range advisory.CVES {
+			if latestUpdate >= advisory.UpdatedAt {
+				break
+			}
+			if advisory.UpdatedAt > newUpdated {
+				newUpdated = advisory.UpdatedAt
+			}
+			// Get or create the vulnerability.
+			vulnerability, vulnerabilityAlreadyExists := mvulnerabilities[vulnName]
+			if !vulnerabilityAlreadyExists {
+				vulnerability = &database.Vulnerability{
+					Name:        vulnName,
+					Link:        cveURLPrefix + strings.TrimLeft(vulnName, "CVE-"),
+					Severity:    types.Unknown,
+					Description: advisory.Overview,
+				}
+			}
+
+			// Set the priority of the vulnerability.
+			// A vulnerability has one urgency per advisory it affects.
+			// The highest urgency should be the one set.
+			urgency := types.ScoreToPriority(advisory.CvssScore)
+			if urgency.Compare(vulnerability.Severity) > 0 {
+				vulnerability.Severity = urgency
+			}
+
+			// Create and add the feature version.
+			pkg := database.FeatureVersion{
+				Feature: database.Feature{
+					Name: advisory.ModuleName,
+					Namespace: database.Namespace{
+						Name: "nodejs:" + defaultNodejsVersion,
+					},
+				},
+			}
+			if version, err := getAdvisoryVersion(advisory.PatchedVersions); err == nil {
+				pkg.Version = version
+			}
+			vulnerability.FixedIn = append(vulnerability.FixedIn, pkg)
+
+			// Store the vulnerability.
+			mvulnerabilities[vulnName] = vulnerability
+		}
+	}
+
+	// Convert the vulnerabilities map to a slice
+	for _, v := range mvulnerabilities {
+		vulnerabilities = append(vulnerabilities, *v)
+	}
+
+	return
+}
+
+func getAdvisoryVersion(fullVersion string) (types.Version, error) {
+	fixedVersion := types.MinVersion
+	versions := strings.Split(fullVersion, "||")
+	// Pickup a max version, there might be a false alarm, but better than have a security risk
+	for _, version := range versions {
+		ovs := getOperVersions(version)
+		for _, ov := range ovs {
+			if ov.Oper == ">" {
+				curVersion := types.NewVersionUnsafe(ov.Version + defaultVersionSuffix)
+				if curVersion.Compare(fixedVersion) > 0 {
+					fixedVersion = curVersion
+				}
+			}
+			if ov.Oper == ">=" {
+				curVersion := types.NewVersionUnsafe(ov.Version)
+				if curVersion.Compare(fixedVersion) > 0 {
+					fixedVersion = curVersion
+				}
+			}
+		}
+	}
+	if fixedVersion != types.MinVersion {
+		return fixedVersion, nil
+	}
+	return types.MaxVersion, cerrors.ErrNotFound
+}
+
+// Clean deletes any allocated resources.
+func (fetcher *NodejsFetcher) Clean() {}

updater/fetchers/nodejs/nodejs_test.go

@@ -0,0 +1,102 @@
+// Copyright 2016 clair authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package nodejs
+
+import (
+	"encoding/json"
+	"os"
+	"path"
+	"runtime"
+	"testing"
+
+	"github.com/coreos/clair/database"
+	"github.com/coreos/clair/utils/types"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRHELParser(t *testing.T) {
+	_, filename, _, _ := runtime.Caller(0)
+	testFile, _ := os.Open(path.Join(path.Dir(filename)) + "/testdata/fetcher_nodejs_test.json")
+	defer testFile.Close()
+
+	var advisories nodejsAdvisories
+	json.NewDecoder(testFile).Decode(&advisories)
+	assert.Len(t, advisories.Results, 5)
+
+	vulnerabilities, lastUpdated := parseNodejsAdvisories(advisories.Results, "")
+	assert.Len(t, vulnerabilities, 3)
+	assert.Equal(t, "2016-04-28T16:50:25+00:00", lastUpdated)
+
+	for _, vulnerability := range vulnerabilities {
+		if vulnerability.Name == "CVE-2015-7294" {
+			assert.Equal(t, "http://cve.mitre.org/cgi-bin/cvename.cgi?name=2015-7294", vulnerability.Link)
+			assert.Equal(t, types.Medium, vulnerability.Severity)
+			assert.Equal(t, "ldapauth versions <= 2.2.4 are vulnerable to ldap injection through the username parameter.", vulnerability.Description)
+			expectedFeatureVersions := []database.FeatureVersion{
+				{
+					Feature: database.Feature{
+						Namespace: database.Namespace{Name: "nodejs:" + defaultNodejsVersion},
+						Name:      "ldapauth",
+					},
+					Version: types.NewVersionUnsafe("2.2.4" + defaultVersionSuffix),
+				},
+				{
+					Feature: database.Feature{
+						Namespace: database.Namespace{Name: "nodejs:" + defaultNodejsVersion},
+						Name:      "ldapauth-fork",
+					},
+					Version: types.NewVersionUnsafe("2.3.3"),
+				},
+			}
+			for _, expectedFeatureVersion := range expectedFeatureVersions {
+				assert.Contains(t, vulnerability.FixedIn, expectedFeatureVersion)
+			}
+		} else if vulnerability.Name == "CVE-2015-6584" {
+			assert.Equal(t, "http://cve.mitre.org/cgi-bin/cvename.cgi?name=2015-6584", vulnerability.Link)
+			assert.Equal(t, types.Medium, vulnerability.Severity)
+			assert.Equal(t, "Cross-site scripting (XSS) vulnerability in the DataTables plugin 1.10.8 and earlier for jQuery allows remote attackers to inject arbitrary web script or HTML via the scripts parameter to media/unit_testing/templates/6776.php.", vulnerability.Description)
+			expectedFeatureVersions := []database.FeatureVersion{
+				{
+					Feature: database.Feature{
+						Namespace: database.Namespace{Name: "nodejs:" + defaultNodejsVersion},
+						Name:      "datatables",
+					},
+					Version: types.NewVersionUnsafe("1.10.8" + defaultVersionSuffix),
+				},
+			}
+			for _, expectedFeatureVersion := range expectedFeatureVersions {
+				assert.Contains(t, vulnerability.FixedIn, expectedFeatureVersion)
+			}
+		} else if vulnerability.Name == "CVE-2015-2515" {
+			assert.Equal(t, "http://cve.mitre.org/cgi-bin/cvename.cgi?name=2015-2515", vulnerability.Link)
+			assert.Equal(t, types.Medium, vulnerability.Severity)
+			assert.Equal(t, "Specifically crafted long headers or uris can cause a minor denial of service when using hawk versions less than 4.1.1.\n\n\"The Regular expression Denial of Service (ReDoS) is a Denial of Service attack, that exploits the fact that most Regular Expression implementations may reach extreme situations that cause them to work very slowly (exponentially related to input size). An attacker can then cause a program using a Regular Expression to enter these extreme situations and then hang for a very long time.\"\n\nUpdates:\n- Updated to include fix in 3.1.3 ", vulnerability.Description)
+			expectedFeatureVersions := []database.FeatureVersion{
+				{
+					Feature: database.Feature{
+						Namespace: database.Namespace{Name: "nodejs:" + defaultNodejsVersion},
+						Name:      "hawk",
+					},
+					Version: types.NewVersionUnsafe("4.1.1"),
+				},
+			}
+			for _, expectedFeatureVersion := range expectedFeatureVersions {
+				assert.Contains(t, vulnerability.FixedIn, expectedFeatureVersion)
+			}
+		}
+	}
+
+	return
+}

updater/fetchers/nodejs/nodejs_version.go

@@ -0,0 +1,76 @@
+// Copyright 2016 clair authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package nodejs
+
+import (
+	"strings"
+	"unicode"
+)
+
+type operVersion struct {
+	Oper    string
+	Version string
+}
+
+type ovState string
+
+const (
+	ovStateInit    ovState = "init"
+	ovStateOper    ovState = "operation"
+	ovStateVersion ovState = "version"
+)
+
+func isOper(ch rune) bool {
+	return ch == '>' || ch == '<' || ch == '='
+}
+
+func getOperVersions(content string) (ovs []operVersion) {
+	state := ovStateInit
+	begin := 0
+	var ov operVersion
+	for i, ch := range content {
+		if unicode.IsSpace(ch) {
+			continue
+		}
+		switch state {
+		case ovStateInit:
+			if isOper(ch) {
+				state = ovStateOper
+				begin = i
+			} else {
+				return nil
+			}
+		case ovStateOper:
+			if !isOper(ch) {
+				state = ovStateVersion
+				ov.Oper = strings.TrimSpace(content[begin:i])
+				begin = i
+			}
+		case ovStateVersion:
+			if isOper(ch) {
+				state = ovStateOper
+				ov.Version = strings.TrimSpace(content[begin:i])
+				ovs = append(ovs, ov)
+				begin = i
+			}
+		}
+	}
+	if state == ovStateVersion {
+		ov.Version = strings.TrimSpace(content[begin:len(content)])
+		ovs = append(ovs, ov)
+	}
+
+	return
+}