implement nodejs fetcher

Signed-off-by: liang chenye <liangchenye@huawei.com>

updater/fetchers/node/node.go

@@ -0,0 +1,212 @@
+// Copyright 2016 clair authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package node
+
+import (
+	"encoding/json"
+	"net/http"
+	"strings"
+
+	"github.com/coreos/clair/database"
+	"github.com/coreos/clair/updater"
+	cerrors "github.com/coreos/clair/utils/errors"
+	"github.com/coreos/clair/utils/types"
+	"github.com/coreos/pkg/capnslog"
+)
+
+const (
+	url               = "https://api.nodesecurity.io/advisories"
+	cveURLPrefix      = "http://cve.mitre.org/cgi-bin/cvename.cgi?name="
+	updaterFlag       = "nodeUpdater"
+	defaultNPMVersion = "all"
+	//FIXME: When an advisory is fixed after a certain version,
+	//  current Clair could not differentiate it, so add an suffix.
+	defaultVersionSuffix = "-1"
+)
+
+var log = capnslog.NewPackageLogger("github.com/coreos/clair", "updater/fetchers/node")
+
+type nodeCVE struct {
+	ID string
+}
+type nodeAdvisory struct {
+	ID                  int
+	Created_at          string
+	Updated_at          string
+	Publish_date        string
+	Title               string
+	Author              string
+	Module_name         string
+	CVES                []string
+	Vulnerable_versions string
+	Patched_versions    string
+	Slug                string
+	Overview            string
+	Recommandation      string
+	References          string
+	Legacy_slug         string
+	Allowed_scopes      []string
+	CVEs_vector         string
+	Cvss_score          float32
+}
+
+type nodeAdvisories struct {
+	Total   int
+	Count   int
+	Offset  int
+	Results []nodeAdvisory
+}
+
+// NodeFetcher implements updater.Fetcher for the Node Security Tracker
+// (https://nodesecurity.io).
+type NodeFetcher struct{}
+
+func init() {
+	updater.RegisterFetcher("node", &NodeFetcher{})
+}
+
+// FetchUpdate fetches vulnerability updates from the Node Security Tracker.
+func (fetcher *NodeFetcher) FetchUpdate(datastore database.Datastore) (resp updater.FetcherResponse, err error) {
+	log.Info("fetching Node vulnerabilities")
+
+	// Download JSON.
+	r, err := http.Get(url)
+	if err != nil {
+		log.Errorf("could not download Node's update: %s", err)
+		return resp, cerrors.ErrCouldNotDownload
+	}
+
+	// Defer the addition of flag information to the response.
+	defer func() {
+		resp.FlagName = updaterFlag
+	}()
+
+	//	latestUpdate, err := datastore.GetKeyValue(updaterFlag)
+	latestUpdate := ""
+	// Unmarshal JSON.
+	var advisories nodeAdvisories
+	err = json.NewDecoder(r.Body).Decode(&advisories)
+	if err != nil {
+		log.Errorf("could not unmarshal Node's JSON: %s", err)
+		return resp, cerrors.ErrCouldNotParse
+	}
+
+	resp.Vulnerabilities, resp.FlagValue = parseNPMAdvisories(advisories.Results, latestUpdate)
+
+	return resp, nil
+}
+
+func parseNPMAdvisories(advisories []nodeAdvisory, latestUpdate string) (vulnerabilities []database.Vulnerability, newUpdated string) {
+	mvulnerabilities := make(map[string]*database.Vulnerability)
+
+	for _, advisory := range advisories {
+		for _, vulnName := range advisory.CVES {
+			if latestUpdate >= advisory.Updated_at {
+				break
+			}
+			if advisory.Updated_at > newUpdated {
+				newUpdated = advisory.Updated_at
+			}
+			// Get or create the vulnerability.
+			vulnerability, vulnerabilityAlreadyExists := mvulnerabilities[vulnName]
+			if !vulnerabilityAlreadyExists {
+				vulnerability = &database.Vulnerability{
+					Name:        vulnName,
+					Link:        cveURLPrefix + strings.TrimLeft(vulnName, "CVE-"),
+					Severity:    types.Unknown,
+					Description: advisory.Overview,
+				}
+			}
+
+			// Set the priority of the vulnerability.
+			// In the JSON, a vulnerability has one urgency per advisory it affects.
+			// The highest urgency should be the one set.
+			urgency := scoreToSeverity(advisory.Cvss_score)
+			if urgency.Compare(vulnerability.Severity) > 0 {
+				vulnerability.Severity = urgency
+			}
+
+			// Create and add the feature version.
+			pkg := database.FeatureVersion{
+				Feature: database.Feature{
+					Name: advisory.Module_name,
+					Namespace: database.Namespace{
+						Name: "node:" + defaultNPMVersion,
+					},
+				},
+			}
+			if version, err := getAdvisoryVersion(advisory.Patched_versions); err == nil {
+				pkg.Version = version
+			}
+			vulnerability.FixedIn = append(vulnerability.FixedIn, pkg)
+
+			// Store the vulnerability.
+			mvulnerabilities[vulnName] = vulnerability
+		}
+	}
+
+	// Convert the vulnerabilities map to a slice
+	for _, v := range mvulnerabilities {
+		vulnerabilities = append(vulnerabilities, *v)
+	}
+
+	return
+}
+
+func scoreToSeverity(score float32) types.Priority {
+	if score < 0.1 {
+		return types.Negligible
+	} else if score <= 3.9 {
+		return types.Low
+	} else if score <= 6.9 {
+		return types.Medium
+	} else if score <= 8.9 {
+		return types.High
+	} else if score <= 10.0 {
+		return types.Critical
+	}
+
+	return types.Unknown
+}
+
+func getAdvisoryVersion(fullVersion string) (types.Version, error) {
+	fixedVersion := types.MinVersion
+	versions := strings.Split(fullVersion, "||")
+	// Pickup a max version, there might be a false alarm, but better than have security risk
+	for _, version := range versions {
+		ovs := getOperVersions(version)
+		for _, ov := range ovs {
+			if ov.Oper == ">" {
+				curVersion := types.NewVersionUnsafe(ov.Version + defaultVersionSuffix)
+				if curVersion.Compare(fixedVersion) > 0 {
+					fixedVersion = curVersion
+				}
+			}
+			if ov.Oper == ">=" {
+				curVersion := types.NewVersionUnsafe(ov.Version)
+				if curVersion.Compare(fixedVersion) > 0 {
+					fixedVersion = curVersion
+				}
+			}
+		}
+	}
+	if fixedVersion != types.MinVersion {
+		return fixedVersion, nil
+	}
+	return types.MaxVersion, cerrors.ErrNotFound
+}
+
+// Clean deletes any allocated resources.
+func (fetcher *NodeFetcher) Clean() {}

updater/fetchers/node/node_test.go

@@ -0,0 +1,101 @@
+// Copyright 2016 clair authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package node
+
+import (
+	"encoding/json"
+	"os"
+	"path"
+	"runtime"
+	"testing"
+
+	"github.com/coreos/clair/database"
+	"github.com/coreos/clair/utils/types"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRHELParser(t *testing.T) {
+	_, filename, _, _ := runtime.Caller(0)
+	testFile, _ := os.Open(path.Join(path.Dir(filename)) + "/testdata/fetcher_node_test.json")
+
+	var advisories nodeAdvisories
+	json.NewDecoder(testFile).Decode(&advisories)
+	assert.Len(t, advisories.Results, 5)
+
+	vulnerabilities, lastUpdated := parseNPMAdvisories(advisories.Results, "")
+	assert.Len(t, vulnerabilities, 3)
+	assert.Equal(t, "2016-04-28T16:50:25+00:00", lastUpdated)
+
+	for _, vulnerability := range vulnerabilities {
+		if vulnerability.Name == "CVE-2015-7294" {
+			assert.Equal(t, "http://cve.mitre.org/cgi-bin/cvename.cgi?name=2015-7294", vulnerability.Link)
+			assert.Equal(t, types.Medium, vulnerability.Severity)
+			assert.Equal(t, "ldapauth versions <= 2.2.4 are vulnerable to ldap injection through the username parameter.", vulnerability.Description)
+			expectedFeatureVersions := []database.FeatureVersion{
+				{
+					Feature: database.Feature{
+						Namespace: database.Namespace{Name: "node:" + defaultNPMVersion},
+						Name:      "ldapauth",
+					},
+					Version: types.NewVersionUnsafe("2.2.4" + defaultVersionSuffix),
+				},
+				{
+					Feature: database.Feature{
+						Namespace: database.Namespace{Name: "node:" + defaultNPMVersion},
+						Name:      "ldapauth-fork",
+					},
+					Version: types.NewVersionUnsafe("2.3.3"),
+				},
+			}
+			for _, expectedFeatureVersion := range expectedFeatureVersions {
+				assert.Contains(t, vulnerability.FixedIn, expectedFeatureVersion)
+			}
+		} else if vulnerability.Name == "CVE-2015-6584" {
+			assert.Equal(t, "http://cve.mitre.org/cgi-bin/cvename.cgi?name=2015-6584", vulnerability.Link)
+			assert.Equal(t, types.Medium, vulnerability.Severity)
+			assert.Equal(t, "Cross-site scripting (XSS) vulnerability in the DataTables plugin 1.10.8 and earlier for jQuery allows remote attackers to inject arbitrary web script or HTML via the scripts parameter to media/unit_testing/templates/6776.php.", vulnerability.Description)
+			expectedFeatureVersions := []database.FeatureVersion{
+				{
+					Feature: database.Feature{
+						Namespace: database.Namespace{Name: "node:" + defaultNPMVersion},
+						Name:      "datatables",
+					},
+					Version: types.NewVersionUnsafe("1.10.8" + defaultVersionSuffix),
+				},
+			}
+			for _, expectedFeatureVersion := range expectedFeatureVersions {
+				assert.Contains(t, vulnerability.FixedIn, expectedFeatureVersion)
+			}
+		} else if vulnerability.Name == "CVE-2015-2515" {
+			assert.Equal(t, "http://cve.mitre.org/cgi-bin/cvename.cgi?name=2015-2515", vulnerability.Link)
+			assert.Equal(t, types.Medium, vulnerability.Severity)
+			assert.Equal(t, "Specifically crafted long headers or uris can cause a minor denial of service when using hawk versions less than 4.1.1.\n\n\"The Regular expression Denial of Service (ReDoS) is a Denial of Service attack, that exploits the fact that most Regular Expression implementations may reach extreme situations that cause them to work very slowly (exponentially related to input size). An attacker can then cause a program using a Regular Expression to enter these extreme situations and then hang for a very long time.\"\n\nUpdates:\n- Updated to include fix in 3.1.3 ", vulnerability.Description)
+			expectedFeatureVersions := []database.FeatureVersion{
+				{
+					Feature: database.Feature{
+						Namespace: database.Namespace{Name: "node:" + defaultNPMVersion},
+						Name:      "hawk",
+					},
+					Version: types.NewVersionUnsafe("4.1.1"),
+				},
+			}
+			for _, expectedFeatureVersion := range expectedFeatureVersions {
+				assert.Contains(t, vulnerability.FixedIn, expectedFeatureVersion)
+			}
+		}
+	}
+
+	return
+}

updater/fetchers/node/node_version.go

@@ -0,0 +1,76 @@
+// Copyright 2016 clair authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package node
+
+import (
+	"strings"
+	"unicode"
+)
+
+type operVersion struct {
+	Oper    string
+	Version string
+}
+
+type ovState string
+
+const (
+	ovStateInit    ovState = "init"
+	ovStateOper    ovState = "operation"
+	ovStateVersion ovState = "version"
+)
+
+func isOper(ch rune) bool {
+	return ch == '>' || ch == '<' || ch == '='
+}
+
+func getOperVersions(content string) (ovs []operVersion) {
+	state := ovStateInit
+	begin := 0
+	var ov operVersion
+	for i, ch := range content {
+		if unicode.IsSpace(ch) {
+			continue
+		}
+		switch state {
+		case ovStateInit:
+			if isOper(ch) {
+				state = ovStateOper
+				begin = i
+			} else {
+				return nil
+			}
+		case ovStateOper:
+			if !isOper(ch) {
+				state = ovStateVersion
+				ov.Oper = strings.TrimSpace(content[begin:i])
+				begin = i
+			}
+		case ovStateVersion:
+			if isOper(ch) {
+				state = ovStateOper
+				ov.Version = strings.TrimSpace(content[begin:i])
+				ovs = append(ovs, ov)
+				begin = i
+			}
+		}
+	}
+	if state == ovStateVersion {
+		ov.Version = strings.TrimSpace(content[begin:len(content)])
+		ovs = append(ovs, ov)
+	}
+
+	return
+}