implement nodejs fetcher

Signed-off-by: liang chenye <liangchenye@huawei.com>

updater/fetchers/npm/npm.go

@@ -0,0 +1,190 @@
+// Copyright 2016 clair authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package npm
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/coreos/clair/database"
+	"github.com/coreos/clair/updater"
+	cerrors "github.com/coreos/clair/utils/errors"
+	"github.com/coreos/clair/utils/types"
+	"github.com/coreos/pkg/capnslog"
+)
+
+const (
+	url               = "https://api.nodesecurity.io/advisories"
+	cveURLPrefix      = "http://cve.mitre.org/cgi-bin/cvename.cgi?name="
+	updaterFlag       = "npmUpdater"
+	defaultNPMVersion = "all"
+)
+
+var log = capnslog.NewPackageLogger("github.com/coreos/clair", "updater/fetchers/npm")
+
+type npmCVE struct {
+	ID string
+}
+type npmAdvisory struct {
+	ID                  int
+	Created_at          string
+	Updated_at          string
+	Publish_date        string
+	Title               string
+	Author              string
+	Module_name         string
+	CVES                []string
+	Vulnerable_versions string
+	Patched_versions    string
+	Slug                string
+	Overview            string
+	Recommandation      string
+	References          string
+	Legacy_slug         string
+	Allowed_scopes      []string
+	CVEs_vector         string
+	Cvss_score          float32
+}
+
+type npmAdvisories struct {
+	Total   int
+	Count   int
+	Offset  int
+	Results []npmAdvisory
+}
+
+// NpmFetcher implements updater.Fetcher for the Npm Security Tracker
+// (https://nodesecurity.io).
+type NpmFetcher struct{}
+
+func init() {
+	updater.RegisterFetcher("npm", &NpmFetcher{})
+}
+
+// FetchUpdate fetches vulnerability updates from the Npm Security Tracker.
+func (fetcher *NpmFetcher) FetchUpdate(datastore database.Datastore) (resp updater.FetcherResponse, err error) {
+	log.Info("fetching Npm vulnerabilities")
+
+	// Download JSON.
+	r, err := http.Get(url)
+	if err != nil {
+		log.Errorf("could not download Npm's update: %s", err)
+		return resp, cerrors.ErrCouldNotDownload
+	}
+
+	// Defer the addition of flag information to the response.
+	defer func() {
+		resp.FlagName = updaterFlag
+	}()
+
+	//	latestUpdate, err := datastore.GetKeyValue(updaterFlag)
+	latestUpdate := ""
+	// Unmarshal JSON.
+	var advisories npmAdvisories
+	err = json.NewDecoder(r.Body).Decode(&advisories)
+	if err != nil {
+		log.Errorf("could not unmarshal Npm's JSON: %s", err)
+		return resp, cerrors.ErrCouldNotParse
+	}
+
+	resp.Vulnerabilities, resp.FlagValue = parseNPMAdvisories(advisories.Results, latestUpdate)
+
+	return resp, nil
+}
+
+func parseNPMAdvisories(advisories []npmAdvisory, latestUpdate string) (vulnerabilities []database.Vulnerability, newUpdated string) {
+	mvulnerabilities := make(map[string]*database.Vulnerability)
+
+	for _, advisory := range advisories {
+		for _, vulnName := range advisory.CVES {
+			if latestUpdate >= advisory.Updated_at {
+				break
+			}
+			if advisory.Updated_at > newUpdated {
+				newUpdated = advisory.Updated_at
+			}
+			// Get or create the vulnerability.
+			vulnerability, vulnerabilityAlreadyExists := mvulnerabilities[vulnName]
+			if !vulnerabilityAlreadyExists {
+				vulnerability = &database.Vulnerability{
+					Name:        vulnName,
+					Link:        cveURLPrefix + strings.TrimLeft(vulnName, "CVE-"),
+					Severity:    types.Unknown,
+					Description: advisory.Overview,
+				}
+			}
+
+			// Set the priority of the vulnerability.
+			// In the JSON, a vulnerability has one urgency per advisory it affects.
+			// The highest urgency should be the one set.
+			urgency := scoreToSeverity(advisory.Cvss_score)
+			if urgency.Compare(vulnerability.Severity) > 0 {
+				vulnerability.Severity = urgency
+			}
+
+			// Create and add the feature version.
+			pkg := database.FeatureVersion{
+				Feature: database.Feature{
+					Name: advisory.Module_name,
+					Namespace: database.Namespace{
+						Name: "npm:" + defaultNPMVersion,
+					},
+				},
+				//FIXME:
+				//Version: "version",
+			}
+			vulnerability.FixedIn = append(vulnerability.FixedIn, pkg)
+
+			// Store the vulnerability.
+			mvulnerabilities[vulnName] = vulnerability
+		}
+	}
+
+	// Convert the vulnerabilities map to a slice
+	for _, v := range mvulnerabilities {
+		vulnerabilities = append(vulnerabilities, *v)
+	}
+
+	return
+}
+
+func isUpdated(new string, old string) bool {
+	tn, en := time.Parse("2016-04-04T19:46:25+00:00", new)
+	to, eo := time.Parse("2016-04-04T19:46:25+00:00", old)
+	fmt.Println(tn, to, en, eo)
+	return true
+}
+
+func scoreToSeverity(score float32) types.Priority {
+	if score < 0.1 {
+		return types.Negligible
+	} else if score <= 3.9 {
+		return types.Low
+	} else if score <= 6.9 {
+		return types.Medium
+	} else if score <= 8.9 {
+		return types.High
+	} else if score <= 10.0 {
+		return types.Critical
+	}
+
+	return types.Unknown
+}
+
+// Clean deletes any allocated resources.
+func (fetcher *NpmFetcher) Clean() {}

updater/fetchers/npm/npm_test.go

@@ -0,0 +1,41 @@
+// Copyright 2016 clair authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package npm
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path"
+	"runtime"
+	"testing"
+)
+
+func TestRHELParser(t *testing.T) {
+	log.Info("fetching Npm vulnerabilities")
+
+	_, filename, _, _ := runtime.Caller(0)
+	testFile, _ := os.Open(path.Join(path.Dir(filename)) + "/testdata/fetcher_npm_test.json")
+	// Unmarshal JSON.
+	var advisories npmAdvisories
+	json.NewDecoder(testFile).Decode(&advisories)
+	//TODO: add more cases
+	vulnerabilities, nt := parseNPMAdvisories(advisories.Results, "")
+	fmt.Println(nt)
+	for _, vuln := range vulnerabilities {
+		fmt.Println(vuln.Name, vuln.Link)
+	}
+	return
+}

updater/fetchers/npm/testdata/fetcher_npm_test.json

@@ -0,0 +1,127 @@
+{
+	"Total": 5,
+	"Count": 5,
+	"Offset": 0,
+	"Results": [
+		{
+			"ID": 19,
+			"Created_at": "2015-10-17T19:41:46.382+00:00",
+			"Updated_at": "2016-04-25T15:31:07+00:00",
+			"Publish_date": "2015-09-18T19:30:10+00:00",
+			"Title": "LDAP Injection",
+			"Author": "David Black, Jerome Touffe-Blin",
+			"Module_name": "ldapauth",
+			"CVES": [
+				"CVE-2015-7294"
+			],
+			"Vulnerable_versions": "<=2.2.4",
+			"Patched_versions": "<=0.0.0",
+			"Slug": "ldapauth_ldap-injection",
+			"Overview": "ldapauth versions <= 2.2.4 are vulnerable to ldap injection through the username parameter.",
+			"Recommandation": "",
+			"References": "- http://www.openwall.com/lists/oss-security/2015/09/18/4",
+			"Legacy_slug": "ldapauth-ldap-injection",
+			"Allowed_scopes": [
+				"public",
+				"admin"
+			],
+			"CVEs_vector": ""
+		},
+		{
+			"ID": 5,
+			"Created_at": "2015-10-17T19:41:46.382+00:00",
+			"Updated_at": "2016-04-25T15:25:16+00:00",
+			"Publish_date": "2015-09-18T19:29:10+00:00",
+			"Title": "Cross-Site Scripting",
+			"Author": "Onur Yilmaz",
+			"Module_name": "datatables",
+			"CVES": [
+				"CVE-2015-6584"
+			],
+			"Vulnerable_versions": "<=1.10.8",
+			"Patched_versions": ">1.10.8",
+			"Slug": "datatables_cross-site-scripting",
+			"Overview": "Cross-site scripting (XSS) vulnerability in the DataTables plugin 1.10.8 and earlier for jQuery allows remote attackers to inject arbitrary web script or HTML via the scripts parameter to media/unit_testing/templates/6776.php.",
+			"Recommandation": "",
+			"References": "- http://www.securityfocus.com/archive/1/archive/1/536437/100/0/threaded\n- https://www.netsparker.com/cve-2015-6384-xss-vulnerability-identified-in-datatables/\n- https://github.com/DataTables/DataTables/issues/602\n- https://github.com/DataTables/DataTablesSrc/commit/ccf86dc5982bd8e16d",
+			"Legacy_slug": "datatables-CVE-2015-6584",
+			"Allowed_scopes": [
+				"public",
+				"admin"
+			],
+			"CVEs_vector": ""
+		},
+		{
+			"ID": 18,
+			"Created_at": "2015-10-17T19:41:46.382+00:00",
+			"Updated_at": "2016-04-28T16:50:25+00:00",
+			"Publish_date": "2015-09-18T19:29:10+00:00",
+			"Title": "LDAP Injection",
+			"Author": "Jerome Touffe-Blin",
+			"Module_name": "ldapauth-fork",
+			"CVES": [
+				"CVE-2015-7294"
+			],
+			"Vulnerable_versions": "< 2.3.3",
+			"Patched_versions": ">= 2.3.3",
+			"Slug": "ldapauth-fork_ldap-injection",
+			"Overview": "ldapauth-fork is a module forked from node-ldapauth and is used for ldap authentication  \nThe username parameter is not filtered as per [LDAP Escape Specifications](https://tools.ietf.org/search/rfc4515#section-3)  \nA malicious user is able to change their name to certain LDAP commands and run anything that they want.",
+			"Recommandation": "",
+			"References": "- https://github.com/vesse/node-ldapauth-fork/issues/21\n- https://github.com/vesse/node-ldapauth-fork/commit/3feea43e243698bcaeffa904a7324f4d96df60e4",
+			"Legacy_slug": "ldapauth-fork-ldap-injection",
+			"Allowed_scopes": [
+				"public",
+				"admin"
+			],
+			"CVEs_vector": ""
+		},
+		{
+			"ID": 99,
+			"Created_at": "2016-04-04T19:46:25+00:00",
+			"Updated_at": "2016-05-13T20:39:38+00:00",
+			"Publish_date": "2016-04-26T16:24:32+00:00",
+			"Title": "Insecure Defaults Allow MITM Over TLS",
+			"Author": "David Johansson",
+			"Module_name": "engine.io-client",
+			"CVES": [],
+			"Vulnerable_versions": "<= 1.6.8",
+			"Patched_versions": ">= 1.6.9",
+			"Slug": "engineio-client_tls-connections-over-websockets-vulnerable-to-mitm",
+			"Overview": "engine.io-client is the client for [engine.io](https://github.com/socketio/engine.io), the implementation of a transport-based cross-browser/cross-device bi-directional communication layer for Socket.IO.\n\nThe vulnerability is related to the way that node.js handles the `rejectUnauthorized` setting. If the value is something that evaluates to false, certificate verification will be disabled.\n\nThis is problematic as engine.io-client passes in an object for settings that includes the rejectUnauthorized property, whether it has been set or not. If the value has not been explicitly changed, it will be passed in as `null`, resulting in certificate verification being turned off:\n\n``` \n // line that causes bug\nthis.rejectUnauthorized = opts.rejectUnauthorized === undefined ? null : opts.rejectUnauthorized;\n ```",
+			"Recommandation": "",
+			"References": "- https://github.com/socketio/engine.io-client/commit/2c55b278a491bf45313ecc0825cf800e2f7ff5c1\n- https://www.cigital.com/blog/node-js-socket-io/",
+			"Legacy_slug": "",
+			"Allowed_scopes": [
+				"admin",
+				"public"
+			],
+			"CVEs_vector": "",
+			"Cvss_score": 6.8
+		},
+		{
+			"ID": 77,
+			"Created_at": "2016-01-19T21:50:30.175+00:00",
+			"Updated_at": "2016-04-21T00:16:58+00:00",
+			"Publish_date": "2016-01-19T21:51:35.396+00:00",
+			"Title": "Regular Expression Denial of Service",
+			"Author": "Adam Baldwin",
+			"Module_name": "hawk",
+			"CVES": [
+				"CVE-2016-2515"
+			],
+			"Vulnerable_versions": "< 3.1.3  || >= 4.0.0 < 4.1.1",
+			"Patched_versions": ">=3.1.3 < 4.0.0 || >=4.1.1",
+			"Slug": "hawk_regular-expression-denial-of-service",
+			"Overview": "Specifically crafted long headers or uris can cause a minor denial of service when using hawk versions less than 4.1.1.\n\n\"The Regular expression Denial of Service (ReDoS) is a Denial of Service attack, that exploits the fact that most Regular Expression implementations may reach extreme situations that cause them to work very slowly (exponentially related to input size). An attacker can then cause a program using a Regular Expression to enter these extreme situations and then hang for a very long time.\"\n\nUpdates:\n- Updated to include fix in 3.1.3 ",
+			"Recommandation": "",
+			"References": "- https://github.com/hueniverse/hawk/issues/168\n- https://www.owasp.org/index.php/Regular_expression_Denial_of_Service_-_ReDoS",
+			"Legacy_slug": "",
+			"Allowed_scopes": [
+				"admin",
+				"public"
+			],
+			"CVEs_vector": "",
+			"Cvss_score": 5.3
+		}
+	]
+}