fix(fetch): not to use htcat, use backoff retry, set user-agent as cu…

…rl (#221)

* fix(fetch): enable to set the number of htcat

* fix

* fix

* fix: random sleep while fetching NVD

* set timeout 180sec

* not to use htcat, backoff retry, set user-agent

fetcher/fetcher.go

@@ -7,29 +7,51 @@ import (
 	"io/ioutil"
 	"net/http"
 	"net/url"
+	"time"
 
-	"github.com/htcat/htcat"
+	"github.com/cenkalti/backoff"
 	"github.com/spf13/viper"
 	"github.com/vulsio/go-cve-dictionary/log"
 	"golang.org/x/xerrors"
 )
 
 //FetchFeedFile fetches vulnerability feed file concurrently
-func FetchFeedFile(urlstr string, gzip bool) ([]byte, error) {
+func FetchFeedFile(urlstr string, gzip bool) (body []byte, err error) {
 	log.Infof("Fetching... %s", urlstr)
-	body, err := fetchFile(urlstr, gzip, 5)
+
+	count, retryMax := 0, 20
+	f := func() (err error) {
+		if body, err = fetchFile(urlstr, gzip); err != nil {
+			count++
+			if count == retryMax {
+				return nil
+			}
+			return xerrors.Errorf("HTTP GET error, url: %s, err: %w", urlstr, err)
+		}
+		return nil
+	}
+	notify := func(err error, t time.Duration) {
+		log.Warnf("Failed to HTTP GET. retrying in %s seconds. err: %+v", t, err)
+	}
+	err = backoff.RetryNotify(f, backoff.NewExponentialBackOff(), notify)
 	if err != nil {
-		return nil, xerrors.Errorf("Failed to fetch file. err: %w", err)
+		return nil, xerrors.Errorf("Failed to fetch file: %w", err)
+	}
+
+	if count == retryMax {
+		return nil, xerrors.Errorf("Failed to fetch file. Retry count exceeded: %d", retryMax)
 	}
+
 	return body, nil
 }
 
-func fetchFile(urlstr string, isGzip bool, parallelism int) (body []byte, err error) {
+func fetchFile(urlstr string, isGzip bool) (body []byte, err error) {
 	var proxyURL *url.URL
 	httpClient := &http.Client{
 		Transport: &http.Transport{
 			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
 		},
+		Timeout: time.Duration(180 * time.Second),
 	}
 	if viper.GetString("http-proxy") != "" {
 		if proxyURL, err = url.Parse(viper.GetString("http-proxy")); err != nil {
@@ -42,27 +64,31 @@ func fetchFile(urlstr string, isGzip bool, parallelism int) (body []byte, err er
 			},
 		}
 	}
-
-	u, err := url.Parse(urlstr)
+	req, err := http.NewRequest("GET", urlstr, nil)
+	if err != nil {
+		return nil, xerrors.Errorf("Failed to new request. url: %s, err: %w", urlstr, err)
+	}
+	req.Header.Set("User-Agent", "curl/7.58.0")
+	resp, err := httpClient.Do(req)
 	if err != nil {
-		return nil, xerrors.Errorf("aborting: could not parse given URL: %w", err)
+		return nil, xerrors.Errorf("Failed to GET. url: %s, err: %w", urlstr, err)
 	}
-	buf := bytes.Buffer{}
-	htc := htcat.New(httpClient, u, parallelism)
-	if _, err := htc.WriteTo(&buf); err != nil {
-		return nil, xerrors.Errorf("aborting: could not write to output stream: %w", err)
+
+	defer resp.Body.Close()
+	buf, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, xerrors.Errorf("Failed to read body. url: %s, err: %w", urlstr, err)
 	}
 
 	if isGzip {
-		reader, err := gzip.NewReader(bytes.NewReader(buf.Bytes()))
+		reader, err := gzip.NewReader(bytes.NewReader(buf))
 		defer func() {
 			if reader != nil {
 				_ = reader.Close()
 			}
 		}()
 		if err != nil {
-			return nil, xerrors.Errorf(
-				"Failed to decompress NVD feedfile. url: %s, err: %w", urlstr, err)
+			return nil, xerrors.Errorf("Failed to decompress NVD feedfile. url: %s, err: %w", urlstr, err)
 		}
 
 		bytes, err := ioutil.ReadAll(reader)
@@ -72,5 +98,5 @@ func fetchFile(urlstr string, isGzip bool, parallelism int) (body []byte, err er
 		return bytes, nil
 	}
 
-	return buf.Bytes(), nil
+	return buf, nil
 }

fetcher/nvd/nvd.go

@@ -3,6 +3,7 @@ package nvd
 import (
 	"encoding/json"
 	"fmt"
+	"math/rand"
 	"runtime"
 	"strconv"
 	"strings"
@@ -46,6 +47,8 @@ func FetchConvert(uniqCve map[string]map[string]models.Nvd, years []string) erro
 			return xerrors.Errorf("Failed to convert. err: %w", err)
 		}
 		distributeCvesByYear(uniqCve, cves)
+
+		time.Sleep(time.Duration(rand.Intn(1000)) * time.Microsecond)
 	}
 	return nil
 }

go.mod

@@ -5,11 +5,10 @@ go 1.17
 require (
 	github.com/PuerkitoBio/goquery v1.6.1
 	github.com/andybalholm/cascadia v1.2.0 // indirect
+	github.com/cenkalti/backoff v2.2.1+incompatible
 	github.com/cheggaaa/pb/v3 v3.0.5
-	github.com/fatih/color v1.10.0
 	github.com/go-redis/redis/v8 v8.4.11
 	github.com/hashicorp/go-version v1.2.1
-	github.com/htcat/htcat v1.0.2
 	github.com/inconshreveable/log15 v0.0.0-20201112154412-8562bdadbbac
 	github.com/jackc/pgx/v4 v4.12.0 // indirect
 	github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 // indirect
@@ -39,6 +38,7 @@ require (
 	github.com/cespare/xxhash/v2 v2.1.1 // indirect
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
+	github.com/fatih/color v1.10.0 // indirect
 	github.com/fsnotify/fsnotify v1.4.9 // indirect
 	github.com/go-sql-driver/mysql v1.6.0 // indirect
 	github.com/go-stack/stack v1.8.0 // indirect

go.sum

@@ -73,6 +73,7 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
 github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM=
 github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ=
+github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=
 github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
@@ -257,8 +258,6 @@ github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0m
 github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
 github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
-github.com/htcat/htcat v1.0.2 h1:zro95dGwkKDeZOgq9ei+9szd5qurGxBGfHY8hRehA7k=
-github.com/htcat/htcat v1.0.2/go.mod h1:i8ViQbjSi2+lJzM6Lx20FIxHENCz6mzJglK3HH06W3s=
 github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=