Skip to content

Commit

Permalink
feat(db) no progress when --log-json option (#394)
Browse files Browse the repository at this point in the history
  • Loading branch information
sadayuki-matsuno authored Jul 4, 2024
1 parent dbc1681 commit 400255c
Show file tree
Hide file tree
Showing 2 changed files with 50 additions and 8 deletions.
29 changes: 25 additions & 4 deletions db/rdb.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"encoding/json"
"errors"
"fmt"
"io"
"log"
"os"
"time"
Expand Down Expand Up @@ -630,7 +631,12 @@ func deleteJvn(tx *gorm.DB) error {
}

func insertJvn(tx *gorm.DB, cves []models.Jvn, batchSize int) error {
bar := pb.StartNew(len(cves))
bar := pb.StartNew(len(cves)).SetWriter(func() io.Writer {
if viper.GetBool("log-json") {
return io.Discard
}
return os.Stderr
}())
for _, cve := range cves {
if err := tx.Omit("Cpes").Create(&cve).Error; err != nil {
return xerrors.Errorf("Failed to insert. err: %w", err)
Expand Down Expand Up @@ -729,7 +735,12 @@ func deleteNvd(tx *gorm.DB) error {
}

func insertNvd(tx *gorm.DB, cves []models.Nvd, batchSize int) error {
bar := pb.StartNew(len(cves))
bar := pb.StartNew(len(cves)).SetWriter(func() io.Writer {
if viper.GetBool("log-json") {
return io.Discard
}
return os.Stderr
}())
for _, cve := range cves {
if err := tx.Omit("Cpes").Create(&cve).Error; err != nil {
return xerrors.Errorf("Failed to insert. err: %w", err)
Expand Down Expand Up @@ -784,7 +795,12 @@ func (r *RDBDriver) InsertFortinet(advs []models.Fortinet) (err error) {
}

logger.Infof("Inserting fetched %d CVEs...", len(advs))
bar := pb.StartNew(len(advs))
bar := pb.StartNew(len(advs)).SetWriter(func() io.Writer {
if viper.GetBool("log-json") {
return io.Discard
}
return os.Stderr
}())
for _, adv := range advs {
if err := tx.Omit("Cpes").Create(&adv).Error; err != nil {
return xerrors.Errorf("Failed to insert. err: %w", err)
Expand Down Expand Up @@ -932,7 +948,12 @@ func deleteMitre(tx *gorm.DB) error {
}

func insertMitre(tx *gorm.DB, cves []models.Mitre, _ int) error {
bar := pb.StartNew(len(cves))
bar := pb.StartNew(len(cves)).SetWriter(func() io.Writer {
if viper.GetBool("log-json") {
return io.Discard
}
return os.Stderr
}())
for _, cve := range cves {
if err := tx.Create(&cve).Error; err != nil {
return xerrors.Errorf("Failed to insert. err: %w", err)
Expand Down
29 changes: 25 additions & 4 deletions db/redis.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"encoding/json"
"errors"
"fmt"
"io"
"os"
"strconv"
"strings"
Expand Down Expand Up @@ -443,7 +444,12 @@ func (r *RedisDriver) InsertJvn(years []string) error {
delete(uniqCves, year)

log.Infof("Inserting fetched CVEs(%s)...", year)
bar := pb.StartNew(len(cves))
bar := pb.StartNew(len(cves)).SetWriter(func() io.Writer {
if viper.GetBool("log-json") {
return io.Discard
}
return os.Stderr
}())
for idx := range chunkSlice(len(cves), batchSize) {
pipe := r.conn.Pipeline()
for _, cve := range cves[idx.From:idx.To] {
Expand Down Expand Up @@ -575,7 +581,12 @@ func (r *RedisDriver) InsertNvd(years []string) error {
}

log.Infof("Inserting fetched CVEs(%s)...", year)
bar := pb.StartNew(len(cves))
bar := pb.StartNew(len(cves)).SetWriter(func() io.Writer {
if viper.GetBool("log-json") {
return io.Discard
}
return os.Stderr
}())
for idx := range chunkSlice(len(cves), batchSize) {
pipe := r.conn.Pipeline()
for _, cve := range cves[idx.From:idx.To] {
Expand Down Expand Up @@ -681,7 +692,12 @@ func (r *RedisDriver) InsertFortinet(advs []models.Fortinet) error {
}

log.Infof("Inserting fetched %d CVEs...", len(advs))
bar := pb.StartNew(len(advs))
bar := pb.StartNew(len(advs)).SetWriter(func() io.Writer {
if viper.GetBool("log-json") {
return io.Discard
}
return os.Stderr
}())
for idx := range chunkSlice(len(advs), batchSize) {
pipe := r.conn.Pipeline()
for _, adv := range advs[idx.From:idx.To] {
Expand Down Expand Up @@ -810,7 +826,12 @@ func (r *RedisDriver) InsertMitre(years []string) error {
}

log.Infof("Inserting fetched CVEs(%s)...", year)
bar := pb.StartNew(len(cves))
bar := pb.StartNew(len(cves)).SetWriter(func() io.Writer {
if viper.GetBool("log-json") {
return io.Discard
}
return os.Stderr
}())
for idx := range chunkSlice(len(cves), batchSize) {
pipe := r.conn.Pipeline()
for _, cve := range cves[idx.From:idx.To] {
Expand Down

0 comments on commit 400255c

Please sign in to comment.