Clair Logic, Extensions: updated mock tests, extensions, basic logic

Main Clair logic is changed in worker, updater, notifier for better adapting ancestry schema. Extensions are updated with the new model and feature lister and namespace detector drivers are able to specify the specific listers and detectors used to process layer's content. InRange and GetFixedIn interfaces are added to Version format for adapting ranged affected features and next available fixed in in the future. Tests for worker, updater and extensions are fixed.
quay · Aug 10, 2017 · fb32dcf · fb32dcf
1 parent 57b146d
commit fb32dcf
Show file tree

Hide file tree

Showing 36 changed files with 2,593 additions and 812 deletions.
diff --git a/cmd/clair/config.go b/cmd/clair/config.go
@@ -20,13 +20,17 @@ import (
 	"os"
 	"time"
 
+	"github.com/fernet/fernet-go"
+	log "github.com/sirupsen/logrus"
 	"gopkg.in/yaml.v2"
 
 	"github.com/coreos/clair"
 	"github.com/coreos/clair/api"
 	"github.com/coreos/clair/database"
+	"github.com/coreos/clair/ext/featurefmt"
+	"github.com/coreos/clair/ext/featurens"
 	"github.com/coreos/clair/ext/notification"
-	"github.com/fernet/fernet-go"
+	"github.com/coreos/clair/ext/vulnsrc"
 )
 
 // ErrDatasourceNotLoaded is returned when the datasource variable in the
@@ -43,6 +47,7 @@ type File struct {
 type Config struct {
 	Database database.RegistrableComponentConfig
 	Updater  *clair.UpdaterConfig
+	Worker   *clair.WorkerConfig
 	Notifier *notification.Config
 	API      *api.Config
 }
@@ -54,12 +59,16 @@ func DefaultConfig() Config {
 			Type: "pgsql",
 		},
 		Updater: &clair.UpdaterConfig{
-			Interval: 1 * time.Hour,
+			EnabledUpdaters: vulnsrc.ListUpdaters(),
+			Interval:        1 * time.Hour,
+		},
+		Worker: &clair.WorkerConfig{
+			EnabledDetectors: featurens.ListDetectors(),
+			EnabledListers:   featurefmt.ListListers(),
 		},
 		API: &api.Config{
-			Port:       6060,
 			HealthPort: 6061,
-			GrpcPort:   6070,
+			GrpcPort:   6060,
 			Timeout:    900 * time.Second,
 		},
 		Notifier: &notification.Config{
@@ -97,14 +106,15 @@ func LoadConfig(path string) (config *Config, err error) {
 	config = &cfgFile.Clair
 
 	// Generate a pagination key if none is provided.
-	if config.API.PaginationKey == "" {
+	if v, ok := config.Database.Options["paginationkey"]; !ok || v == nil || v.(string) == "" {
+		log.Warn("pagination key is empty, generating...")
 		var key fernet.Key
 		if err = key.Generate(); err != nil {
 			return
 		}
-		config.API.PaginationKey = key.Encode()
+		config.Database.Options["paginationkey"] = key.Encode()
 	} else {
-		_, err = fernet.DecodeKey(config.API.PaginationKey)
+		_, err = fernet.DecodeKey(config.Database.Options["paginationkey"].(string))
 		if err != nil {
 			err = errors.New("Invalid Pagination key; must be 32-bit URL-safe base64")
 			return

diff --git a/cmd/clair/main.go b/cmd/clair/main.go
@@ -30,9 +30,13 @@ import (
 	"github.com/coreos/clair"
 	"github.com/coreos/clair/api"
 	"github.com/coreos/clair/database"
+	"github.com/coreos/clair/ext/featurefmt"
+	"github.com/coreos/clair/ext/featurens"
 	"github.com/coreos/clair/ext/imagefmt"
+	"github.com/coreos/clair/ext/vulnsrc"
 	"github.com/coreos/clair/pkg/formatter"
 	"github.com/coreos/clair/pkg/stopper"
+	"github.com/coreos/clair/pkg/strutil"
 
 	// Register database driver.
 	_ "github.com/coreos/clair/database/pgsql"
@@ -85,6 +89,43 @@ func stopCPUProfiling(f *os.File) {
 	log.Info("stopped CPU profiling")
 }
 
+func configClairVersion(config *Config) {
+	listers := featurefmt.ListListers()
+	detectors := featurens.ListDetectors()
+	updaters := vulnsrc.ListUpdaters()
+
+	log.WithFields(log.Fields{
+		"Listers":   strings.Join(listers, ","),
+		"Detectors": strings.Join(detectors, ","),
+		"Updaters":  strings.Join(updaters, ","),
+	}).Info("Clair registered components")
+
+	unregDetectors := strutil.CompareStringLists(config.Worker.EnabledDetectors, detectors)
+	unregListers := strutil.CompareStringLists(config.Worker.EnabledListers, listers)
+	unregUpdaters := strutil.CompareStringLists(config.Updater.EnabledUpdaters, updaters)
+	if len(unregDetectors) != 0 || len(unregListers) != 0 || len(unregUpdaters) != 0 {
+		log.WithFields(log.Fields{
+			"Unknown Detectors":   strings.Join(unregDetectors, ","),
+			"Unknown Listers":     strings.Join(unregListers, ","),
+			"Unknown Updaters":    strings.Join(unregUpdaters, ","),
+			"Available Listers":   strings.Join(featurefmt.ListListers(), ","),
+			"Available Detectors": strings.Join(featurens.ListDetectors(), ","),
+			"Available Updaters":  strings.Join(vulnsrc.ListUpdaters(), ","),
+		}).Fatal("Unknown or unregistered components are configured")
+	}
+
+	// verify the user specified detectors/listers/updaters are implemented. If
+	// some are not registered, it logs warning and won't use the unregistered
+	// extensions.
+
+	clair.Processors = database.Processors{
+		Detectors: strutil.CompareStringListsInBoth(config.Worker.EnabledDetectors, detectors),
+		Listers:   strutil.CompareStringListsInBoth(config.Worker.EnabledListers, listers),
+	}
+
+	clair.EnabledUpdaters = strutil.CompareStringListsInBoth(config.Updater.EnabledUpdaters, updaters)
+}
+
 // Boot starts Clair instance with the provided config.
 func Boot(config *Config) {
 	rand.Seed(time.Now().UnixNano())
@@ -102,9 +143,8 @@ func Boot(config *Config) {
 	go clair.RunNotifier(config.Notifier, db, st)
 
 	// Start API
-	st.Begin()
-	go api.Run(config.API, db, st)
 	go api.RunV2(config.API, db)
+
 	st.Begin()
 	go api.RunHealth(config.API, db, st)
 
@@ -135,19 +175,17 @@ func main() {
 		}
 	}
 
-	// Load configuration
-	config, err := LoadConfig(*flagConfigPath)
-	if err != nil {
-		log.WithError(err).Fatal("failed to load configuration")
-	}
-
 	// Initialize logging system
-
 	logLevel, err := log.ParseLevel(strings.ToUpper(*flagLogLevel))
 	log.SetLevel(logLevel)
 	log.SetOutput(os.Stdout)
 	log.SetFormatter(&formatter.JSONExtendedFormatter{ShowLn: true})
 
+	config, err := LoadConfig(*flagConfigPath)
+	if err != nil {
+		log.WithError(err).Fatal("failed to load configuration")
+	}
+
 	// Enable CPU Profiling if specified
 	if *flagCPUProfilePath != "" {
 		defer stopCPUProfiling(startCPUProfiling(*flagCPUProfilePath))
@@ -159,5 +197,8 @@ func main() {
 		imagefmt.SetInsecureTLS(*flagInsecureTLS)
 	}
 
+	// configure updater and worker
+	configClairVersion(config)
+
 	Boot(config)
 }
diff --git a/config.example.yaml b/config.example.yaml
@@ -25,23 +25,22 @@ clair:
       # Number of elements kept in the cache
       # Values unlikely to change (e.g. namespaces) are cached in order to save prevent needless roundtrips to the database.
       cachesize: 16384
+      # 32-bit URL-safe base64 key used to encrypt pagination tokens
+      # If one is not provided, it will be generated.
+      # Multiple clair instances in the same cluster need the same value.
+      paginationkey: 
 
   api:
-    # API server port
-    port: 6060
-    grpcPort: 6070
+    # v2 grpc/RESTful API server port
+    grpcport : 6060
+
     # Health server port
     # This is an unencrypted endpoint useful for load balancers to check to healthiness of the clair server.
     healthport: 6061
 
     # Deadline before an API request will respond with a 503
     timeout: 900s
 
-    # 32-bit URL-safe base64 key used to encrypt pagination tokens
-    # If one is not provided, it will be generated.
-    # Multiple clair instances in the same cluster need the same value.
-    paginationkey:
-
     # Optional PKI configuration
     # If you want to easily generate client certificates and CAs, try the following projects:
     # https://github.com/coreos/etcd-ca
@@ -51,10 +50,29 @@ clair:
     keyfile:
     certfile:
 
+  worker:
+    namespace_detectors:
+      - os-release
+      - lsb-release
+      - apt-sources
+      - alpine-release
+      - redhat-release
+
+    feature_listers:
+      - apk
+      - dpkg
+      - rpm
+
   updater:
     # Frequency the database will be updated with vulnerabilities from the default data sources
     # The value 0 disables the updater entirely.
     interval: 2h
+    enabledupdaters: 
+      - debian
+      - ubuntu
+      - rhel
+      - oracle
+      - alpine
 
   notifier:
     # Number of attempts before the notification is marked as failed to be sent
@@ -72,9 +90,9 @@ clair:
       # https://github.com/cloudflare/cfssl
       # https://github.com/coreos/etcd-ca
       servername:
-      cafile:
-      keyfile:
-      certfile:
+      cafile: 
+      keyfile: 
+      certfile: 
 
       # Optional HTTP Proxy: must be a valid URL (including the scheme).
       proxy: