Initial services refactoring of database.Datastore

This breaks out the following service interfaces:
 - locks.Service
 - keyvalue.Service
 - notifications.Service
 - vulnerabilities.Service

This also updates the Mock implementations along similar lines.

Make Travis work on my fork by rsyncing the build dir as coreos/clair

.travis.yml

@@ -14,6 +14,9 @@ install:
   - echo 'nop'
 
 script:
+  - mkdir -p  $HOME/gopath/src/github.com/coreos/clair/
+  - rsync -az ${TRAVIS_BUILD_DIR}/ $HOME/gopath/src/github.com/coreos/clair/
+  - cd $HOME/gopath/src/github.com/coreos/clair/
   - go test -v $(go list ./... | grep -v /vendor/)
 
 services:

Dockerfile.test

@@ -0,0 +1,36 @@
+# Copyright 2015 clair authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM golang:1.6
+
+MAINTAINER Quentin Machu <quentin.machu@coreos.com>
+
+RUN apt-get update && \
+    apt-get install -y bzr rpm xz-utils && \
+    apt-get autoremove -y && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* # 18MAR2016
+
+VOLUME /config
+
+EXPOSE 6060 6061
+
+ADD .   /go/src/github.com/coreos/clair/
+WORKDIR /go/src/github.com/coreos/clair/
+
+RUN go install -v github.com/coreos/clair/cmd/clair
+
+RUN go test $(go list ./... | grep -v /vendor/)
+
+ENTRYPOINT ["clair"]

api/context/context.go

@@ -24,7 +24,10 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 
 	"github.com/coreos/clair/config"
-	"github.com/coreos/clair/database"
+	"github.com/coreos/clair/services/keyvalue"
+	"github.com/coreos/clair/services/locks"
+	"github.com/coreos/clair/services/notifications"
+	"github.com/coreos/clair/services/vulnerabilities"
 	"github.com/coreos/clair/utils"
 )
 
@@ -59,6 +62,9 @@ func HTTPHandler(handler Handler, ctx *RouteContext) httprouter.Handle {
 }
 
 type RouteContext struct {
-	Store  database.Datastore
-	Config *config.APIConfig
+	LockService        locks.Service
+	KeyValueStore      keyvalue.Service
+	VulnerabilityStore vulnerabilities.Service
+	NotificationState  notifications.Service
+	Config             *config.APIConfig
 }

api/router.go

@@ -66,9 +66,18 @@ func getHealth(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx
 	header := w.Header()
 	header.Set("Server", "clair")
 
-	status := http.StatusInternalServerError
-	if ctx.Store.Ping() {
-		status = http.StatusOK
+	status := http.StatusOK
+	if !ctx.LockService.Ping() {
+		status = http.StatusInternalServerError
+	}
+	if !ctx.KeyValueStore.Ping() {
+		status = http.StatusInternalServerError
+	}
+	if !ctx.VulnerabilityStore.Ping() {
+		status = http.StatusInternalServerError
+	}
+	if !ctx.NotificationState.Ping() {
+		status = http.StatusInternalServerError
 	}
 
 	w.WriteHeader(status)

api/v1/models.go

@@ -21,7 +21,7 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/coreos/clair/database"
+	"github.com/coreos/clair/services"
 	"github.com/coreos/clair/utils/types"
 	"github.com/coreos/pkg/capnslog"
 	"github.com/fernet/fernet-go"
@@ -44,7 +44,7 @@ type Layer struct {
 	Features         []Feature         `json:"Features,omitempty"`
 }
 
-func LayerFromDatabaseModel(dbLayer database.Layer, withFeatures, withVulnerabilities bool) Layer {
+func LayerFromDatabaseModel(dbLayer services.Layer, withFeatures, withVulnerabilities bool) Layer {
 	layer := Layer{
 		Name:             dbLayer.Name,
 		IndexedByVersion: dbLayer.EngineVersion,
@@ -104,25 +104,25 @@ type Vulnerability struct {
 	FixedIn       []Feature              `json:"FixedIn,omitempty"`
 }
 
-func (v Vulnerability) DatabaseModel() (database.Vulnerability, error) {
+func (v Vulnerability) DatabaseModel() (services.Vulnerability, error) {
 	severity := types.Priority(v.Severity)
 	if !severity.IsValid() {
-		return database.Vulnerability{}, errors.New("Invalid severity")
+		return services.Vulnerability{}, errors.New("Invalid severity")
 	}
 
-	var dbFeatures []database.FeatureVersion
+	var dbFeatures []services.FeatureVersion
 	for _, feature := range v.FixedIn {
 		dbFeature, err := feature.DatabaseModel()
 		if err != nil {
-			return database.Vulnerability{}, err
+			return services.Vulnerability{}, err
 		}
 
 		dbFeatures = append(dbFeatures, dbFeature)
 	}
 
-	return database.Vulnerability{
+	return services.Vulnerability{
 		Name:        v.Name,
-		Namespace:   database.Namespace{Name: v.NamespaceName},
+		Namespace:   services.Namespace{Name: v.NamespaceName},
 		Description: v.Description,
 		Link:        v.Link,
 		Severity:    severity,
@@ -131,7 +131,7 @@ func (v Vulnerability) DatabaseModel() (database.Vulnerability, error) {
 	}, nil
 }
 
-func VulnerabilityFromDatabaseModel(dbVuln database.Vulnerability, withFixedIn bool) Vulnerability {
+func VulnerabilityFromDatabaseModel(dbVuln services.Vulnerability, withFixedIn bool) Vulnerability {
 	vuln := Vulnerability{
 		Name:          dbVuln.Name,
 		NamespaceName: dbVuln.Namespace.Name,
@@ -158,7 +158,7 @@ type Feature struct {
 	AddedBy         string          `json:"AddedBy,omitempty"`
 }
 
-func FeatureFromDatabaseModel(dbFeatureVersion database.FeatureVersion) Feature {
+func FeatureFromDatabaseModel(dbFeatureVersion services.FeatureVersion) Feature {
 	versionStr := dbFeatureVersion.Version.String()
 	if versionStr == types.MaxVersion.String() {
 		versionStr = "None"
@@ -172,22 +172,22 @@ func FeatureFromDatabaseModel(dbFeatureVersion database.FeatureVersion) Feature
 	}
 }
 
-func (f Feature) DatabaseModel() (database.FeatureVersion, error) {
+func (f Feature) DatabaseModel() (services.FeatureVersion, error) {
 	var version types.Version
 	if f.Version == "None" {
 		version = types.MaxVersion
 	} else {
 		var err error
 		version, err = types.NewVersion(f.Version)
 		if err != nil {
-			return database.FeatureVersion{}, err
+			return services.FeatureVersion{}, err
 		}
 	}
 
-	return database.FeatureVersion{
-		Feature: database.Feature{
+	return services.FeatureVersion{
+		Feature: services.Feature{
 			Name:      f.Name,
-			Namespace: database.Namespace{Name: f.NamespaceName},
+			Namespace: services.Namespace{Name: f.NamespaceName},
 		},
 		Version: version,
 	}, nil
@@ -205,7 +205,7 @@ type Notification struct {
 	New      *VulnerabilityWithLayers `json:"New,omitempty"`
 }
 
-func NotificationFromDatabaseModel(dbNotification database.VulnerabilityNotification, limit int, pageToken string, nextPage database.VulnerabilityNotificationPageNumber, key string) Notification {
+func NotificationFromDatabaseModel(dbNotification services.VulnerabilityNotification, limit int, pageToken string, nextPage services.VulnerabilityNotificationPageNumber, key string) Notification {
 	var oldVuln *VulnerabilityWithLayers
 	if dbNotification.OldVulnerability != nil {
 		v := VulnerabilityWithLayersFromDatabaseModel(*dbNotification.OldVulnerability)
@@ -219,7 +219,7 @@ func NotificationFromDatabaseModel(dbNotification database.VulnerabilityNotifica
 	}
 
 	var nextPageStr string
-	if nextPage != database.NoVulnerabilityNotificationPage {
+	if nextPage != services.NoVulnerabilityNotificationPage {
 		nextPageBytes, _ := tokenMarshal(nextPage, key)
 		nextPageStr = string(nextPageBytes)
 	}
@@ -255,7 +255,7 @@ type VulnerabilityWithLayers struct {
 	LayersIntroducingVulnerability []string       `json:"LayersIntroducingVulnerability,omitempty"`
 }
 
-func VulnerabilityWithLayersFromDatabaseModel(dbVuln database.Vulnerability) VulnerabilityWithLayers {
+func VulnerabilityWithLayersFromDatabaseModel(dbVuln services.Vulnerability) VulnerabilityWithLayers {
 	vuln := VulnerabilityFromDatabaseModel(dbVuln, true)
 
 	var layers []string

api/v1/routes.go

@@ -26,7 +26,7 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 
 	"github.com/coreos/clair/api/context"
-	"github.com/coreos/clair/database"
+	"github.com/coreos/clair/services"
 	"github.com/coreos/clair/utils"
 	cerrors "github.com/coreos/clair/utils/errors"
 	"github.com/coreos/clair/worker"
@@ -109,7 +109,7 @@ func postLayer(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx
 		return postLayerRoute, http.StatusBadRequest
 	}
 
-	err = worker.Process(ctx.Store, request.Layer.Format, request.Layer.Name, request.Layer.ParentName, request.Layer.Path, request.Layer.Headers)
+	err = worker.Process(ctx.VulnerabilityStore, request.Layer.Format, request.Layer.Name, request.Layer.ParentName, request.Layer.Path, request.Layer.Headers)
 	if err != nil {
 		if err == utils.ErrCouldNotExtract ||
 			err == utils.ErrExtractedFileTooBig ||
@@ -142,7 +142,7 @@ func getLayer(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *
 	_, withFeatures := r.URL.Query()["features"]
 	_, withVulnerabilities := r.URL.Query()["vulnerabilities"]
 
-	dbLayer, err := ctx.Store.FindLayer(p.ByName("layerName"), withFeatures, withVulnerabilities)
+	dbLayer, err := ctx.VulnerabilityStore.FindLayer(p.ByName("layerName"), withFeatures, withVulnerabilities)
 	if err == cerrors.ErrNotFound {
 		writeResponse(w, r, http.StatusNotFound, LayerEnvelope{Error: &Error{err.Error()}})
 		return getLayerRoute, http.StatusNotFound
@@ -158,7 +158,7 @@ func getLayer(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *
 }
 
 func deleteLayer(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) (string, int) {
-	err := ctx.Store.DeleteLayer(p.ByName("layerName"))
+	err := ctx.VulnerabilityStore.DeleteLayer(p.ByName("layerName"))
 	if err == cerrors.ErrNotFound {
 		writeResponse(w, r, http.StatusNotFound, LayerEnvelope{Error: &Error{err.Error()}})
 		return deleteLayerRoute, http.StatusNotFound
@@ -172,7 +172,7 @@ func deleteLayer(w http.ResponseWriter, r *http.Request, p httprouter.Params, ct
 }
 
 func getNamespaces(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) (string, int) {
-	dbNamespaces, err := ctx.Store.ListNamespaces()
+	dbNamespaces, err := ctx.VulnerabilityStore.ListNamespaces()
 	if err != nil {
 		writeResponse(w, r, http.StatusInternalServerError, NamespaceEnvelope{Error: &Error{err.Error()}})
 		return getNamespacesRoute, http.StatusInternalServerError
@@ -219,7 +219,7 @@ func getVulnerabilities(w http.ResponseWriter, r *http.Request, p httprouter.Par
 		return getNotificationRoute, http.StatusBadRequest
 	}
 
-	dbVulns, nextPage, err := ctx.Store.ListVulnerabilities(namespace, limit, page)
+	dbVulns, nextPage, err := ctx.VulnerabilityStore.ListVulnerabilities(namespace, limit, page)
 	if err == cerrors.ErrNotFound {
 		writeResponse(w, r, http.StatusNotFound, VulnerabilityEnvelope{Error: &Error{err.Error()}})
 		return getVulnerabilityRoute, http.StatusNotFound
@@ -267,7 +267,7 @@ func postVulnerability(w http.ResponseWriter, r *http.Request, p httprouter.Para
 		return postVulnerabilityRoute, http.StatusBadRequest
 	}
 
-	err = ctx.Store.InsertVulnerabilities([]database.Vulnerability{vuln}, true)
+	err = ctx.VulnerabilityStore.InsertVulnerabilities([]services.Vulnerability{vuln}, true)
 	if err != nil {
 		switch err.(type) {
 		case *cerrors.ErrBadRequest:
@@ -286,7 +286,7 @@ func postVulnerability(w http.ResponseWriter, r *http.Request, p httprouter.Para
 func getVulnerability(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) (string, int) {
 	_, withFixedIn := r.URL.Query()["fixedIn"]
 
-	dbVuln, err := ctx.Store.FindVulnerability(p.ByName("namespaceName"), p.ByName("vulnerabilityName"))
+	dbVuln, err := ctx.VulnerabilityStore.FindVulnerability(p.ByName("namespaceName"), p.ByName("vulnerabilityName"))
 	if err == cerrors.ErrNotFound {
 		writeResponse(w, r, http.StatusNotFound, VulnerabilityEnvelope{Error: &Error{err.Error()}})
 		return getVulnerabilityRoute, http.StatusNotFound
@@ -328,7 +328,7 @@ func putVulnerability(w http.ResponseWriter, r *http.Request, p httprouter.Param
 	vuln.Namespace.Name = p.ByName("namespaceName")
 	vuln.Name = p.ByName("vulnerabilityName")
 
-	err = ctx.Store.InsertVulnerabilities([]database.Vulnerability{vuln}, true)
+	err = ctx.VulnerabilityStore.InsertVulnerabilities([]services.Vulnerability{vuln}, true)
 	if err != nil {
 		switch err.(type) {
 		case *cerrors.ErrBadRequest:
@@ -345,7 +345,7 @@ func putVulnerability(w http.ResponseWriter, r *http.Request, p httprouter.Param
 }
 
 func deleteVulnerability(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) (string, int) {
-	err := ctx.Store.DeleteVulnerability(p.ByName("namespaceName"), p.ByName("vulnerabilityName"))
+	err := ctx.VulnerabilityStore.DeleteVulnerability(p.ByName("namespaceName"), p.ByName("vulnerabilityName"))
 	if err == cerrors.ErrNotFound {
 		writeResponse(w, r, http.StatusNotFound, VulnerabilityEnvelope{Error: &Error{err.Error()}})
 		return deleteVulnerabilityRoute, http.StatusNotFound
@@ -359,7 +359,7 @@ func deleteVulnerability(w http.ResponseWriter, r *http.Request, p httprouter.Pa
 }
 
 func getFixes(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) (string, int) {
-	dbVuln, err := ctx.Store.FindVulnerability(p.ByName("namespaceName"), p.ByName("vulnerabilityName"))
+	dbVuln, err := ctx.VulnerabilityStore.FindVulnerability(p.ByName("namespaceName"), p.ByName("vulnerabilityName"))
 	if err == cerrors.ErrNotFound {
 		writeResponse(w, r, http.StatusNotFound, FeatureEnvelope{Error: &Error{err.Error()}})
 		return getFixesRoute, http.StatusNotFound
@@ -397,7 +397,7 @@ func putFix(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *co
 		return putFixRoute, http.StatusBadRequest
 	}
 
-	err = ctx.Store.InsertVulnerabilityFixes(p.ByName("vulnerabilityNamespace"), p.ByName("vulnerabilityName"), []database.FeatureVersion{dbFix})
+	err = ctx.VulnerabilityStore.InsertVulnerabilityFixes(p.ByName("vulnerabilityNamespace"), p.ByName("vulnerabilityName"), []services.FeatureVersion{dbFix})
 	if err != nil {
 		switch err.(type) {
 		case *cerrors.ErrBadRequest:
@@ -418,7 +418,7 @@ func putFix(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *co
 }
 
 func deleteFix(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) (string, int) {
-	err := ctx.Store.DeleteVulnerabilityFix(p.ByName("vulnerabilityNamespace"), p.ByName("vulnerabilityName"), p.ByName("fixName"))
+	err := ctx.VulnerabilityStore.DeleteVulnerabilityFix(p.ByName("vulnerabilityNamespace"), p.ByName("vulnerabilityName"), p.ByName("fixName"))
 	if err == cerrors.ErrNotFound {
 		writeResponse(w, r, http.StatusNotFound, FeatureEnvelope{Error: &Error{err.Error()}})
 		return deleteFixRoute, http.StatusNotFound
@@ -446,7 +446,7 @@ func getNotification(w http.ResponseWriter, r *http.Request, p httprouter.Params
 	}
 
 	var pageToken string
-	page := database.VulnerabilityNotificationFirstPage
+	page := services.VulnerabilityNotificationFirstPage
 	pageStrs, pageExists := query["page"]
 	if pageExists {
 		err := tokenUnmarshal(pageStrs[0], ctx.Config.PaginationKey, &page)
@@ -464,7 +464,7 @@ func getNotification(w http.ResponseWriter, r *http.Request, p httprouter.Params
 		pageToken = string(pageTokenBytes)
 	}
 
-	dbNotification, nextPage, err := ctx.Store.GetNotification(p.ByName("notificationName"), limit, page)
+	dbNotification, nextPage, err := ctx.NotificationState.GetNotification(p.ByName("notificationName"), limit, page)
 	if err == cerrors.ErrNotFound {
 		writeResponse(w, r, http.StatusNotFound, NotificationEnvelope{Error: &Error{err.Error()}})
 		return deleteNotificationRoute, http.StatusNotFound
@@ -480,7 +480,7 @@ func getNotification(w http.ResponseWriter, r *http.Request, p httprouter.Params
 }
 
 func deleteNotification(w http.ResponseWriter, r *http.Request, p httprouter.Params, ctx *context.RouteContext) (string, int) {
-	err := ctx.Store.DeleteNotification(p.ByName("notificationName"))
+	err := ctx.NotificationState.DeleteNotification(p.ByName("notificationName"))
 	if err == cerrors.ErrNotFound {
 		writeResponse(w, r, http.StatusNotFound, NotificationEnvelope{Error: &Error{err.Error()}})
 		return deleteNotificationRoute, http.StatusNotFound