From 9a38524c17bcd93cd5ac4740e4be5f2431cc2904 Mon Sep 17 00:00:00 2001 From: Dan Sun Date: Sat, 12 Oct 2019 23:58:36 -0400 Subject: [PATCH] Fix missing alibi key in explainers configmap (#439) * Fix missing alibi key in configmap * Remove hardcode alibi image name * Add alibi explainer default test * Add validation/default tests for explainer * Add alibi explainer container creation test * Fix format * Fix alibiexplainer image registry --- .../default/configmap/inferenceservice.yaml | 14 +- install/0.2.0/kfserving.yaml | 12 +- pkg/apis/serving/v1alpha2/explainer_alibi.go | 8 +- .../serving/v1alpha2/explainer_alibi_test.go | 99 ++++++++++++++ .../inferenceservice_defaults_test.go | 37 ++++++ .../inferenceservice_validation_test.go | 12 ++ .../serving/v1alpha2/v1alpha2_suite_test.go | 122 ++++++++++-------- 7 files changed, 230 insertions(+), 74 deletions(-) create mode 100644 pkg/apis/serving/v1alpha2/explainer_alibi_test.go diff --git a/config/default/configmap/inferenceservice.yaml b/config/default/configmap/inferenceservice.yaml index fb93cb31048..a93848d620d 100644 --- a/config/default/configmap/inferenceservice.yaml +++ b/config/default/configmap/inferenceservice.yaml @@ -62,15 +62,17 @@ data: } explainers: |- { - "image" : "docker.io/seldonio/alibiexplainer", - "defaultImageVersion": "0.2.3", - "allowedImageVersions": [ - "0.2.3" - ] + "alibi": { + "image" : "gcr.io/kfserving/alibi-explainer", + "defaultImageVersion": "0.2.0", + "allowedImageVersions": [ + "0.2.0" + ] + } } storageInitializer: |- { - "image" : "gcr.io/kfserving/storage-initializer:latest" + "image" : "gcr.io/kfserving/storage-initializer:0.2.0" } credentials: |- { diff --git a/install/0.2.0/kfserving.yaml b/install/0.2.0/kfserving.yaml index e32662612c4..b5e8f1737cd 100644 --- a/install/0.2.0/kfserving.yaml +++ b/install/0.2.0/kfserving.yaml @@ -756,11 +756,13 @@ data: } explainers: |- { - "image" : "docker.io/seldonio/alibiexplainer", - "defaultImageVersion": "0.2.3", - "allowedImageVersions": [ - "0.2.3" - ] + "alibi": { + "image" : "gcr.io/kfserving/alibi-explainer", + "defaultImageVersion": "0.2.0", + "allowedImageVersions": [ + "0.2.0" + ] + } } ingress: |- { diff --git a/pkg/apis/serving/v1alpha2/explainer_alibi.go b/pkg/apis/serving/v1alpha2/explainer_alibi.go index 42eb8d90d96..3f89b673d3f 100644 --- a/pkg/apis/serving/v1alpha2/explainer_alibi.go +++ b/pkg/apis/serving/v1alpha2/explainer_alibi.go @@ -9,7 +9,6 @@ import ( ) var ( - AlibiImageName = "docker.io/seldonio/alibiexplainer" InvalidAlibiRuntimeVersionError = "RuntimeVersion must be one of %s" ) @@ -18,11 +17,6 @@ func (s *AlibiExplainerSpec) GetStorageUri() string { } func (s *AlibiExplainerSpec) CreateExplainerContainer(modelName string, predictorHost string, config *InferenceServicesConfig) *v1.Container { - imageName := AlibiImageName - if config.Explainers.AlibiExplainer.ContainerImage != "" { - imageName = config.Explainers.AlibiExplainer.ContainerImage - } - var args = []string{ constants.ArgumentModelName, modelName, constants.ArgumentPredictorHost, predictorHost, @@ -40,7 +34,7 @@ func (s *AlibiExplainerSpec) CreateExplainerContainer(modelName string, predicto } return &v1.Container{ - Image: imageName + ":" + s.RuntimeVersion, + Image: config.Explainers.AlibiExplainer.ContainerImage + ":" + s.RuntimeVersion, Resources: s.Resources, Args: args, } diff --git a/pkg/apis/serving/v1alpha2/explainer_alibi_test.go b/pkg/apis/serving/v1alpha2/explainer_alibi_test.go new file mode 100644 index 00000000000..ae161299cf6 --- /dev/null +++ b/pkg/apis/serving/v1alpha2/explainer_alibi_test.go @@ -0,0 +1,99 @@ +package v1alpha2 + +import ( + "fmt" + "github.com/onsi/gomega" + "github.com/onsi/gomega/types" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "strings" + "testing" +) + +func TestAlibiExplainer(t *testing.T) { + g := gomega.NewGomegaWithT(t) + allowedAlibiImageVersionsArray := []string{ + DefaultAlibiExplainerRuntimeVersion, + } + allowedAlibiImageVersions := strings.Join(allowedAlibiImageVersionsArray, ", ") + + scenarios := map[string]struct { + spec AlibiExplainerSpec + matcher types.GomegaMatcher + }{ + "AcceptGoodRuntimeVersion": { + spec: AlibiExplainerSpec{ + RuntimeVersion: DefaultAlibiExplainerRuntimeVersion, + }, + matcher: gomega.Succeed(), + }, + "RejectBadRuntimeVersion": { + spec: AlibiExplainerSpec{ + RuntimeVersion: "", + }, + matcher: gomega.MatchError(fmt.Sprintf(InvalidAlibiRuntimeVersionError, allowedAlibiImageVersions)), + }, + } + + for name, scenario := range scenarios { + config := &InferenceServicesConfig{ + Explainers: &ExplainersConfig{ + AlibiExplainer: ExplainerConfig{ + ContainerImage: "seldon.io/alibi", + DefaultImageVersion: "latest", + AllowedImageVersions: allowedAlibiImageVersionsArray, + }, + }, + } + g.Expect(scenario.spec.Validate(config)).Should(scenario.matcher, fmt.Sprintf("Testing %s", name)) + } +} + +func TestCreateAlibiExplainerContainer(t *testing.T) { + + var requestedResource = v1.ResourceRequirements{ + Limits: v1.ResourceList{ + "cpu": resource.Quantity{ + Format: "100", + }, + }, + Requests: v1.ResourceList{ + "cpu": resource.Quantity{ + Format: "90", + }, + }, + } + config := &InferenceServicesConfig{ + Explainers: &ExplainersConfig{ + AlibiExplainer: ExplainerConfig{ + ContainerImage: "seldon.io/alibi", + DefaultImageVersion: "latest", + }, + }, + } + var spec = AlibiExplainerSpec{ + Type: "Anchor", + StorageURI: "gs://someUri", + Resources: requestedResource, + RuntimeVersion: "0.1.0", + } + g := gomega.NewGomegaWithT(t) + + expectedContainer := &v1.Container{ + Image: "seldon.io/alibi:0.1.0", + Resources: requestedResource, + Args: []string{ + "--model_name", + "someName", + "--predictor_host", + "predictor.svc.cluster.local", + "--storage_uri", + "/mnt/models", + "Anchor", + }, + } + + // Test Create with config + container := spec.CreateExplainerContainer("someName", "predictor.svc.cluster.local", config) + g.Expect(container).To(gomega.Equal(expectedContainer)) +} diff --git a/pkg/apis/serving/v1alpha2/inferenceservice_defaults_test.go b/pkg/apis/serving/v1alpha2/inferenceservice_defaults_test.go index 23ced84d7bf..3cebecb4186 100644 --- a/pkg/apis/serving/v1alpha2/inferenceservice_defaults_test.go +++ b/pkg/apis/serving/v1alpha2/inferenceservice_defaults_test.go @@ -231,3 +231,40 @@ func TestTensorRTDefaults(t *testing.T) { g.Expect(isvc.Spec.Canary.Predictor.TensorRT.Resources.Requests[v1.ResourceCPU]).To(gomega.Equal(DefaultCPU)) g.Expect(isvc.Spec.Canary.Predictor.TensorRT.Resources.Requests[v1.ResourceMemory]).To(gomega.Equal(resource.MustParse("3Gi"))) } + +func TestAlibiExplainerDefaults(t *testing.T) { + g := gomega.NewGomegaWithT(t) + isvc := InferenceService{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "default", + }, + Spec: InferenceServiceSpec{ + Default: EndpointSpec{ + Predictor: PredictorSpec{ + Tensorflow: &TensorflowSpec{ + StorageURI: "gs://testbucket/testmodel", + }, + }, + Explainer: &ExplainerSpec{ + Alibi: &AlibiExplainerSpec{ + StorageURI: "gs://testbucket/testmodel", + }, + }, + }, + }, + } + isvc.Spec.Canary = isvc.Spec.Default.DeepCopy() + isvc.Spec.Canary.Explainer.Alibi.RuntimeVersion = "0.2.4" + isvc.Spec.Canary.Explainer.Alibi.Resources.Requests = v1.ResourceList{v1.ResourceMemory: resource.MustParse("3Gi")} + isvc.Default(c) + + g.Expect(isvc.Spec.Default.Explainer.Alibi.RuntimeVersion).To(gomega.Equal(DefaultAlibiExplainerRuntimeVersion)) + g.Expect(isvc.Spec.Default.Explainer.Alibi.Resources.Requests[v1.ResourceCPU]).To(gomega.Equal(DefaultCPU)) + g.Expect(isvc.Spec.Default.Explainer.Alibi.Resources.Requests[v1.ResourceMemory]).To(gomega.Equal(DefaultMemory)) + g.Expect(isvc.Spec.Default.Explainer.Alibi.Resources.Limits[v1.ResourceCPU]).To(gomega.Equal(DefaultCPU)) + g.Expect(isvc.Spec.Default.Explainer.Alibi.Resources.Limits[v1.ResourceMemory]).To(gomega.Equal(DefaultMemory)) + g.Expect(isvc.Spec.Canary.Explainer.Alibi.RuntimeVersion).To(gomega.Equal("0.2.4")) + g.Expect(isvc.Spec.Canary.Explainer.Alibi.Resources.Requests[v1.ResourceCPU]).To(gomega.Equal(DefaultCPU)) + g.Expect(isvc.Spec.Canary.Explainer.Alibi.Resources.Requests[v1.ResourceMemory]).To(gomega.Equal(resource.MustParse("3Gi"))) +} diff --git a/pkg/apis/serving/v1alpha2/inferenceservice_validation_test.go b/pkg/apis/serving/v1alpha2/inferenceservice_validation_test.go index e8b1991ed7d..b94a4d8c3e7 100644 --- a/pkg/apis/serving/v1alpha2/inferenceservice_validation_test.go +++ b/pkg/apis/serving/v1alpha2/inferenceservice_validation_test.go @@ -211,3 +211,15 @@ func TestRejectBadExplainer(t *testing.T) { isvc.Spec.Default.Explainer = &ExplainerSpec{} g.Expect(isvc.ValidateCreate(c)).Should(gomega.MatchError(ExactlyOneExplainerViolatedError)) } + +func TestGoodExplainer(t *testing.T) { + g := gomega.NewGomegaWithT(t) + isvc := makeTestInferenceService() + isvc.Spec.Default.Explainer = &ExplainerSpec{ + Alibi: &AlibiExplainerSpec{ + StorageURI: "gs://testbucket/testmodel", + }, + } + isvc.Default(c) + g.Expect(isvc.ValidateCreate(c)).Should(gomega.Succeed()) +} diff --git a/pkg/apis/serving/v1alpha2/v1alpha2_suite_test.go b/pkg/apis/serving/v1alpha2/v1alpha2_suite_test.go index 9bf5c55ff52..b57a6cfb400 100644 --- a/pkg/apis/serving/v1alpha2/v1alpha2_suite_test.go +++ b/pkg/apis/serving/v1alpha2/v1alpha2_suite_test.go @@ -36,13 +36,14 @@ var cfg *rest.Config var c client.Client const ( - DefaultTensorflowRuntimeVersion = "latest" - DefaultTensorflowRuntimeVersionGPU = "latest-gpu" - DefaultSKLearnRuntimeVersion = "0.1.0" - DefaultPyTorchRuntimeVersion = "0.1.0" - DefaultXGBoostRuntimeVersion = "0.1.0" - DefaultTensorRTRuntimeVersion = "19.05-py3" - DefaultONNXRuntimeVersion = "v0.5.0" + DefaultTensorflowRuntimeVersion = "latest" + DefaultTensorflowRuntimeVersionGPU = "latest-gpu" + DefaultSKLearnRuntimeVersion = "0.1.0" + DefaultPyTorchRuntimeVersion = "0.1.0" + DefaultXGBoostRuntimeVersion = "0.1.0" + DefaultTensorRTRuntimeVersion = "19.05-py3" + DefaultONNXRuntimeVersion = "v0.5.0" + DefaultAlibiExplainerRuntimeVersion = "0.2.3" ) func TestMain(m *testing.M) { @@ -67,55 +68,64 @@ func TestMain(m *testing.M) { // Create configmap configs := map[string]string{ "predictors": `{ - "tensorflow" : { - "image" : "tensorflow/serving", - "defaultImageVersion": "latest", - "defaultGPUImageVersion": "latest-gpu", - "allowedImageVersions": [ - "latest", - "latest-gpu" - ] - }, - "sklearn" : { - "image" : "kfserving/sklearnserver", - "defaultImageVersion": "0.1.0", - "allowedImageVersions": [ - "latest", - "0.1.0" - ] - }, - "xgboost" : { - "image" : "kfserving/xgbserver", - "defaultImageVersion": "0.1.0", - "allowedImageVersions": [ - "latest", - "0.1.0" - ] - }, - "pytorch" : { - "image" : "kfserving/pytorchserver", - "defaultImageVersion": "0.1.0", - "allowedImageVersions": [ - "latest", - "0.1.0" - ] - }, - "onnx" : { - "image" : "onnxruntime/server", - "defaultImageVersion": "v0.5.0", - "allowedImageVersions": [ - "latest", - "v0.5.0" - ] - }, - "tensorrt" : { - "image" : "nvcr.io/nvidia/tensorrtserver", - "defaultImageVersion": "19.05-py3", - "allowedImageVersions": [ - "19.05-py3" - ] - } - }`, + "tensorflow" : { + "image" : "tensorflow/serving", + "defaultImageVersion": "latest", + "defaultGPUImageVersion": "latest-gpu", + "allowedImageVersions": [ + "latest", + "latest-gpu" + ] + }, + "sklearn" : { + "image" : "kfserving/sklearnserver", + "defaultImageVersion": "0.1.0", + "allowedImageVersions": [ + "latest", + "0.1.0" + ] + }, + "xgboost" : { + "image" : "kfserving/xgbserver", + "defaultImageVersion": "0.1.0", + "allowedImageVersions": [ + "latest", + "0.1.0" + ] + }, + "pytorch" : { + "image" : "kfserving/pytorchserver", + "defaultImageVersion": "0.1.0", + "allowedImageVersions": [ + "latest", + "0.1.0" + ] + }, + "onnx" : { + "image" : "onnxruntime/server", + "defaultImageVersion": "v0.5.0", + "allowedImageVersions": [ + "latest", + "v0.5.0" + ] + }, + "tensorrt" : { + "image" : "nvcr.io/nvidia/tensorrtserver", + "defaultImageVersion": "19.05-py3", + "allowedImageVersions": [ + "19.05-py3" + ] + } + }`, + "explainers": `{ + "alibi" : { + "image" : "docker.io/seldonio/alibiexplainer", + "defaultImageVersion": "0.2.3", + "allowedImageVersions": [ + "0.2.3" + ] + } + }`, } var configMap = &v1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{