added protocolversion in servingruntime spec (kubeflow#2118)

* added protocolversion in servingruntime spec Signed-off-by: Suresh Nakkeran <suresh.n@ideas2it.com> * allow a list of protocol versions in servingruntime Signed-off-by: Suresh Nakkeran <suresh.n@ideas2it.com> * 1. added logic to list runtimes order by protocol v1, created timestamp and name 2. removed predictor protocolversion default to v1 Signed-off-by: Suresh Nakkeran <suresh.n@ideas2it.com> * added protocolversion grpc-v1, grpc-v2 and updated protocol version sorting logic Signed-off-by: Suresh Nakkeran <suresh.n@ideas2it.com> * updated openapi, swagger Signed-off-by: Suresh Nakkeran <suresh.n@ideas2it.com>
magdalenakuhn17 · Apr 14, 2022 · aa2bf63 · aa2bf63
1 parent 7b67a38
commit aa2bf63
Show file tree

Hide file tree

Showing 24 changed files with 393 additions and 190 deletions.
diff --git a/config/crd/serving.kserve.io_clusterservingruntimes.yaml b/config/crd/serving.kserve.io_clusterservingruntimes.yaml
@@ -1038,6 +1038,10 @@ spec:
                 additionalProperties:
                   type: string
                 type: object
+              protocolVersions:
+                items:
+                  type: string
+                type: array
               replicas:
                 type: integer
               storageHelper:

diff --git a/config/crd/serving.kserve.io_servingruntimes.yaml b/config/crd/serving.kserve.io_servingruntimes.yaml
@@ -1038,6 +1038,10 @@ spec:
                 additionalProperties:
                   type: string
                 type: object
+              protocolVersions:
+                items:
+                  type: string
+                type: array
               replicas:
                 type: integer
               storageHelper:

diff --git a/config/runtimes/kserve-lgbserver.yaml b/config/runtimes/kserve-lgbserver.yaml
@@ -7,6 +7,8 @@ spec:
     - name: lightgbm
       version: "2"
       autoSelect: true
+  protocolVersions:
+    - v1
   containers:
     - name: kserve-container
       image: kserve-lgbserver:replace

diff --git a/config/runtimes/kserve-mlserver.yaml b/config/runtimes/kserve-mlserver.yaml
@@ -6,13 +6,18 @@ spec:
   supportedModelFormats:
     - name: sklearn
       version: "0"
+      autoSelect: true
     - name: xgboost
       version: "1"
+      autoSelect: true
     - name: lightgbm
       version: "3"
+      autoSelect: true
     - name: mlflow
       version: "1"
       autoSelect: true
+  protocolVersions:
+    - v2
   containers:
     - name: kserve-container
       image: mlserver:replace

diff --git a/config/runtimes/kserve-paddleserver.yaml b/config/runtimes/kserve-paddleserver.yaml
@@ -7,6 +7,8 @@ spec:
     - name: paddle
       version: "2"
       autoSelect: true
+  protocolVersions:
+    - v1
   containers:
     - name: kserve-container
       image: kserve-paddleserver:replace

diff --git a/config/runtimes/kserve-pmmlserver.yaml b/config/runtimes/kserve-pmmlserver.yaml
@@ -10,6 +10,8 @@ spec:
     - name: pmml
       version: "4"
       autoSelect: true
+  protocolVersions:
+    - v1
   containers:
     - name: kserve-container
       image: kserve-pmmlserver:replace

diff --git a/config/runtimes/kserve-sklearnserver.yaml b/config/runtimes/kserve-sklearnserver.yaml
@@ -7,6 +7,8 @@ spec:
     - name: sklearn
       version: "1"
       autoSelect: true
+  protocolVersions:
+    - v1
   containers:
     - name: kserve-container
       image: kserve-sklearnserver:replace

diff --git a/config/runtimes/kserve-tensorflow-serving.yaml b/config/runtimes/kserve-tensorflow-serving.yaml
@@ -10,6 +10,9 @@ spec:
     - name: tensorflow
       version: "2"
       autoSelect: true
+  protocolVersions:
+    - v1
+    - grpc-v1
   containers:
     - name: kserve-container
       image: tensorflow-serving:replace

diff --git a/config/runtimes/kserve-torchserve.yaml b/config/runtimes/kserve-torchserve.yaml
@@ -7,6 +7,10 @@ spec:
     - name: pytorch
       version: "1"
       autoSelect: true
+  protocolVersions:
+    - v1
+    - v2
+    - grpc-v1
   containers:
     - name: kserve-container
       image: kserve-torchserve:replace

diff --git a/config/runtimes/kserve-tritonserver.yaml b/config/runtimes/kserve-tritonserver.yaml
@@ -6,10 +6,13 @@ spec:
   supportedModelFormats:
     - name: tensorrt
       version: "8"
+      autoSelect: true
     - name: tensorflow
       version: "1"
+      autoSelect: true
     - name: tensorflow
       version: "2"
+      autoSelect: true
     - name: onnx
       version: "1"
       autoSelect: true
@@ -18,6 +21,9 @@ spec:
     - name: triton
       version: "2"
       autoSelect: true
+  protocolVersions:
+    - v2
+    - grpc-v2
   containers:
     - name: kserve-container
       image: kserve-tritonserver:replace

diff --git a/config/runtimes/kserve-xgbserver.yaml b/config/runtimes/kserve-xgbserver.yaml
@@ -7,6 +7,8 @@ spec:
     - name: xgboost
       version: "1"
       autoSelect: true
+  protocolVersions:
+    - v1
   containers:
     - name: kserve-container
       image: kserve-xgbserver:replace

diff --git a/hack/violation_exceptions.list b/hack/violation_exceptions.list
@@ -1,6 +1,7 @@
 API rule violation: list_type_missing,./pkg/apis/serving/v1alpha1,BuiltInAdapter,Env
 API rule violation: list_type_missing,./pkg/apis/serving/v1alpha1,ServingRuntimePodSpec,Containers
 API rule violation: list_type_missing,./pkg/apis/serving/v1alpha1,ServingRuntimePodSpec,Tolerations
+API rule violation: list_type_missing,./pkg/apis/serving/v1alpha1,ServingRuntimeSpec,ProtocolVersions
 API rule violation: list_type_missing,./pkg/apis/serving/v1alpha1,ServingRuntimeSpec,SupportedModelFormats
 API rule violation: list_type_missing,./pkg/apis/serving/v1alpha1,TrainedModelList,Items
 API rule violation: list_type_missing,./pkg/apis/serving/v1beta1,ComponentStatusSpec,Traffic

diff --git a/pkg/apis/serving/v1alpha1/servingruntime_types.go b/pkg/apis/serving/v1alpha1/servingruntime_types.go
@@ -17,6 +17,7 @@ limitations under the License.
 package v1alpha1
 
 import (
+	"github.com/kserve/kserve/pkg/constants"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
@@ -86,6 +87,10 @@ type ServingRuntimeSpec struct {
 	// +optional
 	Disabled *bool `json:"disabled,omitempty"`
 
+	// Supported protocol versions (i.e. v1 or v2 or grpc-v1 or grpc-v2)
+	// +optional
+	ProtocolVersions []constants.InferenceServiceProtocol `json:"protocolVersions,omitempty"`
+
 	ServingRuntimePodSpec `json:",inline"`
 
 	// The following fields apply to ModelMesh deployments.
@@ -200,6 +205,12 @@ type ClusterServingRuntimeList struct {
 	Items           []ClusterServingRuntime `json:"items"`
 }
 
+// SupportedRuntime is the schema for supported runtime result of automatic selection
+type SupportedRuntime struct {
+	Name string
+	Spec ServingRuntimeSpec
+}
+
 func init() {
 	SchemeBuilder.Register(&ServingRuntime{}, &ServingRuntimeList{})
 	SchemeBuilder.Register(&ClusterServingRuntime{}, &ClusterServingRuntimeList{})
@@ -212,3 +223,15 @@ func (srSpec *ServingRuntimeSpec) IsDisabled() bool {
 func (srSpec *ServingRuntimeSpec) IsMultiModelRuntime() bool {
 	return srSpec.MultiModel != nil && *srSpec.MultiModel
 }
+
+func (srSpec *ServingRuntimeSpec) IsProtocolVersionSupported(modelProtocolVersion constants.InferenceServiceProtocol) bool {
+	if len(modelProtocolVersion) == 0 || srSpec.ProtocolVersions == nil || len(srSpec.ProtocolVersions) == 0 {
+		return true
+	}
+	for _, srProtocolVersion := range srSpec.ProtocolVersions {
+		if srProtocolVersion == modelProtocolVersion {
+			return true
+		}
+	}
+	return false
+}
diff --git a/pkg/apis/serving/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/serving/v1alpha1/zz_generated.deepcopy.go