Add SDK API to create Creds (kubeflow#288)

* Add SDK API to create Creds

* try to resolve unit test timeout

python/kfserving/docs/KFServingClient.md

@@ -16,11 +16,66 @@ The APIs for KFServingClient are as following:
 
 Class | Method |  Description
 ------------ | ------------- | -------------
+KFServingClient | [set_credentials](#set_credentials) | Set Credentials|
 KFServingClient | [create](#create) | Create KFService|
 KFServingClient | [get](#get)    | Get the specified KFService|
 KFServingClient | [patch](#patch)  | Patch the specified KFService|
 KFServingClient | [delete](#delete) | Delete the specified KFService |
 
+## set_credentials
+> set_credentials(storage_type, namespace=None, credentials_file=None, service_account='kfserving-service-credentials', **kwargs):
+
+Create or update a `Secret` and `Service Account` for GCS and S3 for the provided credentials. Once the `Service Account` is applied, it may be used in the `Service Account` field of a KFService's [V1alpha2ModelSpec](V1alpha2ModelSpec.md).
+
+### Example
+
+Example for creating GCP credentials.
+```python
+from kfserving import KFServingClient
+
+KFServing = KFServingClient()
+KFServing.set_credentials(storage_type='GCS',
+                          namespace='kubeflow',
+                          credentials_file='/tmp/gcp.json',
+                          service_account='user_specified_sa_name')
+```
+
+The API supports specifying a Service Account by `service_account`, or using default Service Account `kfserving-service-credentials`, if the Service Account does not exist, the API will create it and attach the created secret with the Service Account, if exists, only patch it to attach the created Secret.
+
+Example for creating S3 credentials.
+```python
+from kfserving import KFServingClient
+
+KFServing = KFServingClient()
+KFServing.set_credentials(storage_type='S3',
+                          namespace='kubeflow',
+                          credentials_file='/tmp/awcredentials',
+                          s3_profile='default',
+                          s3_endpoint='s3.us-west-amazonaws.com',
+                          s3_region='us-west-2',
+                          s3_use_https='1',
+                          s3_verify_ssl='0')
+```
+
+The created or patched `Secret` and `Service Account` will be shown as following:
+```
+INFO:kfserving.api.set_credentials:Created Secret: kfserving-secret-6tv6l in namespace kubeflow
+INFO:kfserving.api.set_credentials:Created (or Patched) Service account: kfserving-service-credentials in namespace kubeflow
+```
+
+### Parameters
+Name | Type | Storage Type | Description
+------------ | ------------- | ------------- | -------------
+storage_type | str | All |Required. Valid values: GCS or S3 |
+namespace | str | All |Optional. The kubernetes namespace. Defaults to current or default namespace.|
+credentials_file | str | All |Optional. The path for the GCS or S3 credentials file. The default file for GCS is `~/.config/gcloud/application_default_credentials.json`, and default file for S3 is `~/.aws/credentials`. |
+service_account  | str | All |Optional. The name of service account. Supports specifying the `service_account`, or using default Service Account `kfserving-service-credentials`. If the Service Account does not exist, the API will create it and attach the created Secret with the Service Account, if exists, only patch it to attach the created Secret.|
+s3_endpoint  | str | S3 only |Optional. The S3 endpoint. |
+s3_region  | str | S3 only|Optional. The S3 region By default, regional endpoint is used for S3.| |
+s3_use_https  | str | S3 only |Optional. HTTPS is used to access S3 by default, unless `s3_use_https=0` |
+s3_verify_ssl  | str | S3 only|Optional. If HTTPS is used, SSL verification could be disabled with `s3_verify_ssl=0` |
+
+
 ## create
 > create(kfservice, namespace=None)
 

python/kfserving/kfserving/api/creds_utils.py

@@ -0,0 +1,220 @@
+# Copyright 2019 The Kubeflow Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import json
+import configparser
+from os.path import expanduser
+
+from kubernetes import client
+from ..constants import constants
+
+logger = logging.getLogger(__name__)
+
+
+def set_gcs_credentials(namespace, credentials_file, service_account):
+    '''
+    Set GCS Credentails (secret and service account) with credentials file.
+    Args:
+        namespace(str): The kubernetes namespace.
+        credentials_file(str): The path for the gcs credentials file.
+        service_account(str): The name of service account. If the service_account
+                              is specified, will attach created secret with the service account,
+                              otherwise will create new one and attach with created secret.
+    '''
+
+    with open(expanduser(credentials_file)) as f:
+        gcs_creds_content = f.read()
+
+    # Try to get GCS creds file name from configmap, set default value then if cannot.
+    gcs_creds_file_name = get_creds_name_from_config_map(
+        'gcsCredentialFileName')
+    if not gcs_creds_file_name:
+        gcs_creds_file_name = constants.GCS_CREDS_FILE_DEFAULT_NAME
+
+    string_data = {gcs_creds_file_name: gcs_creds_content}
+
+    secret_name = create_secret(
+        namespace=namespace, string_data=string_data)
+
+    set_service_account(namespace=namespace,
+                        service_account=service_account,
+                        secret_name=secret_name)
+
+
+def set_s3_credentials(namespace, credentials_file, service_account, s3_profile='default',  # pylint: disable=too-many-locals,too-many-arguments
+                       s3_endpoint=None, s3_region=None, s3_use_https=None, s3_verify_ssl=None):  # pylint: disable=unused-argument
+    '''
+    Set S3 Credentails (secret and service account).
+    Args:
+        namespace(str): The kubernetes namespace.
+        credentials_file(str): The path for the S3 credentials file.
+        s3_profile(str): The profile for S3, default value is 'default'.
+        service_account(str): The name of service account(Optional). If the service_account
+                              is specified, will attach created secret with the service account,
+                              otherwise will create new one and attach with created secret.
+        s3_endpoint(str): S3 settings variable S3_ENDPOINT.
+        s3_region(str): S3 settings variable AWS_REGION.
+        s3_use_https(str): S3 settings variable S3_USE_HTTPS.
+        s3_verify_ssl(str): S3 settings variable S3_VERIFY_SSL.
+    '''
+
+    config = configparser.ConfigParser()
+    config.read([expanduser(credentials_file)])
+    s3_access_key_id = config.get(s3_profile, 'aws_access_key_id')
+    s3_secret_access_key = config.get(
+        s3_profile, 'aws_secret_access_key')
+
+    # Try to get S3 creds name from configmap, set default value then if cannot.
+    s3_access_key_id_name = get_creds_name_from_config_map(
+        's3AccessKeyIDName')
+    if not s3_access_key_id_name:
+        s3_access_key_id_name = constants.S3_ACCESS_KEY_ID_DEFAULT_NAME
+
+    s3_secret_access_key_name = get_creds_name_from_config_map(
+        's3SecretAccessKeyName')
+    if not s3_secret_access_key_name:
+        s3_secret_access_key_name = constants.S3_SECRET_ACCESS_KEY_DEFAULT_NAME
+
+    data = {
+        s3_access_key_id_name: s3_access_key_id,
+        s3_secret_access_key_name: s3_secret_access_key,
+    }
+
+    s3_cred_sets = {
+        's3_endpoint': constants.KFSERVING_GROUP + "/s3-endpoint",
+        's3_region': constants.KFSERVING_GROUP + "/s3-region",
+        's3_use_https': constants.KFSERVING_GROUP + "/s3-usehttps",
+        's3_verify_ssl': constants.KFSERVING_GROUP + "/s3-verifyssl",
+    }
+
+    s3_annotations = {}
+    for key, value in s3_cred_sets.items():
+        arg = vars()[key]
+        if arg is not None:
+            s3_annotations.update({value: arg})
+
+    secret_name = create_secret(
+        namespace=namespace, annotations=s3_annotations, data=data)
+
+    set_service_account(namespace=namespace,
+                        service_account=service_account,
+                        secret_name=secret_name)
+
+
+def create_secret(namespace, annotations=None, data=None, string_data=None):
+    'Create namespaced secret, and return the secret name.'
+    try:
+        created_secret = client.CoreV1Api().create_namespaced_secret(
+            namespace,
+            client.V1Secret(
+                api_version='v1',
+                kind='Secret',
+                metadata=client.V1ObjectMeta(
+                    generate_name=constants.DEFAULT_SECRET_NAME,
+                    annotations=annotations),
+                data=data,
+                string_data=string_data))
+    except client.rest.ApiException as e:
+        raise RuntimeError(
+            "Exception when calling CoreV1Api->create_namespaced_secret: %s\n" % e)
+
+    secret_name = created_secret.metadata.name
+    logger.info('Created Secret: %s in namespace %s', secret_name, namespace)
+    return secret_name
+
+
+def set_service_account(namespace, service_account, secret_name):
+    '''Set service account, create if service_account does not exist, otherwise patch it.'''
+    if check_sa_exists(namespace=namespace, service_account=service_account):
+        patch_service_account(secret_name=secret_name,
+                              namespace=namespace,
+                              sa_name=service_account)
+    else:
+        create_service_account(secret_name=secret_name,
+                               namespace=namespace,
+                               sa_name=service_account)
+
+
+def check_sa_exists(namespace, service_account):
+    '''Check if the specified service account existing.'''
+    sa_list = client.CoreV1Api().list_namespaced_service_account(namespace=namespace)
+
+    sa_name_list = []
+    for item in range(0, len(sa_list.items)-1):
+        sa_name_list.append(sa_list.items[item].metadata.name)
+
+    if service_account in sa_name_list:
+        return True
+
+    return False
+
+
+def create_service_account(secret_name, namespace, sa_name):
+    'Create namespaced service account, and return the service account name'
+    try:
+        client.CoreV1Api().create_namespaced_service_account(
+            namespace,
+            client.V1ServiceAccount(
+                metadata=client.V1ObjectMeta(
+                    name=sa_name
+                ),
+                secrets=[client.V1ObjectReference(
+                    kind='Secret',
+                    name=secret_name)]))
+    except client.rest.ApiException as e:
+        raise RuntimeError(
+            "Exception when calling CoreV1Api->create_namespaced_service_account: %s\n" % e)
+
+    logger.info('Created Service account: %s in namespace %s', sa_name, namespace)
+
+
+def patch_service_account(secret_name, namespace, sa_name):
+    'Patch namespaced service account to attach with created secret.'
+    try:
+        client.CoreV1Api().patch_namespaced_service_account(
+            sa_name,
+            namespace,
+            client.V1ServiceAccount(
+                secrets=[client.V1ObjectReference(
+                    kind='Secret',
+                    name=secret_name)]))
+    except client.rest.ApiException as e:
+        raise RuntimeError(
+            "Exception when calling CoreV1Api->patch_namespaced_service_account: %s\n" % e)
+
+    logger.info('Pacthed Service account: %s in namespace %s', sa_name, namespace)
+
+
+def get_creds_name_from_config_map(creds):
+    '''Get the credentials name from kfservice config map.'''
+    try:
+        kfsvc_config_map = client.CoreV1Api().read_namespaced_config_map(
+            constants.KFSERVICE_CONFIG_MAP_NAME,
+            constants.KFSERVICE_SYSTEM_NAMESPACE)
+    except client.rest.ApiException as e:
+        raise RuntimeError(
+            "Exception when calling CoreV1Api->read_namespaced_config_map: %s\n" % e)
+
+    kfsvc_creds_str = kfsvc_config_map.data['credentials']
+    kfsvc_creds_json = json.loads(kfsvc_creds_str)
+
+    if creds == 'gcsCredentialFileName':
+        return kfsvc_creds_json['gcs']['gcsCredentialFileName']
+    elif creds == 's3AccessKeyIDName':
+        return kfsvc_creds_json['s3']['s3AccessKeyIDName']
+    elif creds == 's3SecretAccessKeyName':
+        return kfsvc_creds_json['s3']['s3SecretAccessKeyName']
+    else:
+        raise RuntimeError("Unknown credentials.")

python/kfserving/kfserving/api/kf_serving_client.py

@@ -16,10 +16,11 @@
 
 from ..constants import constants
 from ..utils import utils
+from .creds_utils import set_gcs_credentials, set_s3_credentials
 
 
 class KFServingClient(object):
-    """KFServing Apis."""
+    '''KFServing Client Apis.'''
 
     def __init__(self, config_file=None, context=None,
                  client_configuration=None, persist_config=True):
@@ -34,6 +35,38 @@ def __init__(self, config_file=None, context=None,
 
         self.api_instance = client.CustomObjectsApi()
 
+    def set_credentials(self, storage_type, namespace=None, credentials_file=None,
+                        service_account=constants.DEFAULT_SA_NAME, **kwargs):
+        '''
+        Set GCS and S3 Credentials for KFServing.
+        Args:
+            storage_type(str): Valid value: GCS or S3 (required).
+            namespace(str): The kubenertes namespace (Optional).
+            credentials_file(str): The path for the credentials file.
+            service_account(str): The name of service account.
+            kwargs(dict): Others parameters for each storage_type.
+        '''
+        if namespace is None:
+            namespace = utils.get_default_target_namespace()
+
+        if storage_type.lower() == 'gcs':
+            if credentials_file is None:
+                credentials_file = constants.GCS_DEFAULT_CREDS_FILE
+            set_gcs_credentials(namespace=namespace,
+                                credentials_file=credentials_file,
+                                service_account=service_account)
+        elif storage_type.lower() == 's3':
+            if credentials_file is None:
+                credentials_file = constants.S3_DEFAULT_CREDS_FILE
+            set_s3_credentials(namespace=namespace,
+                               credentials_file=credentials_file,
+                               service_account=service_account,
+                               **kwargs)
+        else:
+            raise RuntimeError("Invalid storage_type: %s, only support GCS and S3\
+                currently.\n" % storage_type)
+
+
     def create(self, kfservice, namespace=None):
         """Create the provided KFService in the specified namespace"""
 

python/kfserving/kfserving/constants/constants.py

@@ -3,3 +3,18 @@
 KFSERVING_KIND = "KFService"
 KFSERVING_PLURAL = "kfservices"
 KFSERVING_VERSION = "v1alpha2"
+
+# KFservice credentials common constants
+KFSERVICE_CONFIG_MAP_NAME = 'kfservice-config'
+KFSERVICE_SYSTEM_NAMESPACE = 'kfserving-system'
+DEFAULT_SECRET_NAME = "kfserving-secret-"
+DEFAULT_SA_NAME = "kfserving-service-credentials"
+
+# S3 credentials constants
+S3_ACCESS_KEY_ID_DEFAULT_NAME = "awsAccessKeyID"
+S3_SECRET_ACCESS_KEY_DEFAULT_NAME = "awsSecretAccessKey"
+S3_DEFAULT_CREDS_FILE = '~/.aws/credentials'
+
+# GCS credentials constants
+GCS_CREDS_FILE_DEFAULT_NAME = 'gcloud-application-credentials.json'
+GCS_DEFAULT_CREDS_FILE = '~/.config/gcloud/application_default_credentials.json'

python/kfserving/test/aws_credentials

@@ -0,0 +1,3 @@
+[default]
+aws_access_key_id = ARQ7AKIBFOFN6F6WXBSZ
+aws_secret_access_key = 4tdCyzIxlbPzoBfQGbAPw2hY+Rkzgv/b4vB0Q+ps

python/kfserving/test/gcp_credentials.json

@@ -0,0 +1,6 @@
+{
+    "client_id": "760518506408-6qr4p6gpi6hn506pt8ejuq83di341hur.apps.googleusercontent.com",
+    "client_secret": "d-FL95Q19q7MQmHD0TyFpd7h",
+    "refresh_token": "1/bqYmkxnDbxEstG12XcmOZrN0-eyI3beanJbRd4kqs6g",
+    "type": "authorized_user"
+}

python/kfserving/test/test_kfservice_client.py

@@ -53,7 +53,7 @@ def test_kfservice_client_get():
         assert mocked_unit_result == KFServing.get('flower-sample', namespace='kubeflow')
 
 # Unit test for kfserving patch api
-def test_kfservice_clienti_patch():
+def test_kfservice_client_patch():
     with patch('kfserving.api.kf_serving_client.KFServingClient.patch',
                return_value=mocked_unit_result):
         kfsvc = generate_kfservice()