From bd462d102144ce773b04f199f8229119fdd9b02e Mon Sep 17 00:00:00 2001 From: Jiaxin Shan Date: Thu, 16 Apr 2020 17:17:10 -0700 Subject: [PATCH] Infer artifact store endpoint in metadata writer Signed-off-by: Jiaxin Shan --- backend/metadata_writer/src/metadata_writer.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/backend/metadata_writer/src/metadata_writer.py b/backend/metadata_writer/src/metadata_writer.py index 4d27ede2cee..a48f6bc63b5 100644 --- a/backend/metadata_writer/src/metadata_writer.py +++ b/backend/metadata_writer/src/metadata_writer.py @@ -16,6 +16,7 @@ import hashlib import os import sys +import re import kubernetes import yaml from time import sleep @@ -83,11 +84,21 @@ def output_name_to_argo(name: str) -> str: import re return re.sub('-+', '-', re.sub('[^-0-9a-z]+', '-', name.lower())).strip('-') +def is_s3_endpoint(endpoint: str) -> bool: + return re.search('^.*s3.*amazonaws.com.*$', endpoint) + +def get_object_store_provider(endpoint: str) -> bool: + if is_s3_endpoint(endpoint): + return 's3' + else: + return 'minio' def argo_artifact_to_uri(artifact: dict) -> str: + # s3 here means s3 compatible object storage. not AWS S3. if 's3' in artifact: s3_artifact = artifact['s3'] - return 'minio://{bucket}/{key}'.format( + return '{provider}://{bucket}/{key}'.format( + provider=get_object_store_provider(s3_artifact['endpoint']), bucket=s3_artifact.get('bucket', ''), key=s3_artifact.get('key', ''), )