From e0c41a757ad4b854b40c901f5cef0d4c201c0fb9 Mon Sep 17 00:00:00 2001 From: Li Wan Date: Wed, 8 Feb 2023 10:50:18 +1100 Subject: [PATCH 01/14] bug fix --- src/marqo/s2_inference/processing/custom_clip_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/marqo/s2_inference/processing/custom_clip_utils.py b/src/marqo/s2_inference/processing/custom_clip_utils.py index 32723f520..10f57a89e 100644 --- a/src/marqo/s2_inference/processing/custom_clip_utils.py +++ b/src/marqo/s2_inference/processing/custom_clip_utils.py @@ -6,7 +6,7 @@ import os import urllib from tqdm import tqdm -from src.marqo.s2_inference.configs import ModelCache +from marqo.s2_inference.configs import ModelCache def whitespace_clean(text): text = re.sub(r'\s+', ' ', text) text = text.strip() From 36a91b6563295318b50a6af76d5b5b79dae099a0 Mon Sep 17 00:00:00 2001 From: Li Wan Date: Wed, 8 Feb 2023 10:51:43 +1100 Subject: [PATCH 02/14] bug fix --- src/marqo/s2_inference/processing/custom_clip_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/marqo/s2_inference/processing/custom_clip_utils.py b/src/marqo/s2_inference/processing/custom_clip_utils.py index 10f57a89e..32723f520 100644 --- a/src/marqo/s2_inference/processing/custom_clip_utils.py +++ b/src/marqo/s2_inference/processing/custom_clip_utils.py @@ -6,7 +6,7 @@ import os import urllib from tqdm import tqdm -from marqo.s2_inference.configs import ModelCache +from src.marqo.s2_inference.configs import ModelCache def whitespace_clean(text): text = re.sub(r'\s+', ' ', text) text = text.strip() From fc49e9d8f510563fff4a15eddeece55a3c79a795 Mon Sep 17 00:00:00 2001 From: Li Wan Date: Wed, 8 Feb 2023 10:52:32 +1100 Subject: [PATCH 03/14] bug fix - remove src in import --- src/marqo/s2_inference/processing/custom_clip_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/marqo/s2_inference/processing/custom_clip_utils.py b/src/marqo/s2_inference/processing/custom_clip_utils.py index 32723f520..10f57a89e 100644 --- a/src/marqo/s2_inference/processing/custom_clip_utils.py +++ b/src/marqo/s2_inference/processing/custom_clip_utils.py @@ -6,7 +6,7 @@ import os import urllib from tqdm import tqdm -from src.marqo.s2_inference.configs import ModelCache +from marqo.s2_inference.configs import ModelCache def whitespace_clean(text): text = re.sub(r'\s+', ' ', text) text = text.strip() From 105c43f06716e269c826070919576cea68af797c Mon Sep 17 00:00:00 2001 From: Li Wan Date: Wed, 8 Feb 2023 12:44:34 +1100 Subject: [PATCH 04/14] bug fix --- src/marqo/s2_inference/s2_inference.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/marqo/s2_inference/s2_inference.py b/src/marqo/s2_inference/s2_inference.py index 5d51645f2..c9605d4f2 100644 --- a/src/marqo/s2_inference/s2_inference.py +++ b/src/marqo/s2_inference/s2_inference.py @@ -66,7 +66,7 @@ def _create_model_cache_key(model_name: str, device: str, model_properties: dict if model_properties is None: model_properties = dict() - model_cache_key = (model_name + "||" + + model_cache_key = (model_name + "||" + model_properties.get('name', '') + "||" + str(model_properties.get('dimensions', '')) + "||" + model_properties.get('type', '') + "||" + @@ -317,6 +317,7 @@ def eject_model(model_name:str, device:str): # we can't handle the situation where there are two models with the same name and device # but different properties. + logger.info(f"{model_cache_keys}") for key in model_cache_keys: if key.startswith(model_name) and key.endswith(device): model_cache_key = key From fffe58f8811dc322a61a1d8bb812e81796bf8fa3 Mon Sep 17 00:00:00 2001 From: Li Wan Date: Wed, 8 Feb 2023 13:53:05 +1100 Subject: [PATCH 05/14] bug fix - work only on str model cache key --- src/marqo/s2_inference/s2_inference.py | 9 ++++++--- src/marqo/tensor_search/tensor_search.py | 11 ++++++----- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/marqo/s2_inference/s2_inference.py b/src/marqo/s2_inference/s2_inference.py index c9605d4f2..44e9cfe38 100644 --- a/src/marqo/s2_inference/s2_inference.py +++ b/src/marqo/s2_inference/s2_inference.py @@ -319,9 +319,12 @@ def eject_model(model_name:str, device:str): # but different properties. logger.info(f"{model_cache_keys}") for key in model_cache_keys: - if key.startswith(model_name) and key.endswith(device): - model_cache_key = key - break + if isinstance(key, str): + if key.startswith(model_name) and key.endswith(device): + model_cache_key = key + break + else: + continue if model_cache_key is None: raise ModelNotInCacheError(f"The model_name `{model_name}` device `{device}` is not cached or found") diff --git a/src/marqo/tensor_search/tensor_search.py b/src/marqo/tensor_search/tensor_search.py index 1f0371d7b..111dad7a6 100644 --- a/src/marqo/tensor_search/tensor_search.py +++ b/src/marqo/tensor_search/tensor_search.py @@ -1368,13 +1368,14 @@ def _get_model_properties(index_info): return model_properties + def get_loaded_models() -> dict: available_models = s2_inference.get_available_models() - message = { - "models" : [ - {"model_name": ix.split("||")[0], "model_device": ix.split("||")[-1]} for ix in available_models.keys() - ] - } + message = {"models":[]} + + for ix in available_models: + if isinstance(ix, str): + message["models"].append({"model_name": ix.split("||")[0], "model_device": ix.split("||")[-1]}) return message From 284888364f1413f972ff9efdaa3a08cc89cc25ed Mon Sep 17 00:00:00 2001 From: Li Wan Date: Wed, 8 Feb 2023 14:15:21 +1100 Subject: [PATCH 06/14] bug fix - work only on str model cache key --- src/marqo/s2_inference/reranking/model_utils.py | 2 +- tests/tensor_search/test_model_cache_management.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/marqo/s2_inference/reranking/model_utils.py b/src/marqo/s2_inference/reranking/model_utils.py index 9c2890073..6ff972bde 100644 --- a/src/marqo/s2_inference/reranking/model_utils.py +++ b/src/marqo/s2_inference/reranking/model_utils.py @@ -309,7 +309,7 @@ def load_owl_vit(model_name: str, device: str = 'cpu') -> Dict: Dict: _description_ """ - model_cache_key = (model_name, device) + model_cache_key = _create_model_cache_key(model_name, device) if model_cache_key in available_models: logger.info(f"loading {model_cache_key} from cache...") diff --git a/tests/tensor_search/test_model_cache_management.py b/tests/tensor_search/test_model_cache_management.py index dbe9f7494..e833aff58 100644 --- a/tests/tensor_search/test_model_cache_management.py +++ b/tests/tensor_search/test_model_cache_management.py @@ -20,6 +20,7 @@ def setUp(self) -> None: # We pre-define 3 dummy models for testing purpose self.MODEL_1 = "ViT-B/32" self.MODEL_2 = "hf/all-MiniLM-L6-v2" + self.rerank_model = self.MODEL_LIST = [self.MODEL_1, self.MODEL_2] self.CUDA_FLAG = torch.cuda.is_available() From e6b3cf8a47849e2551852a6cab922cbe292d6356 Mon Sep 17 00:00:00 2001 From: Li Wan Date: Wed, 8 Feb 2023 14:17:56 +1100 Subject: [PATCH 07/14] bug fix - work only on str model cache key --- tests/tensor_search/test_model_cache_management.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tensor_search/test_model_cache_management.py b/tests/tensor_search/test_model_cache_management.py index e833aff58..a3cd38dda 100644 --- a/tests/tensor_search/test_model_cache_management.py +++ b/tests/tensor_search/test_model_cache_management.py @@ -20,7 +20,7 @@ def setUp(self) -> None: # We pre-define 3 dummy models for testing purpose self.MODEL_1 = "ViT-B/32" self.MODEL_2 = "hf/all-MiniLM-L6-v2" - self.rerank_model = + self.rerank_model = "" self.MODEL_LIST = [self.MODEL_1, self.MODEL_2] self.CUDA_FLAG = torch.cuda.is_available() From 1141318e1f8837f30d80616119c7ac7ccd7afd68 Mon Sep 17 00:00:00 2001 From: Li Wan Date: Wed, 8 Feb 2023 14:18:26 +1100 Subject: [PATCH 08/14] bug fix - work only on str model cache key --- tests/tensor_search/test_model_cache_management.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tensor_search/test_model_cache_management.py b/tests/tensor_search/test_model_cache_management.py index a3cd38dda..15121ba46 100644 --- a/tests/tensor_search/test_model_cache_management.py +++ b/tests/tensor_search/test_model_cache_management.py @@ -20,7 +20,7 @@ def setUp(self) -> None: # We pre-define 3 dummy models for testing purpose self.MODEL_1 = "ViT-B/32" self.MODEL_2 = "hf/all-MiniLM-L6-v2" - self.rerank_model = "" + self.rerank_model = 'google/owlvit-base-patch32' self.MODEL_LIST = [self.MODEL_1, self.MODEL_2] self.CUDA_FLAG = torch.cuda.is_available() From d33a545cbecb14981db1a60088a77ce0a39ee1e6 Mon Sep 17 00:00:00 2001 From: Li Wan Date: Wed, 8 Feb 2023 14:57:42 +1100 Subject: [PATCH 09/14] bug fix - work only on str model cache key --- tests/tensor_search/test_model_cache_management.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/tensor_search/test_model_cache_management.py b/tests/tensor_search/test_model_cache_management.py index 15121ba46..14b2630a5 100644 --- a/tests/tensor_search/test_model_cache_management.py +++ b/tests/tensor_search/test_model_cache_management.py @@ -21,7 +21,7 @@ def setUp(self) -> None: self.MODEL_1 = "ViT-B/32" self.MODEL_2 = "hf/all-MiniLM-L6-v2" self.rerank_model = 'google/owlvit-base-patch32' - self.MODEL_LIST = [self.MODEL_1, self.MODEL_2] + self.MODEL_LIST = [self.MODEL_1, self.MODEL_2, self.rerank_model] self.CUDA_FLAG = torch.cuda.is_available() @@ -36,9 +36,9 @@ def setUp(self) -> None: # We loaded 6 models (3 in cuda, 3 in cpu) as initial setup if self.CUDA_FLAG: - assert len(available_models) >= 4 + assert len(available_models) >= 6 else: - assert len(available_models) >= 2 + assert len(available_models) >= 3 def tearDown(self) -> None: clear_loaded_models() From e9c5ebd426380ab36bec5c86ed923586d3629752 Mon Sep 17 00:00:00 2001 From: Li Wan Date: Wed, 8 Feb 2023 15:33:12 +1100 Subject: [PATCH 10/14] bug fix - work only on str model cache key --- .../test_model_cache_management.py | 27 ++++++++++++++++--- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/tests/tensor_search/test_model_cache_management.py b/tests/tensor_search/test_model_cache_management.py index 14b2630a5..24681bf4e 100644 --- a/tests/tensor_search/test_model_cache_management.py +++ b/tests/tensor_search/test_model_cache_management.py @@ -4,7 +4,11 @@ _create_model_cache_key, _update_available_models, available_models, clear_loaded_models from marqo.tensor_search.tensor_search import eject_model, get_cuda_info, get_loaded_models, get_cpu_info from marqo.errors import ModelNotInCacheError, HardwareCompatabilityError +from marqo.s2_inference.reranking.cross_encoders import ReRankerText, ReRankerOwl +from marqo.s2_inference.reranking.model_utils import load_owl_vit +from marqo.s2_inference.reranking import rerank import psutil +from marqo.tensor_search import tensor_search @@ -20,8 +24,7 @@ def setUp(self) -> None: # We pre-define 3 dummy models for testing purpose self.MODEL_1 = "ViT-B/32" self.MODEL_2 = "hf/all-MiniLM-L6-v2" - self.rerank_model = 'google/owlvit-base-patch32' - self.MODEL_LIST = [self.MODEL_1, self.MODEL_2, self.rerank_model] + self.MODEL_LIST = [self.MODEL_1, self.MODEL_2] self.CUDA_FLAG = torch.cuda.is_available() @@ -36,9 +39,9 @@ def setUp(self) -> None: # We loaded 6 models (3 in cuda, 3 in cpu) as initial setup if self.CUDA_FLAG: - assert len(available_models) >= 6 + assert len(available_models) >= 4 else: - assert len(available_models) >= 3 + assert len(available_models) >= 2 def tearDown(self) -> None: clear_loaded_models() @@ -260,6 +263,22 @@ def test_overall_eject_and_load_model(self): raise AssertionError + def test_model_cache_management_with_text_reranker(self): + model_name = 'google/owlvit-base-patch32' + + _ = load_owl_vit('google/owlvit-base-patch32', "cpu") + model_cache_key = _create_model_cache_key(model_name, "cpu", model_properties=None) + + if model_cache_key not in available_models: + raise AssertionError + + eject_model(model_name, "cpu") + if model_cache_key in available_models: + raise AssertionError + + + + From 92902c29702bc679fa0ac33389cd225c6d782007 Mon Sep 17 00:00:00 2001 From: Li Wan Date: Wed, 8 Feb 2023 15:35:32 +1100 Subject: [PATCH 11/14] add a reranker test for model-cache-management --- tests/tensor_search/test_model_cache_management.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/tensor_search/test_model_cache_management.py b/tests/tensor_search/test_model_cache_management.py index 24681bf4e..01566f765 100644 --- a/tests/tensor_search/test_model_cache_management.py +++ b/tests/tensor_search/test_model_cache_management.py @@ -19,7 +19,6 @@ def load_model(model_name: str, device: str, model_properteis: dict = None) -> N class TestModelCacheManagement(MarqoTestCase): - def setUp(self) -> None: # We pre-define 3 dummy models for testing purpose self.MODEL_1 = "ViT-B/32" From bf0c2f87f7f7e1d324035a1d2091a2e21cabc298 Mon Sep 17 00:00:00 2001 From: Li Wan Date: Wed, 8 Feb 2023 15:45:12 +1100 Subject: [PATCH 12/14] add a reranker test for model-cache-management --- src/marqo/s2_inference/s2_inference.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/marqo/s2_inference/s2_inference.py b/src/marqo/s2_inference/s2_inference.py index 44e9cfe38..e2b462110 100644 --- a/src/marqo/s2_inference/s2_inference.py +++ b/src/marqo/s2_inference/s2_inference.py @@ -317,7 +317,6 @@ def eject_model(model_name:str, device:str): # we can't handle the situation where there are two models with the same name and device # but different properties. - logger.info(f"{model_cache_keys}") for key in model_cache_keys: if isinstance(key, str): if key.startswith(model_name) and key.endswith(device): From f73555ed79148762177ccbfba329df564c58c351 Mon Sep 17 00:00:00 2001 From: Li Wan Date: Wed, 8 Feb 2023 16:53:18 +1100 Subject: [PATCH 13/14] update the model_cache_key as string to image preprocess --- src/marqo/s2_inference/processing/image.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/marqo/s2_inference/processing/image.py b/src/marqo/s2_inference/processing/image.py index dd0b1050e..8f4705b69 100644 --- a/src/marqo/s2_inference/processing/image.py +++ b/src/marqo/s2_inference/processing/image.py @@ -5,7 +5,7 @@ import torch import torchvision -from marqo.s2_inference.s2_inference import available_models +from marqo.s2_inference.s2_inference import available_models,_create_model_cache_key from marqo.s2_inference.s2_inference import get_logger from marqo.s2_inference.types import Dict, List, Union, ImageType, Tuple, ndarray, Literal from marqo.s2_inference.clip_utils import format_and_load_CLIP_image @@ -200,8 +200,9 @@ def _get_model_specific_parameters(self): def _load_and_cache_model(self): model_type = (self.model_name, self.device) + model_cache_key = _create_model_cache_key(self.model_name, self.device) - if model_type not in available_models: + if model_cache_key not in available_models: logger.info(f"loading model {model_type}") if model_type[0] in self.allowed_model_types: func = self.model_load_function @@ -210,9 +211,9 @@ def _load_and_cache_model(self): self.model, self.preprocess = func(self.model_name, self.device) - available_models[model_type] = (self.model, self.preprocess) + available_models[model_cache_key] = model_type else: - self.model, self.preprocess = available_models[model_type] + self.model, self.preprocess = available_models[model_cache_key] def _load_image(self, image): self.image, self.image_pt, self.original_size = load_rcnn_image(image, size=self.size) From e36719b8db28f9f8e5fa35e8754d8c22827f4a5c Mon Sep 17 00:00:00 2001 From: Li Wan Date: Wed, 8 Feb 2023 17:10:59 +1100 Subject: [PATCH 14/14] update the model_cache_key as string to image preprocess --- src/marqo/s2_inference/processing/image.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/marqo/s2_inference/processing/image.py b/src/marqo/s2_inference/processing/image.py index 8f4705b69..95bd9b157 100644 --- a/src/marqo/s2_inference/processing/image.py +++ b/src/marqo/s2_inference/processing/image.py @@ -211,7 +211,7 @@ def _load_and_cache_model(self): self.model, self.preprocess = func(self.model_name, self.device) - available_models[model_cache_key] = model_type + available_models[model_cache_key] = self.model, self.preprocess else: self.model, self.preprocess = available_models[model_cache_key]