Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Replicas limit #465

Merged
merged 5 commits into from
May 8, 2023
Merged

Replicas limit #465

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
e64b33d
try to mock
wanliAlex May 5, 2023
1ab5971
add tests
wanliAlex May 5, 2023
3cc0244
use read_env_vars_and_defaults_ints
wanliAlex May 8, 2023
2ab362e
add new a test
wanliAlex May 8, 2023
8f66941
add new a test
wanliAlex May 8, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion src/marqo/tensor_search/configs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ def default_env_vars() -> dict:
EnvVars.MARQO_EF_CONSTRUCTION_MAX_VALUE: 4096,
EnvVars.MARQO_MAX_VECTORISE_BATCH_SIZE: 16,
EnvVars.MARQO_MAX_DELETE_DOCS_COUNT: 10000,
EnvVars.MARQO_MAX_SEARCHABLE_TENSOR_ATTRIBUTES: None
EnvVars.MARQO_MAX_SEARCHABLE_TENSOR_ATTRIBUTES: None,
EnvVars.MARQO_MAX_NUMBER_OF_REPLICAS: 1,
}

1 change: 1 addition & 0 deletions src/marqo/tensor_search/enums.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,7 @@ class EnvVars:
MARQO_MAX_VECTORISE_BATCH_SIZE = "MARQO_MAX_VECTORISE_BATCH_SIZE"
MARQO_MAX_SEARCHABLE_TENSOR_ATTRIBUTES = "MARQO_MAX_SEARCHABLE_TENSOR_ATTRIBUTES"
MARQO_MAX_DELETE_DOCS_COUNT = "MARQO_MAX_DELETE_DOCS_COUNT"
MARQO_MAX_NUMBER_OF_REPLICAS = "MARQO_MAX_NUMBER_OF_REPLICAS"


class RequestType:
Expand Down
5 changes: 3 additions & 2 deletions src/marqo/tensor_search/models/settings_object.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from marqo.tensor_search import enums as ns_enums
from marqo.tensor_search.enums import IndexSettingsField as NsFields, EnvVars
from marqo.tensor_search.utils import read_env_vars_and_defaults
from marqo.tensor_search.utils import read_env_vars_and_defaults, read_env_vars_and_defaults_ints

settings_schema = {
"$schema": "https://json-schema.org/draft/2019-09/schema",
Expand Down Expand Up @@ -128,7 +128,7 @@
NsFields.hnsw_ef_construction: {
"type": "integer",
"minimum": 1,
"maximum": int(read_env_vars_and_defaults(EnvVars.MARQO_EF_CONSTRUCTION_MAX_VALUE)),
"maximum": read_env_vars_and_defaults_ints(EnvVars.MARQO_EF_CONSTRUCTION_MAX_VALUE),
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

may be the wrong PR to comment this on, but maybe we should rename MARQO_EF_CONSTRUCTION_MAX_VALUE to MARQO_MAX_EF_CONSTRUCTION_VALUE, because if you observe the other env vars, the word MAX always comes first. It may confuse users if we keep EF like that.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

good thinking, to keep things consistent, but that would technically be a breaking change

"examples": [
128
]
Expand Down Expand Up @@ -192,6 +192,7 @@
NsFields.number_of_replicas: {
"type": "integer",
"minimum": 0,
"maximum": read_env_vars_and_defaults_ints(EnvVars.MARQO_MAX_NUMBER_OF_REPLICAS),
"examples": [
1
]
Expand Down
122 changes: 116 additions & 6 deletions tests/tensor_search/test_create_index.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
import pprint
from typing import Any, Dict
import pytest
import os
from unittest.mock import patch
import requests
from marqo.tensor_search.enums import IndexSettingsField, EnvVars
from marqo.errors import MarqoApiError, MarqoError, IndexNotFoundError
Expand All @@ -11,6 +10,7 @@
from marqo.tensor_search.enums import IndexSettingsField as NsField
from unittest import mock
from marqo import errors
from marqo.errors import InvalidArgError

class TestCreateIndex(MarqoTestCase):

Expand Down Expand Up @@ -324,24 +324,134 @@ def test_set_number_of_shards(self):
assert intended_shard_count == int(resp.json()[self.index_name_1]['settings']['index']['number_of_shards'])

def test_set_number_of_replicas(self):
""" does it work if other params are filled?"""
intended_replicas_count = 4
res_0 = tensor_search.create_vector_index(
from marqo.tensor_search.models.settings_object import settings_schema
with patch.dict(settings_schema["properties"][NsField.number_of_replicas], maximum=10):
res_0 = tensor_search.create_vector_index(
index_name=self.index_name_1, config=self.config,
index_settings={
"index_defaults": {
"treat_urls_and_pointers_as_images": True,
"model": "ViT-B/32",
},
NsField.number_of_replicas: intended_replicas_count
}
)
resp = requests.get(
url=self.authorized_url + f"/{self.index_name_1}",
headers=self.generic_header,
verify=False
)
assert intended_replicas_count == int(resp.json()[self.index_name_1]['settings']['index']['number_of_replicas'])

def test_configurable_max_number_of_replicas(self):
maximum_number_of_replicas = 5
large_intended_replicas_count = 10
small_intended_replicas_count = 3
from marqo.tensor_search.models.settings_object import settings_schema

with patch.dict(settings_schema["properties"][NsField.number_of_replicas], maximum=maximum_number_of_replicas):
# a large value exceeding limits should not work
try:
res_0 = tensor_search.create_vector_index(
index_name=self.index_name_1, config=self.config,
index_settings={
"index_defaults": {
"treat_urls_and_pointers_as_images": True,
"model": "ViT-B/32",
},
NsField.number_of_replicas: large_intended_replicas_count
}
)
raise AssertionError
except InvalidArgError as e:
pass

try:
tensor_search.delete_index(config=self.config, index_name=self.index_name_1)
except IndexNotFoundError:
pass

# a small value should work
res_1 = tensor_search.create_vector_index(
index_name=self.index_name_1, config=self.config,
index_settings={
"index_defaults": {
"treat_urls_and_pointers_as_images": True,
"model": "ViT-B/32",
},
NsField.number_of_replicas: small_intended_replicas_count
}
)
resp = requests.get(
url=self.authorized_url + f"/{self.index_name_1}",
headers=self.generic_header,
verify=False
)
assert small_intended_replicas_count == int(
resp.json()[self.index_name_1]['settings']['index']['number_of_replicas'])

try:
tensor_search.delete_index(config=self.config, index_name=self.index_name_1)
except IndexNotFoundError:
pass

# the same number should also work
res_1 = tensor_search.create_vector_index(
index_name=self.index_name_1, config=self.config,
index_settings={
"index_defaults": {
"treat_urls_and_pointers_as_images": True,
"model": "ViT-B/32",
},
NsField.number_of_replicas: maximum_number_of_replicas
}
)
resp = requests.get(
url=self.authorized_url + f"/{self.index_name_1}",
headers=self.generic_header,
verify=False
)
assert maximum_number_of_replicas == int(
resp.json()[self.index_name_1]['settings']['index']['number_of_replicas'])

def test_default_max_number_of_replicas(self):
large_intended_replicas_count = 2
small_intended_replicas_count = 0
# a large value exceeding limits should not work
try:
res_0 = tensor_search.create_vector_index(
index_name=self.index_name_1, config=self.config,
index_settings={
"index_defaults": {
"treat_urls_and_pointers_as_images": True,
"model": "ViT-B/32",
},
NsField.number_of_replicas: large_intended_replicas_count
}
)
raise AssertionError
except InvalidArgError as e:
pass

# a small value should work
res_1 = tensor_search.create_vector_index(
index_name=self.index_name_1, config=self.config,
index_settings={
"index_defaults": {
"treat_urls_and_pointers_as_images": True,
"model": "ViT-B/32",
},
NsField.number_of_replicas: intended_replicas_count
NsField.number_of_replicas: small_intended_replicas_count
}
)
resp = requests.get(
url=self.authorized_url + f"/{self.index_name_1}",
headers=self.generic_header,
verify=False
)
assert intended_replicas_count == int(resp.json()[self.index_name_1]['settings']['index']['number_of_replicas'])
assert small_intended_replicas_count == int(
resp.json()[self.index_name_1]['settings']['index']['number_of_replicas'])

def test_field_limits(self):
index_limits = [1, 5, 10, 100, 1000]
Expand Down