From ef01e57f30ea3bbfba171e9c18940cfb07381a74 Mon Sep 17 00:00:00 2001 From: Xiang Yan Date: Thu, 4 Mar 2021 16:45:51 -0800 Subject: [PATCH 1/6] search perf tests --- .../tests/perfstress_tests/README.md | 44 +++++++++++++++++++ .../tests/perfstress_tests/__init__.py | 0 .../tests/perfstress_tests/autocomplete.py | 32 ++++++++++++++ .../perfstress_tests/search_documents.py | 36 +++++++++++++++ .../tests/perfstress_tests/suggest.py | 32 ++++++++++++++ 5 files changed, 144 insertions(+) create mode 100644 sdk/search/azure-search-documents/tests/perfstress_tests/README.md create mode 100644 sdk/search/azure-search-documents/tests/perfstress_tests/__init__.py create mode 100644 sdk/search/azure-search-documents/tests/perfstress_tests/autocomplete.py create mode 100644 sdk/search/azure-search-documents/tests/perfstress_tests/search_documents.py create mode 100644 sdk/search/azure-search-documents/tests/perfstress_tests/suggest.py diff --git a/sdk/search/azure-search-documents/tests/perfstress_tests/README.md b/sdk/search/azure-search-documents/tests/perfstress_tests/README.md new file mode 100644 index 000000000000..00b98e2bc2b1 --- /dev/null +++ b/sdk/search/azure-search-documents/tests/perfstress_tests/README.md @@ -0,0 +1,44 @@ +# Search Performance Tests + +In order to run the performance tests, the `azure-devtools` package must be installed. This is done as part of the `dev_requirements`. +Start be creating a new virtual environment for your perf tests. This will need to be a Python 3 environment, preferably >=3.7. + +### Setup for test resources + +These tests will run against a pre-configured search service. The following environment variable will need to be set for the tests to access the live resources: +``` +AZURE_SEARCH_API_KEY= +AZURE_SEARCH_SERVICE_ENDPOINT= +AZURE_SEARCH_INDEX_NAME= +``` + +### Setup for perf test runs + +```cmd +(env) ~/search/azure-search-documents> pip install -r dev_requirements.txt +(env) ~/search/azure-search-documents> pip install -e . +``` + +## Test commands + +When `azure-devtools` is installed, you will have access to the `perfstress` command line tool, which will scan the current module for runable perf tests. Only a specific test can be run at a time (i.e. there is no "run all" feature). + +```cmd +(env) ~/search/azure-search-documents> cd tests +(env) ~/search/azure-search-documents/tests> perfstress +``` +Using the `perfstress` command alone will list the available perf tests found. + +### Common perf command line options +These options are available for all perf tests: +- `--duration=10` Number of seconds to run as many operations (the "run" function) as possible. Default is 10. +- `--iterations=1` Number of test iterations to run. Default is 1. +- `--parallel=1` Number of tests to run in parallel. Default is 1. +- `--warm-up=5` Number of seconds to spend warming up the connection before measuring begins. Default is 5. +- `--sync` Whether to run the tests in sync or async. Default is False (async). This flag must be used for Storage legacy tests, which do not support async. +- `--no-cleanup` Whether to keep newly created resources after test run. Default is False (resources will be deleted). + +## Example command +```cmd +(env) ~/search/azure-search-documents/tests> perfstress SearchDocumentsTest +``` diff --git a/sdk/search/azure-search-documents/tests/perfstress_tests/__init__.py b/sdk/search/azure-search-documents/tests/perfstress_tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/sdk/search/azure-search-documents/tests/perfstress_tests/autocomplete.py b/sdk/search/azure-search-documents/tests/perfstress_tests/autocomplete.py new file mode 100644 index 000000000000..5dbb01b8f26b --- /dev/null +++ b/sdk/search/azure-search-documents/tests/perfstress_tests/autocomplete.py @@ -0,0 +1,32 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import os +from azure_devtools.perfstress_tests import PerfStressTest + +from azure.core.credentials import AzureKeyCredential +from azure.search.documents import SearchClient as SyncAppConfigClient +from azure.search.documents.aio import SearchClient as AsyncAppConfigClient + + +class AutoCompleteTest(PerfStressTest): + def __init__(self, arguments): + super().__init__(arguments) + api_key = self.get_from_env("AZURE_SEARCH_API_KEY") + service_endpoint = os.getenv("AZURE_SEARCH_SERVICE_ENDPOINT") + index_name = os.getenv("AZURE_SEARCH_INDEX_NAME") + key = os.getenv("AZURE_SEARCH_API_KEY") + self.service_client = SyncAppConfigClient(service_endpoint, index_name, AzureKeyCredential(api_key)) + self.async_service_client = AsyncAppConfigClient(service_endpoint, index_name, AzureKeyCredential(api_key)) + + async def close(self): + await self.async_service_client.close() + await super().close() + + def run_sync(self): + self.service_client.autocomplete(search_text="mot", suggester_name="sg") + + async def run_async(self): + await self.async_service_client.autocomplete(search_text="mot", suggester_name="sg") diff --git a/sdk/search/azure-search-documents/tests/perfstress_tests/search_documents.py b/sdk/search/azure-search-documents/tests/perfstress_tests/search_documents.py new file mode 100644 index 000000000000..c260c7a3724e --- /dev/null +++ b/sdk/search/azure-search-documents/tests/perfstress_tests/search_documents.py @@ -0,0 +1,36 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import os + +from azure_devtools.perfstress_tests import PerfStressTest + +from azure.core.credentials import AzureKeyCredential +from azure.search.documents import SearchClient as SyncAppConfigClient +from azure.search.documents.aio import SearchClient as AsyncAppConfigClient + + +class SearchDocumentsTest(PerfStressTest): + def __init__(self, arguments): + super().__init__(arguments) + api_key = self.get_from_env("AZURE_SEARCH_API_KEY") + service_endpoint = os.getenv("AZURE_SEARCH_SERVICE_ENDPOINT") + index_name = os.getenv("AZURE_SEARCH_INDEX_NAME") + key = os.getenv("AZURE_SEARCH_API_KEY") + self.service_client = SyncAppConfigClient(service_endpoint, index_name, AzureKeyCredential(api_key)) + self.async_service_client = AsyncAppConfigClient(service_endpoint, index_name, AzureKeyCredential(api_key)) + + async def global_setup(self): + await super().global_setup() + + async def close(self): + await self.async_service_client.close() + await super().close() + + def run_sync(self): + self.service_client.search(search_text="luxury") + + async def run_async(self): + await self.async_service_client.search(search_text="luxury") diff --git a/sdk/search/azure-search-documents/tests/perfstress_tests/suggest.py b/sdk/search/azure-search-documents/tests/perfstress_tests/suggest.py new file mode 100644 index 000000000000..8987ecab6545 --- /dev/null +++ b/sdk/search/azure-search-documents/tests/perfstress_tests/suggest.py @@ -0,0 +1,32 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import os +from azure_devtools.perfstress_tests import PerfStressTest + +from azure.core.credentials import AzureKeyCredential +from azure.search.documents import SearchClient as SyncAppConfigClient +from azure.search.documents.aio import SearchClient as AsyncAppConfigClient + + +class SuggestTest(PerfStressTest): + def __init__(self, arguments): + super().__init__(arguments) + api_key = self.get_from_env("AZURE_SEARCH_API_KEY") + service_endpoint = os.getenv("AZURE_SEARCH_SERVICE_ENDPOINT") + index_name = os.getenv("AZURE_SEARCH_INDEX_NAME") + key = os.getenv("AZURE_SEARCH_API_KEY") + self.service_client = SyncAppConfigClient(service_endpoint, index_name, AzureKeyCredential(api_key)) + self.async_service_client = AsyncAppConfigClient(service_endpoint, index_name, AzureKeyCredential(api_key)) + + async def close(self): + await self.async_service_client.close() + await super().close() + + def run_sync(self): + self.service_client.suggest(search_text="mot", suggester_name="sg") + + async def run_async(self): + await self.async_service_client.suggest(search_text="mot", suggester_name="sg") From 1887ab5d5cf619f21608f656a63d8ef8379399aa Mon Sep 17 00:00:00 2001 From: Xiang Yan Date: Thu, 18 Mar 2021 14:38:06 -0700 Subject: [PATCH 2/6] updates --- .../tests/perfstress_tests/autocomplete.py | 8 ++++---- .../tests/perfstress_tests/search_documents.py | 8 ++++---- .../tests/perfstress_tests/suggest.py | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/sdk/search/azure-search-documents/tests/perfstress_tests/autocomplete.py b/sdk/search/azure-search-documents/tests/perfstress_tests/autocomplete.py index 5dbb01b8f26b..4826678a9ea8 100644 --- a/sdk/search/azure-search-documents/tests/perfstress_tests/autocomplete.py +++ b/sdk/search/azure-search-documents/tests/perfstress_tests/autocomplete.py @@ -7,8 +7,8 @@ from azure_devtools.perfstress_tests import PerfStressTest from azure.core.credentials import AzureKeyCredential -from azure.search.documents import SearchClient as SyncAppConfigClient -from azure.search.documents.aio import SearchClient as AsyncAppConfigClient +from azure.search.documents import SearchClient as SyncClient +from azure.search.documents.aio import SearchClient as AsyncClient class AutoCompleteTest(PerfStressTest): @@ -18,8 +18,8 @@ def __init__(self, arguments): service_endpoint = os.getenv("AZURE_SEARCH_SERVICE_ENDPOINT") index_name = os.getenv("AZURE_SEARCH_INDEX_NAME") key = os.getenv("AZURE_SEARCH_API_KEY") - self.service_client = SyncAppConfigClient(service_endpoint, index_name, AzureKeyCredential(api_key)) - self.async_service_client = AsyncAppConfigClient(service_endpoint, index_name, AzureKeyCredential(api_key)) + self.service_client = SyncClient(service_endpoint, index_name, AzureKeyCredential(api_key)) + self.async_service_client = AsyncClient(service_endpoint, index_name, AzureKeyCredential(api_key)) async def close(self): await self.async_service_client.close() diff --git a/sdk/search/azure-search-documents/tests/perfstress_tests/search_documents.py b/sdk/search/azure-search-documents/tests/perfstress_tests/search_documents.py index c260c7a3724e..b20a7404d33e 100644 --- a/sdk/search/azure-search-documents/tests/perfstress_tests/search_documents.py +++ b/sdk/search/azure-search-documents/tests/perfstress_tests/search_documents.py @@ -8,8 +8,8 @@ from azure_devtools.perfstress_tests import PerfStressTest from azure.core.credentials import AzureKeyCredential -from azure.search.documents import SearchClient as SyncAppConfigClient -from azure.search.documents.aio import SearchClient as AsyncAppConfigClient +from azure.search.documents import SearchClient as SyncClient +from azure.search.documents.aio import SearchClient as AsyncClient class SearchDocumentsTest(PerfStressTest): @@ -19,8 +19,8 @@ def __init__(self, arguments): service_endpoint = os.getenv("AZURE_SEARCH_SERVICE_ENDPOINT") index_name = os.getenv("AZURE_SEARCH_INDEX_NAME") key = os.getenv("AZURE_SEARCH_API_KEY") - self.service_client = SyncAppConfigClient(service_endpoint, index_name, AzureKeyCredential(api_key)) - self.async_service_client = AsyncAppConfigClient(service_endpoint, index_name, AzureKeyCredential(api_key)) + self.service_client = SyncClient(service_endpoint, index_name, AzureKeyCredential(api_key)) + self.async_service_client = AsyncClient(service_endpoint, index_name, AzureKeyCredential(api_key)) async def global_setup(self): await super().global_setup() diff --git a/sdk/search/azure-search-documents/tests/perfstress_tests/suggest.py b/sdk/search/azure-search-documents/tests/perfstress_tests/suggest.py index 8987ecab6545..7ab6d8a1f1ce 100644 --- a/sdk/search/azure-search-documents/tests/perfstress_tests/suggest.py +++ b/sdk/search/azure-search-documents/tests/perfstress_tests/suggest.py @@ -7,8 +7,8 @@ from azure_devtools.perfstress_tests import PerfStressTest from azure.core.credentials import AzureKeyCredential -from azure.search.documents import SearchClient as SyncAppConfigClient -from azure.search.documents.aio import SearchClient as AsyncAppConfigClient +from azure.search.documents import SearchClient as SyncClient +from azure.search.documents.aio import SearchClient as AsyncClient class SuggestTest(PerfStressTest): @@ -18,8 +18,8 @@ def __init__(self, arguments): service_endpoint = os.getenv("AZURE_SEARCH_SERVICE_ENDPOINT") index_name = os.getenv("AZURE_SEARCH_INDEX_NAME") key = os.getenv("AZURE_SEARCH_API_KEY") - self.service_client = SyncAppConfigClient(service_endpoint, index_name, AzureKeyCredential(api_key)) - self.async_service_client = AsyncAppConfigClient(service_endpoint, index_name, AzureKeyCredential(api_key)) + self.service_client = SyncClient(service_endpoint, index_name, AzureKeyCredential(api_key)) + self.async_service_client = AsyncClient(service_endpoint, index_name, AzureKeyCredential(api_key)) async def close(self): await self.async_service_client.close() From 4a77e6cdfafa3a3f30ea07ce2b74bc8d896fee7d Mon Sep 17 00:00:00 2001 From: Xiang Yan Date: Mon, 22 Mar 2021 13:43:07 -0700 Subject: [PATCH 3/6] update --- .../tests/perfstress_tests/README.md | 1 + .../perfstress_tests/search_documents.py | 21 +++++++++++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/sdk/search/azure-search-documents/tests/perfstress_tests/README.md b/sdk/search/azure-search-documents/tests/perfstress_tests/README.md index 00b98e2bc2b1..85c7491da4ea 100644 --- a/sdk/search/azure-search-documents/tests/perfstress_tests/README.md +++ b/sdk/search/azure-search-documents/tests/perfstress_tests/README.md @@ -37,6 +37,7 @@ These options are available for all perf tests: - `--warm-up=5` Number of seconds to spend warming up the connection before measuring begins. Default is 5. - `--sync` Whether to run the tests in sync or async. Default is False (async). This flag must be used for Storage legacy tests, which do not support async. - `--no-cleanup` Whether to keep newly created resources after test run. Default is False (resources will be deleted). +- `--num-documents` The number of results expect to be returned. ## Example command ```cmd diff --git a/sdk/search/azure-search-documents/tests/perfstress_tests/search_documents.py b/sdk/search/azure-search-documents/tests/perfstress_tests/search_documents.py index b20a7404d33e..c080eb899dd0 100644 --- a/sdk/search/azure-search-documents/tests/perfstress_tests/search_documents.py +++ b/sdk/search/azure-search-documents/tests/perfstress_tests/search_documents.py @@ -22,6 +22,13 @@ def __init__(self, arguments): self.service_client = SyncClient(service_endpoint, index_name, AzureKeyCredential(api_key)) self.async_service_client = AsyncClient(service_endpoint, index_name, AzureKeyCredential(api_key)) + @staticmethod + def add_arguments(parser): + super(SearchDocumentsTest, SearchDocumentsTest).add_arguments(parser) + parser.add_argument('--num-documents', nargs='?', type=int, + help='The number of results expect to be returned.', + default=-1) + async def global_setup(self): await super().global_setup() @@ -30,7 +37,17 @@ async def close(self): await super().close() def run_sync(self): - self.service_client.search(search_text="luxury") + if self.args.num_documents == -1: + results = len(self.service_client.search(search_text="luxury")) + else: + results = len(self.service_client.search(search_text="luxury", top=self.args.num_documents)) + async def run_async(self): - await self.async_service_client.search(search_text="luxury") + if self.args.num_documents == -1: + results = await self.async_service_client.search(search_text="luxury") + else: + results = await self.async_service_client.search(search_text="luxury", top=self.args.num_documents) + count = 0 + async for result in results: + count += count From aee7454d717d7d31774b0f9f8c00a862769c9e0e Mon Sep 17 00:00:00 2001 From: Xiang Yan Date: Wed, 24 Mar 2021 13:44:06 -0700 Subject: [PATCH 4/6] update --- .../tests/perfstress_tests/README.md | 4 +++- .../tests/perfstress_tests/autocomplete.py | 18 ++++++++++++++++-- .../tests/perfstress_tests/suggest.py | 18 ++++++++++++++++-- 3 files changed, 35 insertions(+), 5 deletions(-) diff --git a/sdk/search/azure-search-documents/tests/perfstress_tests/README.md b/sdk/search/azure-search-documents/tests/perfstress_tests/README.md index 85c7491da4ea..d51935583c95 100644 --- a/sdk/search/azure-search-documents/tests/perfstress_tests/README.md +++ b/sdk/search/azure-search-documents/tests/perfstress_tests/README.md @@ -5,7 +5,7 @@ Start be creating a new virtual environment for your perf tests. This will need ### Setup for test resources -These tests will run against a pre-configured search service. The following environment variable will need to be set for the tests to access the live resources: +These tests will run against a pre-configured search service. See [here](https://docs.microsoft.com/en-us/azure/search/search-indexer-tutorial) to figure out how to configure the service. The following environment variable will need to be set for the tests to access the live resources: ``` AZURE_SEARCH_API_KEY= AZURE_SEARCH_SERVICE_ENDPOINT= @@ -42,4 +42,6 @@ These options are available for all perf tests: ## Example command ```cmd (env) ~/search/azure-search-documents/tests> perfstress SearchDocumentsTest +(env) ~/search/azure-search-documents/tests> perfstress AutoCompleteTest +(env) ~/search/azure-search-documents/tests> perfstress SuggestTest ``` diff --git a/sdk/search/azure-search-documents/tests/perfstress_tests/autocomplete.py b/sdk/search/azure-search-documents/tests/perfstress_tests/autocomplete.py index 4826678a9ea8..50cf6bb81c94 100644 --- a/sdk/search/azure-search-documents/tests/perfstress_tests/autocomplete.py +++ b/sdk/search/azure-search-documents/tests/perfstress_tests/autocomplete.py @@ -25,8 +25,22 @@ async def close(self): await self.async_service_client.close() await super().close() + @staticmethod + def add_arguments(parser): + super(AutoCompleteTest, AutoCompleteTest).add_arguments(parser) + parser.add_argument('--num-documents', nargs='?', type=int, + help='The number of results expect to be returned.', + default=-1) + def run_sync(self): - self.service_client.autocomplete(search_text="mot", suggester_name="sg") + if self.args.num_documents == -1: + results = len(self.service_client.autocomplete(search_text="mot", suggester_name="sg")) + else: + results = len(self.service_client.autocomplete(search_text="mot", suggester_name="sg", top=self.args.num_documents)) async def run_async(self): - await self.async_service_client.autocomplete(search_text="mot", suggester_name="sg") + if self.args.num_documents == -1: + results = len(await self.async_service_client.autocomplete(search_text="mot", suggester_name="sg")) + else: + results = len(await self.async_service_client.autocomplete(search_text="mot", suggester_name="sg", top=self.args.num_documents)) + diff --git a/sdk/search/azure-search-documents/tests/perfstress_tests/suggest.py b/sdk/search/azure-search-documents/tests/perfstress_tests/suggest.py index 7ab6d8a1f1ce..44353682418f 100644 --- a/sdk/search/azure-search-documents/tests/perfstress_tests/suggest.py +++ b/sdk/search/azure-search-documents/tests/perfstress_tests/suggest.py @@ -25,8 +25,22 @@ async def close(self): await self.async_service_client.close() await super().close() + @staticmethod + def add_arguments(parser): + super(SuggestTest, SuggestTest).add_arguments(parser) + parser.add_argument('--num-documents', nargs='?', type=int, + help='The number of results expect to be returned.', + default=-1) + def run_sync(self): - self.service_client.suggest(search_text="mot", suggester_name="sg") + if self.args.num_documents == -1: + results = len(self.service_client.suggest(search_text="mot", suggester_name="sg")) + else: + results = len(self.service_client.suggest(search_text="mot", suggester_name="sg", top=self.args.num_documents)) async def run_async(self): - await self.async_service_client.suggest(search_text="mot", suggester_name="sg") + if self.args.num_documents == -1: + results = len(await self.async_service_client.suggest(search_text="mot", suggester_name="sg")) + else: + results = len(await self.async_service_client.suggest(search_text="mot", suggester_name="sg", top=self.args.num_documents)) + From 0e9279e6d0f1573ebee6629aa002496a80d5bf1e Mon Sep 17 00:00:00 2001 From: Xiang Yan Date: Wed, 24 Mar 2021 14:28:55 -0700 Subject: [PATCH 5/6] update --- .../azure-search-documents/tests/perfstress_tests/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/search/azure-search-documents/tests/perfstress_tests/README.md b/sdk/search/azure-search-documents/tests/perfstress_tests/README.md index d51935583c95..8ec34af73ec1 100644 --- a/sdk/search/azure-search-documents/tests/perfstress_tests/README.md +++ b/sdk/search/azure-search-documents/tests/perfstress_tests/README.md @@ -5,7 +5,7 @@ Start be creating a new virtual environment for your perf tests. This will need ### Setup for test resources -These tests will run against a pre-configured search service. See [here](https://docs.microsoft.com/en-us/azure/search/search-indexer-tutorial) to figure out how to configure the service. The following environment variable will need to be set for the tests to access the live resources: +These tests will run against a pre-configured search service. See [here](https://docs.microsoft.com/azure/search/search-indexer-tutorial) to figure out how to configure the service. The following environment variable will need to be set for the tests to access the live resources: ``` AZURE_SEARCH_API_KEY= AZURE_SEARCH_SERVICE_ENDPOINT= From f28ee5914f4aa349e2ea6c53ac9264602e930907 Mon Sep 17 00:00:00 2001 From: Xiang Yan Date: Mon, 29 Mar 2021 10:08:17 -0700 Subject: [PATCH 6/6] update --- .../azure-search-documents/tests/perfstress_tests/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/search/azure-search-documents/tests/perfstress_tests/README.md b/sdk/search/azure-search-documents/tests/perfstress_tests/README.md index 8ec34af73ec1..d6b4b7f5590b 100644 --- a/sdk/search/azure-search-documents/tests/perfstress_tests/README.md +++ b/sdk/search/azure-search-documents/tests/perfstress_tests/README.md @@ -5,7 +5,7 @@ Start be creating a new virtual environment for your perf tests. This will need ### Setup for test resources -These tests will run against a pre-configured search service. See [here](https://docs.microsoft.com/azure/search/search-indexer-tutorial) to figure out how to configure the service. The following environment variable will need to be set for the tests to access the live resources: +These tests will run against a pre-configured search service. See [here](https://docs.microsoft.com/azure/search/search-indexer-tutorial) about how to configure the service and import data. The following environment variable will need to be set for the tests to access the live resources: ``` AZURE_SEARCH_API_KEY= AZURE_SEARCH_SERVICE_ENDPOINT=