Skip to content

Commit

Permalink
Add tests for redis cluster
Browse files Browse the repository at this point in the history
  • Loading branch information
hmstepanek committed May 2, 2023
1 parent 5370eae commit 5b2c662
Show file tree
Hide file tree
Showing 6 changed files with 310 additions and 7 deletions.
70 changes: 70 additions & 0 deletions .github/workflows/tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ jobs:
- postgres
- rabbitmq
- redis
- rediscluster
- solr

steps:
Expand Down Expand Up @@ -339,6 +340,75 @@ jobs:
path: ./**/.coverage.*
retention-days: 1

rediscluster:
env:
TOTAL_GROUPS: 1

strategy:
fail-fast: false
matrix:
group-number: [1]

runs-on: ubuntu-20.04
timeout-minutes: 30

services:
redis:
image: redis
ports:
- 8090:7000
# Set health checks to wait until redis has started
options: >-
--health-cmd "redis-cli ping"
--health-interval 10s
--health-timeout 5s
--health-retries 5
steps:
- uses: actions/checkout@v3
- uses: ./.github/actions/setup-python-matrix

# Start the redis cluser (See https://redis.io/docs/management/scaling/#create-a-redis-cluster)
- name: Start the redis cluster
run: |
mkdir cluster-test
mkdir cluster-test/7000 cluster-test/7001 cluster-test/7002 cluster-test/7003 cluster-test/7004 cluster-test/7005
echo $'port 7000\ncluster-enabled yes\ncluster-config-file nodes.conf\ncluster-node-timeout 5000\nappendonly yes' > cluster-test/7000/redis.conf
echo $'port 7000\ncluster-enabled yes\ncluster-config-file nodes.conf\ncluster-node-timeout 5000\nappendonly yes' > cluster-test/7001/redis.conf
echo $'port 7000\ncluster-enabled yes\ncluster-config-file nodes.conf\ncluster-node-timeout 5000\nappendonly yes' > cluster-test/7002/redis.conf
echo $'port 7000\ncluster-enabled yes\ncluster-config-file nodes.conf\ncluster-node-timeout 5000\nappendonly yes' > cluster-test/7003/redis.conf
echo $'port 7000\ncluster-enabled yes\ncluster-config-file nodes.conf\ncluster-node-timeout 5000\nappendonly yes' > cluster-test/7004/redis.conf
echo $'port 7000\ncluster-enabled yes\ncluster-config-file nodes.conf\ncluster-node-timeout 5000\nappendonly yes' > cluster-test/7005/redis.conf
redis-server ./cluster-test/7000/redis.conf --daemonize yes
redis-server ./cluster-test/7001/redis.conf --daemonize yes
redis-server ./cluster-test/7002/redis.conf --daemonize yes
redis-server ./cluster-test/7003/redis.conf --daemonize yes
redis-server ./cluster-test/7004/redis.conf --daemonize yes
redis-server ./cluster-test/7005/redis.conf --daemonize yes
redis-cli --cluster create 127.0.0.1:7000 127.0.0.1:7001 127.0.0.1:7002 127.0.0.1:7003 127.0.0.1:7004 127.0.0.1:7005 --cluster-replicas 1
sudo apt-get install libcurl4-openssl-dev
- name: Get Environments
id: get-envs
run: |
echo "::set-output name=envs::$(tox -l | grep "^${{ github.job }}\-" | ./.github/workflows/get-envs.py)"
env:
GROUP_NUMBER: ${{ matrix.group-number }}

- name: Test
run: |
tox -vv -e ${{ steps.get-envs.outputs.envs }} -p auto
env:
TOX_PARALLEL_NO_SPINNER: 1
PY_COLORS: 0

- name: Upload Coverage Artifacts
uses: actions/upload-artifact@v3
with:
name: coverage-${{ github.job }}-${{ strategy.job-index }}
path: ./**/.coverage.*
retention-days: 1

redis:
env:
TOTAL_GROUPS: 2
Expand Down
11 changes: 5 additions & 6 deletions tests/datastore_redis/test_uninstrumented_methods.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,19 +14,18 @@

import pytest
import redis

from testing_support.db_settings import redis_settings

DB_SETTINGS = redis_settings()[0]

redis_client = redis.Redis(host=DB_SETTINGS['host'], port=DB_SETTINGS['port'], db=0)
strict_redis_client = redis.StrictRedis(host=DB_SETTINGS['host'], port=DB_SETTINGS['port'], db=0)
redis_client = redis.Redis(host=DB_SETTINGS["host"], port=DB_SETTINGS["port"], db=0)
strict_redis_client = redis.StrictRedis(host=DB_SETTINGS["host"], port=DB_SETTINGS["port"], db=0)


IGNORED_METHODS = {
'MODULE_CALLBACKS',
'MODULE_VERSION',
'NAME',
"MODULE_CALLBACKS",
"MODULE_VERSION",
"NAME",
"add_edge",
"add_node",
"append_bucket_size",
Expand Down
32 changes: 32 additions & 0 deletions tests/datastore_rediscluster/conftest.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
# Copyright 2010 New Relic, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611
collector_agent_registration_fixture,
collector_available_fixture,
)

_default_settings = {
"transaction_tracer.explain_threshold": 0.0,
"transaction_tracer.transaction_threshold": 0.0,
"transaction_tracer.stack_trace_threshold": 0.0,
"debug.log_data_collector_payloads": True,
"debug.record_transaction_failure": True,
}

collector_agent_registration = collector_agent_registration_fixture(
app_name="Python Agent Test (datastore_redis)",
default_settings=_default_settings,
linked_applications=["Python Agent Test (datastore)"],
)
Original file line number Diff line number Diff line change
@@ -0,0 +1,171 @@
# Copyright 2010 New Relic, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import pytest
import redis
from testing_support.db_settings import redis_cluster_settings

DB_CLUSTER_SETTINGS = redis_cluster_settings()[0]

cluster_client = redis.RedisCluster(host=DB_CLUSTER_SETTINGS["host"], port=DB_CLUSTER_SETTINGS["port"])


IGNORED_METHODS = {
"MODULE_CALLBACKS",
"MODULE_VERSION",
"NAME",
"add_edge",
"add_node",
"append_bucket_size",
"append_capacity",
"append_error",
"append_expansion",
"append_items_and_increments",
"append_items",
"append_max_iterations",
"append_no_create",
"append_no_scale",
"append_values_and_weights",
"append_weights",
"batch_indexer",
"BatchIndexer",
"bulk",
"call_procedure",
"client_tracking_off",
"client_tracking_on",
"client",
"close",
"commandmixin",
"connection_pool",
"connection",
"debug_segfault",
"edges",
"execute_command",
"flush",
"from_url",
"get_connection_kwargs",
"get_encoder",
"get_label",
"get_params_args",
"get_property",
"get_relation",
"get_retry",
"hscan_iter",
"index_name",
"labels",
"list_keys",
"load_document",
"load_external_module",
"lock",
"name",
"nodes",
"parse_response",
"pipeline",
"property_keys",
"register_script",
"relationship_types",
"response_callbacks",
"RESPONSE_CALLBACKS",
"sentinel",
"set_file",
"set_path",
"set_response_callback",
"set_retry",
"transaction",
"version",
"ALL_NODES",
"CLUSTER_COMMANDS_RESPONSE_CALLBACKS",
"COMMAND_FLAGS",
"DEFAULT_NODE",
"ERRORS_ALLOW_RETRY",
"NODE_FLAGS",
"PRIMARIES",
"RANDOM",
"REPLICAS",
"RESULT_CALLBACKS",
"RedisClusterRequestTTL",
"SEARCH_COMMANDS",
"cluster_addslotsrange",
"cluster_bumpepoch",
"cluster_delslotsrange",
"cluster_error_retry_attempts",
"cluster_flushslots",
"cluster_links",
"cluster_myid",
"cluster_replicas",
"cluster_response_callbacks",
"cluster_setslot_stable",
"cluster_shards",
"command_flags",
"commands_parser",
"determine_slot",
"disconnect_connection_pools",
"encoder",
"get_default_node",
"get_node",
"get_node_from_key",
"get_nodes",
"get_primaries",
"get_random_node",
"get_redis_connection",
"get_replicas",
"keyslot",
"mget_nonatomic",
"monitor",
"mset_nonatomic",
"node_flags",
"nodes_manager",
"on_connect",
"pubsub",
"read_from_replicas",
"reinitialize_counter",
"reinitialize_steps",
"replace_default_node",
"result_callbacks",
"set_default_node",
"user_on_connect_func",
}

REDIS_MODULES = {
"bf",
"cf",
"cms",
"ft",
"graph",
"json",
"tdigest",
"topk",
"ts",
}

IGNORED_METHODS |= REDIS_MODULES


breakpoint()


@pytest.mark.parametrize("client", (cluster_client))
def test_uninstrumented_methods(client):
methods = {m for m in dir(client) if not m[0] == "_"}
is_wrapped = lambda m: hasattr(getattr(client, m), "__wrapped__")
uninstrumented = {m for m in methods - IGNORED_METHODS if not is_wrapped(m)}

for module in REDIS_MODULES:
if hasattr(client, module):
module_client = getattr(client, module)()
module_methods = {m for m in dir(module_client) if not m[0] == "_"}
is_wrapped = lambda m: hasattr(getattr(module_client, m), "__wrapped__")
uninstrumented |= {m for m in module_methods - IGNORED_METHODS if not is_wrapped(m)}

assert not uninstrumented, "Uninstrumented methods: %s" % sorted(uninstrumented)
28 changes: 28 additions & 0 deletions tests/testing_support/db_settings.py
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,34 @@ def redis_settings():
return settings


def redis_cluster_settings():
"""Return a list of dict of settings for connecting to redis cluster.
Will return the correct settings, depending on which of the environments it
is running in. It attempts to set variables in the following order, where
later environments override earlier ones.
1. Local
2. Github Actions
"""

if "GITHUB_ACTIONS" in os.environ:
instances = 1
base_port = 8090
else:
instances = 1
base_port = 7000

settings = [
{
"host": "localhost",
"port": base_port + instance_num,
}
for instance_num in range(instances)
]
return settings


def memcached_settings():
"""Return a list of dict of settings for connecting to memcached.
Expand Down
5 changes: 4 additions & 1 deletion tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@ envlist =
solr-datastore_pysolr-{py27,py37,py38,py39,py310,py311,pypy,pypy37},
redis-datastore_redis-{py27,py37,py38,pypy,pypy37}-redis03,
redis-datastore_redis-{py37,py38,py39,py310,py311,pypy37}-redis{0400,latest},
redis-datastore_rediscluster-{py37,py311,pypy37}-redis{latest},
redis-datastore_aioredis-{py37,py38,py39,py310,pypy37}-aioredislatest,
redis-datastore_aioredis-{py37,py310}-aioredis01,
redis-datastore_aioredis-{py37,py38,py39,py310,py311,pypy37}-redislatest,
Expand Down Expand Up @@ -248,6 +249,7 @@ deps =
datastore_pymysql: PyMySQL<0.11
datastore_pysolr: pysolr<4.0
datastore_redis-redislatest: redis
datastore_rediscluster-redislatest: redis
datastore_redis-redis0400: redis<4.1
datastore_redis-redis03: redis<4.0
datastore_redis-{py27,pypy}: rb
Expand Down Expand Up @@ -453,7 +455,8 @@ changedir =
datastore_pymongo: tests/datastore_pymongo
datastore_pymysql: tests/datastore_pymysql
datastore_pysolr: tests/datastore_pysolr
datastore_redis: tests/datastore_redis
datastore_redis: tests/datastore_redis/
datastore_rediscluster: tests/datastore_rediscluster/
datastore_aioredis: tests/datastore_aioredis
datastore_aredis: tests/datastore_aredis
datastore_solrpy: tests/datastore_solrpy
Expand Down

0 comments on commit 5b2c662

Please sign in to comment.