Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Introduce test-specific environment variables #1014

Merged
merged 1 commit into from
Dec 18, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions tests/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,20 @@ TIMEOUT_COLLECT_TEST_RESULTS=20

These can be set either in the `environment` section of the tmt plan or using the `-e` option when running tmt, e.g. `-eTIMEOUT_TEST_SETUP=40`.

### Test-specific timeouts

In addition to the mentioned above timeouts, there's a mechanism allowing to set values in specific tests via environment variables, which can be used to adjust test-specific timeouts via the environment. The expected environment variable name would be assembled from the prefix `TEST_`, test name and a provided suffix. These test-specific timeouts can be set in the tmt plan as described [here](#changing-timeouts-for-integration-tests).

mkemel marked this conversation as resolved.
Show resolved Hide resolved
#### Example

Given a test named `bluechi-generic-test` (i.e. the test script located in the directory `bluechi-generic-test`), which uses a timeout `WAIT_TIMEOUT`, defining it as follows:

```python
WAIT_TIMEOUT = get_test_env_value_int("WAIT_TIMEOUT", 1000)
```

The variable would be assigned `WAIT_TIMEOUT = 1000`, unless environment variable `TEST_BLUECHI_GENERIC_TEST_WAIT_TIMEOUT` is set with an integer value, in which case this value would be passed to `WAIT_TIMEOUT`.

## Developing integration tests

### Code Style
Expand Down
41 changes: 14 additions & 27 deletions tests/bluechi_test/fixtures.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,97 +11,84 @@
import yaml
from bluechi_test.config import BluechiAgentConfig, BluechiControllerConfig
from bluechi_test.test import BluechiContainerTest, BluechiSSHTest, BluechiTest
from bluechi_test.util import get_primary_ip
from bluechi_test.util import get_env_value, get_primary_ip, safely_parse_int
from podman import PodmanClient


def _get_env_value(env_var: str, default_value: str) -> str:
value = os.getenv(env_var)
if value is None:
return default_value
return value


@pytest.fixture(scope="session")
def tmt_test_data_dir() -> str:
"""Return directory, where tmt saves data of the relevant test. If the TMT_TEST_DATA env variable is not set, then
use current directory"""
return _get_env_value("TMT_TEST_DATA", os.getcwd())
return get_env_value("TMT_TEST_DATA", os.getcwd())


@pytest.fixture(scope="function")
def tmt_test_serial_number() -> str:
"""Return serial number of current test"""
return _get_env_value("TMT_TEST_SERIAL_NUMBER", "NA")
return get_env_value("TMT_TEST_SERIAL_NUMBER", "NA")


@pytest.fixture(scope="session")
def bluechi_image_name() -> str:
"""Returns the name of bluechi testing container images"""
return _get_env_value("BLUECHI_IMAGE_NAME", "bluechi-image")
return get_env_value("BLUECHI_IMAGE_NAME", "bluechi-image")


@pytest.fixture(scope="session")
def bluechi_ctrl_host_port() -> str:
"""Returns the port, which bluechi controller service is mapped to on a host"""

return _get_env_value("BLUECHI_CTRL_HOST_PORT", "8420")
return get_env_value("BLUECHI_CTRL_HOST_PORT", "8420")


@pytest.fixture(scope="session")
def bluechi_ctrl_svc_port() -> str:
"""Returns the port, which bluechi controller service is using inside a container"""

return _get_env_value("BLUECHI_CTRL_SVC_PORT", "8420")
return get_env_value("BLUECHI_CTRL_SVC_PORT", "8420")


@pytest.fixture(scope="session")
def machines_ssh_user() -> str:
"""Returns the user for connecting to the available hosts via SSH"""

return _get_env_value("SSH_USER", "root")
return get_env_value("SSH_USER", "root")


@pytest.fixture(scope="session")
def machines_ssh_password() -> str:
"""Returns the password for connecting to the available hosts via SSH"""

return _get_env_value("SSH_PASSWORD", "")


def _safely_parse_int(input: str, default: int) -> int:
if input.isdigit():
return int(input)
return default
return get_env_value("SSH_PASSWORD", "")


@pytest.fixture(scope="session")
def timeout_test_setup() -> int:
"""Returns the timeout for setting up the test setup"""

return _safely_parse_int(_get_env_value("TIMEOUT_TEST_SETUP", ""), 20)
return safely_parse_int(get_env_value("TIMEOUT_TEST_SETUP", ""), 20)


@pytest.fixture(scope="session")
def timeout_test_run() -> int:
"""Returns the timeout for executing the actual test"""

return _safely_parse_int(_get_env_value("TIMEOUT_TEST_RUN", ""), 45)
return safely_parse_int(get_env_value("TIMEOUT_TEST_RUN", ""), 45)


@pytest.fixture(scope="session")
def timeout_collecting_test_results() -> int:
"""Returns the timeout for collecting all test results"""

return _safely_parse_int(_get_env_value("TIMEOUT_COLLECT_TEST_RESULTS", ""), 20)
return safely_parse_int(get_env_value("TIMEOUT_COLLECT_TEST_RESULTS", ""), 20)


def _read_topology() -> Dict[str, Any]:
"""
Returns the parsed YAML for the tmt guest topology:
https://tmt.readthedocs.io/en/stable/spec/plans.html#guest-topology-format
"""
tmt_yaml_file = _get_env_value("TMT_TOPOLOGY_YAML", "")
tmt_yaml_file = get_env_value("TMT_TOPOLOGY_YAML", "")
if tmt_yaml_file is None or tmt_yaml_file == "":
return get_primary_ip()

Expand Down Expand Up @@ -170,14 +157,14 @@ def is_multihost_run(available_hosts: Dict[str, List[Tuple[str, str]]]) -> bool:
def run_with_valgrind() -> bool:
"""Returns 1 if bluechi should be run with valgrind for memory management testing"""

return _get_env_value("WITH_VALGRIND", 0) == "1"
return get_env_value("WITH_VALGRIND", 0) == "1"


@pytest.fixture(scope="session")
def run_with_coverage() -> bool:
"""Returns 1 if code coverage should be collected"""

return _get_env_value("WITH_COVERAGE", 0) == "1"
return get_env_value("WITH_COVERAGE", 0) == "1"


@pytest.fixture(scope="session")
Expand Down
32 changes: 32 additions & 0 deletions tests/bluechi_test/util.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,9 @@
#
# SPDX-License-Identifier: LGPL-2.1-or-later

import inspect
import logging
import os
import random
import re
import signal
Expand Down Expand Up @@ -84,3 +86,33 @@ def __enter__(self):

def __exit__(self, type, value, traceback):
signal.alarm(0)


def get_env_value(env_var: str, default_value: str) -> str:
value = os.getenv(env_var)
if value is None:
return default_value
return value


def safely_parse_int(input: str, default: int) -> int:
if input.isdigit():
return int(input)
return default


def _get_test_env_value(varname: str, test_file: str, default_value: str) -> str:
test_name = os.path.basename(os.path.dirname(test_file))
envvar = f"TEST_{test_name.upper().replace('-', '_')}_{varname.upper()}"
return get_env_value(envvar, default_value)


def get_test_env_value(varname: str, default_value: str) -> str:
test_file = inspect.stack()[1].filename
return _get_test_env_value(varname, test_file, default_value)


def get_test_env_value_int(varname: str, default_value: int) -> int:
test_file = inspect.stack()[1].filename
value = _get_test_env_value(varname, test_file, "")
return safely_parse_int(value, default_value)
Loading