From 709a9d0aef374a82db98211a5bdbac57ea5d708b Mon Sep 17 00:00:00 2001 From: Julia <611228+hithwen@users.noreply.github.com> Date: Thu, 23 Dec 2021 12:25:12 +0100 Subject: [PATCH] Add windows support (#10737) * Add support for windows Co-authored-by: Ofek Lev --- .azure-pipelines/changes.yml | 2 +- .../windows/55_install_ibm_mq_client.py | 37 +++++++++++++++++++ .../templates/test-all-checks.yml | 5 ++- .../base/data/agent_requirements.in | 2 +- ibm_mq/README.md | 15 ++++++-- ibm_mq/manifest.json | 3 +- ibm_mq/requirements.in | 2 +- ibm_mq/tests/common.py | 8 ++++ ibm_mq/tests/test_ibm_mq_e2e.py | 6 +-- ibm_mq/tests/test_ibm_mq_int.py | 4 +- ibm_mq/tests/test_ibm_mq_unit.py | 3 ++ ibm_mq/tests/test_metadata.py | 3 +- ibm_mq/tox.ini | 7 +++- 13 files changed, 80 insertions(+), 17 deletions(-) create mode 100644 .azure-pipelines/scripts/ibm_mq/windows/55_install_ibm_mq_client.py diff --git a/.azure-pipelines/changes.yml b/.azure-pipelines/changes.yml index cf1c7119e8580..87b3b7892129c 100644 --- a/.azure-pipelines/changes.yml +++ b/.azure-pipelines/changes.yml @@ -46,7 +46,7 @@ jobs: - template: './templates/test-single-windows.yml' parameters: job_name: Changed - check: '--changed datadog_checks_base datadog_checks_dev active_directory aspdotnet disk dns_check dotnetclr exchange_server iis network pdh_check sqlserver tcp_check win32_event_log windows_performance_counters windows_service wmi_check' + check: '--changed datadog_checks_base datadog_checks_dev active_directory aspdotnet disk dns_check dotnetclr exchange_server ibm_mq iis network pdh_check sqlserver tcp_check win32_event_log windows_performance_counters windows_service wmi_check' ${{ if eq(variables['System.PullRequest.IsFork'], 'False') }}: ddtrace_flag: '--ddtrace' validate_changed: changed diff --git a/.azure-pipelines/scripts/ibm_mq/windows/55_install_ibm_mq_client.py b/.azure-pipelines/scripts/ibm_mq/windows/55_install_ibm_mq_client.py new file mode 100644 index 0000000000000..9b0375b595c1c --- /dev/null +++ b/.azure-pipelines/scripts/ibm_mq/windows/55_install_ibm_mq_client.py @@ -0,0 +1,37 @@ +import os +from tempfile import TemporaryDirectory +from zipfile import ZipFile + +import requests + + +CLIENT_VERSION = '9.2.2.0' +CLIENT_ARCHIVE_NAME = f'{CLIENT_VERSION}-IBM-MQC-Redist-Win64.zip' +CLIENT_URL = f'https://public.dhe.ibm.com/ibmdl/export/pub/software/websphere/messaging/mqdev/redist/{CLIENT_ARCHIVE_NAME}' +CLIENT_TARGET_DIR = 'C:\\ibm_mq' + + +def download_file(url, file_name): + response = requests.get(url, stream=True) + response.raise_for_status() + + with open(file_name, 'wb') as f: + for chunk in response.iter_content(16384): + f.write(chunk) + + +def main(): + with TemporaryDirectory() as d: + temp_dir = os.path.realpath(d) + + print('Downloading client') + client_archive_path = os.path.join(temp_dir, CLIENT_ARCHIVE_NAME) + download_file(CLIENT_URL, client_archive_path) + + print('Extracting client') + with ZipFile(client_archive_path) as zip_file: + zip_file.extractall(CLIENT_TARGET_DIR) + + +if __name__ == '__main__': + main() diff --git a/.azure-pipelines/templates/test-all-checks.yml b/.azure-pipelines/templates/test-all-checks.yml index 8abf178911f78..c58c4d8af626c 100644 --- a/.azure-pipelines/templates/test-all-checks.yml +++ b/.azure-pipelines/templates/test-all-checks.yml @@ -226,8 +226,11 @@ jobs: displayName: IBM i os: linux - checkName: ibm_mq - displayName: IBM MQ + displayName: IBM MQ (Linux) os: linux + - checkName: ibm_mq + displayName: IBM MQ (Windows) + os: windows - checkName: ibm_was displayName: IBM WAS os: linux diff --git a/datadog_checks_base/datadog_checks/base/data/agent_requirements.in b/datadog_checks_base/datadog_checks/base/data/agent_requirements.in index a9b11b25b1d0a..53c5b243ef4d5 100644 --- a/datadog_checks_base/datadog_checks/base/data/agent_requirements.in +++ b/datadog_checks_base/datadog_checks/base/data/agent_requirements.in @@ -59,7 +59,7 @@ pyhdb==0.3.4 pyjwt==1.7.1; python_version < "3.0" pyjwt==2.0.1; python_version > "3.0" pymongo==3.11.4 -pymqi==1.12.0; sys_platform != "win32" +pymqi==1.12.0 pymysql==0.9.3 pyodbc==4.0.26 pyro4==4.73; sys_platform == "win32" diff --git a/ibm_mq/README.md b/ibm_mq/README.md index fa68bc9cce24a..ab8ef0ceae410 100644 --- a/ibm_mq/README.md +++ b/ibm_mq/README.md @@ -10,10 +10,12 @@ This check monitors [IBM MQ][1] versions 8 to 9.0. The IBM MQ check is included in the [Datadog Agent][2] package. -To use the IBM MQ check, you need to: +To use the IBM MQ check, you need to make sure the [IBM MQ Client][3] 9.1+ is installed (unless a compatible version of IBM MQ server is installed on the Agent host). + +#### On Linux -1. Make sure the [IBM MQ Client][3] 9.1+ is installed (unless a compatible version of IBM MQ server is installed on the Agent host). -2. Update your `LD_LIBRARY_PATH` and `C_INCLUDE_PATH` to include the location of the libraries. (Create these two environment variables if they don’t exist yet.) For example: +Update your `LD_LIBRARY_PATH` and `C_INCLUDE_PATH` to include the location of the libraries. (Create these two environment variables if they don’t exist yet.) +For example, if you installed it in `/opt`: ```text export LD_LIBRARY_PATH=/opt/mqm/lib64:/opt/mqm/lib:$LD_LIBRARY_PATH @@ -129,7 +131,12 @@ Update the bindings: sudo ldconfig ``` -#### Permissions and authentication +#### On Windows + +There is a file called `mqclient.ini` in the IBM MQ data directory. It is normally `C:\ProgramData\IBM\MQ`. +Configure the environment variable `MQ_FILE_PATH`, to point at the data directory. + +### Permissions and authentication There are many ways to set up permissions in IBM MQ. Depending on how your setup works, create a `datadog` user within MQ with read only permissions. diff --git a/ibm_mq/manifest.json b/ibm_mq/manifest.json index bfbd40657c6cd..a27ea77379600 100644 --- a/ibm_mq/manifest.json +++ b/ibm_mq/manifest.json @@ -11,7 +11,8 @@ "support": "core", "supported_os": [ "linux", - "mac_os" + "mac_os", + "windows" ], "public_title": "Datadog-IBM MQ Integration", "categories": [ diff --git a/ibm_mq/requirements.in b/ibm_mq/requirements.in index 710b2cfbcc99d..cfb23ec3ee5ad 100644 --- a/ibm_mq/requirements.in +++ b/ibm_mq/requirements.in @@ -1 +1 @@ -pymqi==1.12.0; sys_platform != 'win32' +pymqi==1.12.0 diff --git a/ibm_mq/tests/common.py b/ibm_mq/tests/common.py index c26d244c318fb..3c1e0052a993f 100644 --- a/ibm_mq/tests/common.py +++ b/ibm_mq/tests/common.py @@ -4,7 +4,11 @@ import os +import pytest + from datadog_checks.dev import get_docker_hostname +from datadog_checks.dev.ci import running_on_ci +from datadog_checks.dev.utils import ON_WINDOWS # Ignore missing library to not require it for e2e try: @@ -12,6 +16,10 @@ except ImportError: COUNT = GAUGE = '' +RUNNING_ON_WINDOWS_CI = ON_WINDOWS and running_on_ci() +skip_windows_ci = pytest.mark.skipif(RUNNING_ON_WINDOWS_CI, reason='MQ server cannot be setup on Windows VMs in CI') + + HERE = os.path.dirname(os.path.abspath(__file__)) COMPOSE_DIR = os.path.join(HERE, 'compose') diff --git a/ibm_mq/tests/test_ibm_mq_e2e.py b/ibm_mq/tests/test_ibm_mq_e2e.py index e20dc2c4e4ee7..6c58aedfc9b4a 100644 --- a/ibm_mq/tests/test_ibm_mq_e2e.py +++ b/ibm_mq/tests/test_ibm_mq_e2e.py @@ -6,10 +6,11 @@ from datadog_checks.dev.utils import get_metadata_metrics -from .common import MQ_VERSION, assert_all_metrics +from .common import MQ_VERSION, assert_all_metrics, skip_windows_ci + +pytestmark = [skip_windows_ci, pytest.mark.e2e] -@pytest.mark.e2e def test_e2e_check_all(dd_agent_check, instance_collect_all): aggregator = dd_agent_check(instance_collect_all, rate=True) @@ -17,7 +18,6 @@ def test_e2e_check_all(dd_agent_check, instance_collect_all): aggregator.assert_metrics_using_metadata(get_metadata_metrics()) -@pytest.mark.e2e @pytest.mark.skipif( MQ_VERSION < 9, reason='Only test for for version >=9, for v8 use a custom image with custom setup.' ) diff --git a/ibm_mq/tests/test_ibm_mq_int.py b/ibm_mq/tests/test_ibm_mq_int.py index f433708483063..f5f8896ac19f5 100644 --- a/ibm_mq/tests/test_ibm_mq_int.py +++ b/ibm_mq/tests/test_ibm_mq_int.py @@ -14,9 +14,9 @@ from datadog_checks.dev.utils import get_metadata_metrics from . import common -from .common import QUEUE_METRICS, assert_all_metrics +from .common import QUEUE_METRICS, assert_all_metrics, skip_windows_ci -pytestmark = [pytest.mark.usefixtures("dd_environment"), pytest.mark.integration] +pytestmark = [skip_windows_ci, pytest.mark.usefixtures("dd_environment"), pytest.mark.integration] def test_no_msg_errors_are_caught(get_check, instance, caplog): diff --git a/ibm_mq/tests/test_ibm_mq_unit.py b/ibm_mq/tests/test_ibm_mq_unit.py index 63f543eeaa374..8830700d1868d 100644 --- a/ibm_mq/tests/test_ibm_mq_unit.py +++ b/ibm_mq/tests/test_ibm_mq_unit.py @@ -7,6 +7,8 @@ from datadog_checks.base import AgentCheck, ConfigurationError +from .common import skip_windows_ci + pytestmark = pytest.mark.unit @@ -182,6 +184,7 @@ def test_channel_queue_config_error(instance_config): assert 'channel, queue_manager are required configurations' in str(excinfo.value) +@skip_windows_ci def test_ssl_connection_creation(get_check, instance): """ Test that we are not getting unicode/bytes type error. diff --git a/ibm_mq/tests/test_metadata.py b/ibm_mq/tests/test_metadata.py index 15c0b63dbc442..f45adb7458944 100644 --- a/ibm_mq/tests/test_metadata.py +++ b/ibm_mq/tests/test_metadata.py @@ -3,9 +3,10 @@ # Licensed under a 3-clause BSD style license (see LICENSE) import pytest -from .common import MQ_VERSION_RAW +from .common import MQ_VERSION_RAW, skip_windows_ci +@skip_windows_ci @pytest.mark.integration def test_metadata(get_check, instance, datadog_agent): check = get_check(instance) diff --git a/ibm_mq/tox.ini b/ibm_mq/tox.ini index 095ff94af7037..960fb8075fffb 100644 --- a/ibm_mq/tox.ini +++ b/ibm_mq/tox.ini @@ -24,7 +24,9 @@ dd_mypy_args = '.*/config_models/.*\.py$' dd_mypy_deps = types-mock==0.1.5 -platform = linux|darwin|win32 +platform = + linux|darwin + py38-9: linux|darwin|win32 deps = -e../datadog_checks_base[deps] -rrequirements-dev.txt @@ -33,7 +35,8 @@ commands = pip install -r requirements.in pytest -v {posargs} setenv = - LD_LIBRARY_PATH=/opt/mqm/lib64:/opt/mqm/lib:{env:LD_LIBRARY_PATH:none} + LD_LIBRARY_PATH=/opt/mqm/lib64{:}/opt/mqm/lib{:}C:\ibm_mq{:}{env:LD_LIBRARY_PATH:none} + MQ_FILE_PATH=C:\ibm_mq 8: IBM_MQ_VERSION = 8 8: IBM_MQ_COMPOSE_VERSION = 8 8: IBM_MQ_VERSION_RAW = 8.0.0.4