Add windows support (#10737)

* Add support for windows

Co-authored-by: Ofek Lev <ofekmeister@gmail.com>

.azure-pipelines/changes.yml

@@ -46,7 +46,7 @@ jobs:
 - template: './templates/test-single-windows.yml'
   parameters:
     job_name: Changed
-    check: '--changed datadog_checks_base datadog_checks_dev active_directory aspdotnet disk dns_check dotnetclr exchange_server iis network pdh_check sqlserver tcp_check win32_event_log windows_performance_counters windows_service wmi_check'
+    check: '--changed datadog_checks_base datadog_checks_dev active_directory aspdotnet disk dns_check dotnetclr exchange_server ibm_mq iis network pdh_check sqlserver tcp_check win32_event_log windows_performance_counters windows_service wmi_check'
     ${{ if eq(variables['System.PullRequest.IsFork'], 'False') }}:
       ddtrace_flag: '--ddtrace'
     validate_changed: changed

.azure-pipelines/scripts/ibm_mq/windows/55_install_ibm_mq_client.py

@@ -0,0 +1,37 @@
+import os
+from tempfile import TemporaryDirectory
+from zipfile import ZipFile
+
+import requests
+
+
+CLIENT_VERSION = '9.2.2.0'
+CLIENT_ARCHIVE_NAME = f'{CLIENT_VERSION}-IBM-MQC-Redist-Win64.zip'
+CLIENT_URL = f'https://public.dhe.ibm.com/ibmdl/export/pub/software/websphere/messaging/mqdev/redist/{CLIENT_ARCHIVE_NAME}'
+CLIENT_TARGET_DIR = 'C:\\ibm_mq'
+
+
+def download_file(url, file_name):
+    response = requests.get(url, stream=True)
+    response.raise_for_status()
+
+    with open(file_name, 'wb') as f:
+        for chunk in response.iter_content(16384):
+            f.write(chunk)
+
+
+def main():
+    with TemporaryDirectory() as d:
+        temp_dir = os.path.realpath(d)
+
+        print('Downloading client')
+        client_archive_path = os.path.join(temp_dir, CLIENT_ARCHIVE_NAME)
+        download_file(CLIENT_URL, client_archive_path)
+
+        print('Extracting client')
+        with ZipFile(client_archive_path) as zip_file:
+            zip_file.extractall(CLIENT_TARGET_DIR)
+
+
+if __name__ == '__main__':
+    main()

.azure-pipelines/templates/test-all-checks.yml

@@ -226,8 +226,11 @@ jobs:
       displayName: IBM i
       os: linux
     - checkName: ibm_mq
-      displayName: IBM MQ
+      displayName: IBM MQ (Linux)
       os: linux
+    - checkName: ibm_mq
+      displayName: IBM MQ (Windows)
+      os: windows
     - checkName: ibm_was
       displayName: IBM WAS
       os: linux

datadog_checks_base/datadog_checks/base/data/agent_requirements.in

@@ -59,7 +59,7 @@ pyhdb==0.3.4
 pyjwt==1.7.1; python_version < "3.0"
 pyjwt==2.0.1; python_version > "3.0"
 pymongo==3.11.4
-pymqi==1.12.0; sys_platform != "win32"
+pymqi==1.12.0
 pymysql==0.9.3
 pyodbc==4.0.26
 pyro4==4.73; sys_platform == "win32"

ibm_mq/README.md

@@ -10,10 +10,12 @@ This check monitors [IBM MQ][1] versions 8 to 9.0.
 
 The IBM MQ check is included in the [Datadog Agent][2] package.
 
-To use the IBM MQ check, you need to:
+To use the IBM MQ check, you need to make sure the [IBM MQ Client][3] 9.1+ is installed (unless a compatible version of IBM MQ server is installed on the Agent host).
+
+#### On Linux
 
-1. Make sure the [IBM MQ Client][3] 9.1+ is installed (unless a compatible version of IBM MQ server is installed on the Agent host).
-2. Update your `LD_LIBRARY_PATH` and `C_INCLUDE_PATH` to include the location of the libraries. (Create these two environment variables if they don’t exist yet.) For example:
+Update your `LD_LIBRARY_PATH` and `C_INCLUDE_PATH` to include the location of the libraries. (Create these two environment variables if they don’t exist yet.)
+For example, if you installed it in `/opt`:
 
 ```text
 export LD_LIBRARY_PATH=/opt/mqm/lib64:/opt/mqm/lib:$LD_LIBRARY_PATH
@@ -129,7 +131,12 @@ Update the bindings:
 sudo ldconfig
 ```
 
-#### Permissions and authentication
+#### On Windows
+
+There is a file called `mqclient.ini` in the IBM MQ data directory. It is normally `C:\ProgramData\IBM\MQ`.
+Configure the environment variable `MQ_FILE_PATH`, to point at the data directory.
+
+### Permissions and authentication
 
 There are many ways to set up permissions in IBM MQ. Depending on how your setup works, create a `datadog` user within MQ with read only permissions.
 

ibm_mq/manifest.json

@@ -11,7 +11,8 @@
   "support": "core",
   "supported_os": [
     "linux",
-    "mac_os"
+    "mac_os",
+    "windows"
   ],
   "public_title": "Datadog-IBM MQ Integration",
   "categories": [

ibm_mq/requirements.in

@@ -1 +1 @@
-pymqi==1.12.0; sys_platform != 'win32'
+pymqi==1.12.0

ibm_mq/tests/common.py

@@ -4,14 +4,22 @@
 
 import os
 
+import pytest
+
 from datadog_checks.dev import get_docker_hostname
+from datadog_checks.dev.ci import running_on_ci
+from datadog_checks.dev.utils import ON_WINDOWS
 
 # Ignore missing library to not require it for e2e
 try:
     from datadog_checks.ibm_mq.metrics import COUNT, GAUGE
 except ImportError:
     COUNT = GAUGE = ''
 
+RUNNING_ON_WINDOWS_CI = ON_WINDOWS and running_on_ci()
+skip_windows_ci = pytest.mark.skipif(RUNNING_ON_WINDOWS_CI, reason='MQ server cannot be setup on Windows VMs in CI')
+
+
 HERE = os.path.dirname(os.path.abspath(__file__))
 COMPOSE_DIR = os.path.join(HERE, 'compose')
 

ibm_mq/tests/test_ibm_mq_e2e.py

@@ -6,18 +6,18 @@
 
 from datadog_checks.dev.utils import get_metadata_metrics
 
-from .common import MQ_VERSION, assert_all_metrics
+from .common import MQ_VERSION, assert_all_metrics, skip_windows_ci
+
+pytestmark = [skip_windows_ci, pytest.mark.e2e]
 
 
-@pytest.mark.e2e
 def test_e2e_check_all(dd_agent_check, instance_collect_all):
     aggregator = dd_agent_check(instance_collect_all, rate=True)
 
     assert_all_metrics(aggregator)
     aggregator.assert_metrics_using_metadata(get_metadata_metrics())
 
 
-@pytest.mark.e2e
 @pytest.mark.skipif(
     MQ_VERSION < 9, reason='Only test for for version >=9, for v8 use a custom image with custom setup.'
 )

ibm_mq/tests/test_ibm_mq_int.py

@@ -14,9 +14,9 @@
 from datadog_checks.dev.utils import get_metadata_metrics
 
 from . import common
-from .common import QUEUE_METRICS, assert_all_metrics
+from .common import QUEUE_METRICS, assert_all_metrics, skip_windows_ci
 
-pytestmark = [pytest.mark.usefixtures("dd_environment"), pytest.mark.integration]
+pytestmark = [skip_windows_ci, pytest.mark.usefixtures("dd_environment"), pytest.mark.integration]
 
 
 def test_no_msg_errors_are_caught(get_check, instance, caplog):

ibm_mq/tests/test_ibm_mq_unit.py

@@ -7,6 +7,8 @@
 
 from datadog_checks.base import AgentCheck, ConfigurationError
 
+from .common import skip_windows_ci
+
 pytestmark = pytest.mark.unit
 
 
@@ -182,6 +184,7 @@ def test_channel_queue_config_error(instance_config):
     assert 'channel, queue_manager are required configurations' in str(excinfo.value)
 
 
+@skip_windows_ci
 def test_ssl_connection_creation(get_check, instance):
     """
     Test that we are not getting unicode/bytes type error.

ibm_mq/tests/test_metadata.py

@@ -3,9 +3,10 @@
 # Licensed under a 3-clause BSD style license (see LICENSE)
 import pytest
 
-from .common import MQ_VERSION_RAW
+from .common import MQ_VERSION_RAW, skip_windows_ci
 
 
+@skip_windows_ci
 @pytest.mark.integration
 def test_metadata(get_check, instance, datadog_agent):
     check = get_check(instance)

ibm_mq/tox.ini

@@ -24,7 +24,9 @@ dd_mypy_args =
     '.*/config_models/.*\.py$'
 dd_mypy_deps =
     types-mock==0.1.5
-platform = linux|darwin|win32
+platform =
+    linux|darwin
+    py38-9: linux|darwin|win32
 deps =
     -e../datadog_checks_base[deps]
     -rrequirements-dev.txt
@@ -33,7 +35,8 @@ commands =
     pip install -r requirements.in
     pytest -v {posargs}
 setenv =
-    LD_LIBRARY_PATH=/opt/mqm/lib64:/opt/mqm/lib:{env:LD_LIBRARY_PATH:none}
+    LD_LIBRARY_PATH=/opt/mqm/lib64{:}/opt/mqm/lib{:}C:\ibm_mq{:}{env:LD_LIBRARY_PATH:none}
+    MQ_FILE_PATH=C:\ibm_mq
     8: IBM_MQ_VERSION = 8
     8: IBM_MQ_COMPOSE_VERSION = 8
     8: IBM_MQ_VERSION_RAW = 8.0.0.4