From be092967e5195307eda51d1574be2b96caaa63eb Mon Sep 17 00:00:00 2001 From: Ofek Lev Date: Mon, 20 Dec 2021 12:23:35 -0500 Subject: [PATCH] try to fix build --- .azure-pipelines/changes.yml | 30 ++++----- .../ibm_mq/windows/55_install_client.py | 67 +++++++++++++++++++ .../scripts/ibm_mq/windows/install_ibm_mq.bat | 12 ---- ibm_mq/tests/common.py | 6 +- ibm_mq/tests/conftest.py | 60 +++++++++++------ ibm_mq/tox.ini | 8 ++- 6 files changed, 126 insertions(+), 57 deletions(-) create mode 100644 .azure-pipelines/scripts/ibm_mq/windows/55_install_client.py delete mode 100644 .azure-pipelines/scripts/ibm_mq/windows/install_ibm_mq.bat diff --git a/.azure-pipelines/changes.yml b/.azure-pipelines/changes.yml index 87b3b7892129ca..d521c9ed8bab33 100644 --- a/.azure-pipelines/changes.yml +++ b/.azure-pipelines/changes.yml @@ -14,25 +14,25 @@ variables: DDEV_COLOR: 1 DD_TRACE_AGENT_PORT: 8127 -resources: - containers: - - ${{ if eq(variables['System.PullRequest.IsFork'], 'False') }}: - - container: dd_agent - image: gcr.io/datadoghq/agent:latest - ports: - - 8127:8126 - env: - DD_API_KEY: $(DD_CI_API_KEY) - DD_HOSTNAME: "none" - DD_INSIDE_CI: "true" +# resources: +# containers: +# - ${{ if eq(variables['System.PullRequest.IsFork'], 'False') }}: +# - container: dd_agent +# image: gcr.io/datadoghq/agent:latest +# ports: +# - 8127:8126 +# env: +# DD_API_KEY: $(DD_CI_API_KEY) +# DD_HOSTNAME: "none" +# DD_INSIDE_CI: "true" jobs: - template: './templates/test-single-linux.yml' parameters: job_name: Changed display: Linux - ${{ if eq(variables['System.PullRequest.IsFork'], 'False') }}: - ddtrace_flag: '--ddtrace' + # ${{ if eq(variables['System.PullRequest.IsFork'], 'False') }}: + # ddtrace_flag: '--ddtrace' validate: false validate_codeowners: false validate_changed: changed @@ -47,8 +47,8 @@ jobs: parameters: job_name: Changed check: '--changed datadog_checks_base datadog_checks_dev active_directory aspdotnet disk dns_check dotnetclr exchange_server ibm_mq iis network pdh_check sqlserver tcp_check win32_event_log windows_performance_counters windows_service wmi_check' - ${{ if eq(variables['System.PullRequest.IsFork'], 'False') }}: - ddtrace_flag: '--ddtrace' + # ${{ if eq(variables['System.PullRequest.IsFork'], 'False') }}: + # ddtrace_flag: '--ddtrace' validate_changed: changed display: Windows pip_cache_config: diff --git a/.azure-pipelines/scripts/ibm_mq/windows/55_install_client.py b/.azure-pipelines/scripts/ibm_mq/windows/55_install_client.py new file mode 100644 index 00000000000000..89d15701f46eed --- /dev/null +++ b/.azure-pipelines/scripts/ibm_mq/windows/55_install_client.py @@ -0,0 +1,67 @@ +import os +import subprocess +from tempfile import TemporaryDirectory +from zipfile import ZipFile + +import requests + +SERVER_VERSION = '924' +SERVER_ARCHIVE_NAME = f'mqadv_dev{SERVER_VERSION}_windows.zip' +SERVER_URL = f'https://public.dhe.ibm.com/ibmdl/export/pub/software/websphere/messaging/mqadv/{SERVER_ARCHIVE_NAME}' + +CLIENT_VERSION = '9.2.2.0' +CLIENT_ARCHIVE_NAME = f'{CLIENT_VERSION}-IBM-MQC-Redist-Win64.zip' +CLIENT_URL = f'https://public.dhe.ibm.com/ibmdl/export/pub/software/websphere/messaging/mqdev/redist/{CLIENT_ARCHIVE_NAME}' +CLIENT_TARGET_DIR = 'C:\\ibm_mq' + + +def download_file(url, file_name): + response = requests.get(url, stream=True) + response.raise_for_status() + + with open(file_name, 'wb') as f: + for chunk in response.iter_content(16384): + f.write(chunk) + + +def main(): + with TemporaryDirectory() as d: + temp_dir = os.path.realpath(d) + + print('Downloading client') + client_archive_path = os.path.join(temp_dir, CLIENT_ARCHIVE_NAME) + download_file(CLIENT_URL, client_archive_path) + + print('Extracting client') + with ZipFile(client_archive_path) as zip_file: + zip_file.extractall(CLIENT_TARGET_DIR) + + print('Downloading server') + server_target_dir = os.path.join(temp_dir, 'server') + server_archive_path = os.path.join(temp_dir, SERVER_ARCHIVE_NAME) + download_file(SERVER_URL, server_archive_path) + + print('Extracting server') + with ZipFile(server_archive_path) as zip_file: + zip_file.extractall(server_target_dir) + + print('Installing server') + msi_path = os.path.join(server_target_dir, 'MQServer', 'MSI', 'IBM MQ.msi') + subprocess.check_call( + [ + 'msiexec', + '/quiet', + '/passive', + '/qn', + '/i', + msi_path, + 'TRANSFORMS=1033.mst', + 'AGREETOLICENSE=YES', + 'ADDLOCAL=Server', + ], + shell=True, + ) + + +if __name__ == '__main__': + main() diff --git a/.azure-pipelines/scripts/ibm_mq/windows/install_ibm_mq.bat b/.azure-pipelines/scripts/ibm_mq/windows/install_ibm_mq.bat deleted file mode 100644 index b5a1c0a3019cab..00000000000000 --- a/.azure-pipelines/scripts/ibm_mq/windows/install_ibm_mq.bat +++ /dev/null @@ -1,12 +0,0 @@ -set target=c:\ibm_mq -set version=9.2.2.0 -set file=%version%-IBM-MQC-Redist-Win64.zip -set ibm_source=https://public.dhe.ibm.com/ibmdl/export/pub/software/websphere/messaging/mqdev/redist/%file% -set "source=https://drive.google.com/uc?export=download&id=1ysTsHxAAxIwl0EwFnYzN5cNRBnvKwELw" - -md %target% -:: Downloading IBM MQ client -powershell -command "Invoke-WebRequest -Uri %source% -OutFile %target%\%file%" -:: Extracting IBM MQ client -powershell -command "Expand-Archive -LiteralPath %target%\%file% -DestinationPath %target%" -dir %target% diff --git a/ibm_mq/tests/common.py b/ibm_mq/tests/common.py index 2cb7946133350b..c26d244c318fbf 100644 --- a/ibm_mq/tests/common.py +++ b/ibm_mq/tests/common.py @@ -140,11 +140,7 @@ E2E_METADATA = { 'docker_volumes': ['{}/agent_scripts/start_commands.sh:/tmp/start_commands.sh'.format(HERE)], 'start_commands': ['bash /tmp/start_commands.sh'], - 'env_vars': { - 'LD_LIBRARY_PATH': '/opt/mqm/lib64:/opt/mqm/lib', - 'C_INCLUDE_PATH': '/opt/mqm/inc', - 'MQ_FILE_PATH': 'c:\\ibm_mq', - }, + 'env_vars': {'LD_LIBRARY_PATH': '/opt/mqm/lib64:/opt/mqm/lib', 'C_INCLUDE_PATH': '/opt/mqm/inc'}, } QUEUE_METRICS = [ diff --git a/ibm_mq/tests/conftest.py b/ibm_mq/tests/conftest.py index 7aff4d7696eddd..b1b294a1bcf46f 100644 --- a/ibm_mq/tests/conftest.py +++ b/ibm_mq/tests/conftest.py @@ -5,11 +5,13 @@ import logging import os import re +import subprocess import pytest from six.moves import range -from datadog_checks.dev import docker_run +from datadog_checks.dev import docker_run, environment_run +from datadog_checks.dev.ci import running_on_ci from datadog_checks.dev.conditions import CheckDockerLogs, WaitFor from datadog_checks.dev.utils import ON_WINDOWS @@ -138,11 +140,22 @@ def consume(): def prepare_queue_manager(): import pymqi - conn_info = '{0}({1})'.format(common.HOST, common.PORT) - qm_name = common.QUEUE_MANAGER.lower() + if ON_WINDOWS: + # https://developer.ibm.com/tutorials/mq-connect-app-queue-manager-windows/ + exe_dir = os.path.join('C:', 'Program Files', 'IBM', 'MQ', 'bin') + subprocess.check_call([os.path.join(exe_dir, 'setmqenv'), '-s'], shell=True) + subprocess.check_call([os.path.join(exe_dir, 'crtmqm'), 'QM1'], shell=True) + subprocess.check_call([os.path.join(exe_dir, 'strmqm'), 'QM1'], shell=True) + qmgr = pymqi.connect(common.QUEUE_MANAGER) + else: + conn_info = '{0}({1})'.format(common.HOST, common.PORT) + qm_name = common.QUEUE_MANAGER.lower() + + qmgr = pymqi.QueueManager(None) + qmgr.connectTCPClient( + common.QUEUE_MANAGER, pymqi.CD(), common.CHANNEL, conn_info, common.USERNAME, common.PASSWORD + ) - qmgr = pymqi.QueueManager(None) - qmgr.connectTCPClient(common.QUEUE_MANAGER, pymqi.CD(), common.CHANNEL, conn_info, common.USERNAME, common.PASSWORD) pcf = pymqi.PCFExecute(qmgr, response_wait_interval=5000) attrs = [ @@ -196,21 +209,24 @@ def prepare_queue_manager(): @pytest.fixture(scope='session') def dd_environment(): - - if common.MQ_VERSION == 9: - log_pattern = "AMQ5026I: The listener 'DEV.LISTENER.TCP' has started. ProcessId" - elif common.MQ_VERSION == 8: - log_pattern = r".*QMNAME\({}\)\s*STATUS\(Running\).*".format(common.QUEUE_MANAGER) + if ON_WINDOWS and running_on_ci(): + with environment_run(up=lambda: None, down=lambda: None, conditions=[prepare_queue_manager]): + yield common.INSTANCE else: - raise RuntimeError('Invalid version: {}'.format(common.MQ_VERSION)) - - e2e_meta = copy.deepcopy(common.E2E_METADATA) - e2e_meta.setdefault('docker_volumes', []) - e2e_meta['docker_volumes'].append("{}:/opt/pki/keys".format(os.path.join(common.HERE, 'keys'))) - - conditions = [CheckDockerLogs('ibm_mq1', log_pattern)] - if not ON_WINDOWS: - conditions.append(WaitFor(prepare_queue_manager)) - - with docker_run(compose_file=common.COMPOSE_FILE_PATH, build=True, conditions=conditions, sleep=10, attempts=2): - yield common.INSTANCE, e2e_meta + if common.MQ_VERSION == 9: + log_pattern = "AMQ5026I: The listener 'DEV.LISTENER.TCP' has started. ProcessId" + elif common.MQ_VERSION == 8: + log_pattern = r".*QMNAME\({}\)\s*STATUS\(Running\).*".format(common.QUEUE_MANAGER) + else: + raise RuntimeError('Invalid version: {}'.format(common.MQ_VERSION)) + + e2e_meta = copy.deepcopy(common.E2E_METADATA) + e2e_meta.setdefault('docker_volumes', []) + e2e_meta['docker_volumes'].append("{}:/opt/pki/keys".format(os.path.join(common.HERE, 'keys'))) + + conditions = [CheckDockerLogs('ibm_mq1', log_pattern)] + if not ON_WINDOWS: + conditions.append(WaitFor(prepare_queue_manager)) + + with docker_run(compose_file=common.COMPOSE_FILE_PATH, build=True, conditions=conditions, sleep=10, attempts=2): + yield common.INSTANCE, e2e_meta diff --git a/ibm_mq/tox.ini b/ibm_mq/tox.ini index 06fb0f857f4422..4d9dc3c9cc66c6 100644 --- a/ibm_mq/tox.ini +++ b/ibm_mq/tox.ini @@ -24,7 +24,9 @@ dd_mypy_args = '.*/config_models/.*\.py$' dd_mypy_deps = types-mock==0.1.5 -platform = linux|darwin|win32 +platform = + linux|darwin + py38-9: linux|darwin|win32 deps = -e../datadog_checks_base[deps] -rrequirements-dev.txt @@ -33,8 +35,8 @@ commands = pip install -r requirements.in pytest -v {posargs} setenv = - LD_LIBRARY_PATH=/opt/mqm/lib64:/opt/mqm/lib:{env:LD_LIBRARY_PATH:none} - MQ_FILE_PATH=c:\ibm_mq + LD_LIBRARY_PATH=/opt/mqm/lib64{:}/opt/mqm/lib{:}C:\ibm_mq{:}{env:LD_LIBRARY_PATH:none} + MQ_FILE_PATH=C:\ProgramData\IBM\MQ 8: IBM_MQ_VERSION = 8 8: IBM_MQ_COMPOSE_VERSION = 8 8: IBM_MQ_VERSION_RAW = 8.0.0.4