Switch to Azure Pipelines

.azure-pipelines/All.yml

@@ -0,0 +1,27 @@
+schedules:
+- cron: "0 5 * * *"
+  displayName: Nightly (EDT/EST)
+  always: true
+  branches:
+    include:
+    - master
+
+# Re-enable when we have enough runners
+# trigger:
+#   branches:
+#     include:
+#     - master
+
+pr:
+  branches:
+    include:
+    - master
+  paths:
+    include:
+    - datadog_checks_base/datadog_checks/*
+
+variables:
+  DDEV_COLOR: 1
+
+jobs:
+- template: './templates/test-all.yml'

.azure-pipelines/Changes.yml

@@ -0,0 +1,25 @@
+trigger: none
+
+pr:
+  branches:
+    include:
+    - master
+  paths:
+    exclude:
+    - datadog_checks_base/datadog_checks/*
+
+variables:
+  DDEV_COLOR: 1
+
+jobs:
+- template: './templates/test-single-linux.yml'
+  parameters:
+    job_name: Changed
+    display: Linux
+    validate: true
+
+- template: './templates/test-single-windows.yml'
+  parameters:
+    job_name: Changed
+    check: '--changed datadog_checks_base datadog_checks_dev active_directory aspdotnet disk dotnetclr exchange_server iis pdh_check sqlserver win32_event_log windows_service wmi_check'
+    display: Windows

.azure-pipelines/scripts/aerospike/linux/55_build_client.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -ex
+
+# https://github.com/aerospike/aerospike-client-c#build-prerequisites
+sudo apt-get update
+sudo apt-get install -y --no-install-recommends libc6-dev libssl-dev autoconf automake libtool g++ ncurses-dev
+
+# The binary wheels on PyPI are not yet compatible with OpenSSL 1.1.0+, see:
+# https://github.com/aerospike/aerospike-client-python/issues/214#issuecomment-385451007
+# https://github.com/aerospike/aerospike-client-python/issues/227#issuecomment-423220411
+git clone https://github.com/aerospike/aerospike-client-c.git /tmp/aerospike-client-c
+cd /tmp/aerospike-client-c
+
+# This needs to be kept in sync with whatever the Python library was built with.
+# For example, version 3.7.2 was built with version 4.6.3 of the C library, see:
+# https://github.com/aerospike/aerospike-client-python/blob/3.7.2/setup.py#L32-L33
+git checkout 4.4.0
+
+git submodule update --init
+make clean
+make
+
+set +ex

.azure-pipelines/scripts/cacti/linux/55_install_rrdtool.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -ex
+
+sudo apt-get update
+sudo apt-get install -y --no-install-recommends librrd-dev rrdtool
+
+set +ex

.azure-pipelines/scripts/datadog_checks_base/linux/55_install_kerberos.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -ex
+
+sudo apt-get update
+sudo apt-get install -y --no-install-recommends libkrb5-dev
+
+set +ex

.azure-pipelines/scripts/elastic/linux/55_increase_docker_memory.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -ex
+
+sudo sysctl -w vm.max_map_count=262144
+
+set +ex

.azure-pipelines/scripts/ibm_mq/linux/55_install_client.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+set -ex
+
+TMP_DIR=/tmp/mq
+MQ_URL=https://s3.amazonaws.com/dd-agent-tarball-mirror/mqadv_dev90_linux_x86-64.tar.gz
+MQ_PACKAGES="MQSeriesRuntime-*.rpm MQSeriesServer-*.rpm MQSeriesMsg*.rpm MQSeriesJava*.rpm MQSeriesJRE*.rpm MQSeriesGSKit*.rpm"
+
+if [ -e /opt/mqm/inc/cmqc.h ]; then
+  echo "cmqc.h already exists, exiting"
+  set +ex
+  exit 0
+fi
+
+sudo apt-get update
+sudo apt-get install -y --no-install-recommends \
+  bash \
+  bc \
+  coreutils \
+  curl \
+  debianutils \
+  findutils \
+  gawk \
+  grep \
+  libc-bin \
+  mount \
+  passwd \
+  procps \
+  rpm \
+  sed \
+  tar \
+  util-linux
+
+mkdir -p $TMP_DIR
+pushd $TMP_DIR
+  curl -LO $MQ_URL
+  tar -zxvf ./*.tar.gz
+  pushd MQServer
+    sudo ./mqlicense.sh -text_only -accept
+    sudo rpm -ivh --force-debian *.rpm
+    sudo /opt/mqm/bin/setmqinst -p /opt/mqm -i
+  popd
+popd
+
+ls /opt/mqm
+ls /opt/mqm/inc/
+
+set +ex

.azure-pipelines/scripts/run.py

@@ -0,0 +1,60 @@
+import os
+import platform
+import re
+import subprocess
+import sys
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+PLATFORM = (
+    'windows' if platform.system() == 'Windows'
+    else 'macos' if platform.system() == 'Darwin'
+    else 'linux'
+)
+
+
+def display_action(script_file):
+    display_header = f'Running: {script_file}'
+    print(f'\n{display_header}\n{"-" * len(display_header)}\n')
+
+
+def main():
+    checks = [c.strip() for c in sys.argv[1:]]
+
+    if checks:
+        if checks[0] == 'skip':
+            print('Skipping set up')
+        else:
+            print(f'Checks chosen: {repr(checks).strip("[]")}')
+    else:
+        print(f'Checks chosen: changed')
+
+    command = ['ddev', '--no-color', 'test', '--list']
+    command.extend(checks)
+
+    print('Detecting changed checks...')
+    result = subprocess.run(command, encoding='utf-8', capture_output=True, check=True)
+    checks = sorted(c.strip('`') for c in re.findall('^`[^`]+`', result.stdout, re.M))
+
+    for check in checks:
+        check_path = os.path.join(HERE, check)
+        if not os.path.isdir(check_path):
+            continue
+
+        contents = os.listdir(check_path)
+        if 'run.py' in contents:
+            script_file = os.path.join(check_path, 'run.py')
+            display_action(script_file)
+            subprocess.run([sys.executable, script_file], check=True)
+        elif PLATFORM in contents:
+            print(f'\nSetting up: {check}')
+            scripts_path = os.path.join(check_path, PLATFORM)
+            scripts = sorted(os.listdir(scripts_path))
+
+            for script in scripts:
+                script_file = os.path.join(scripts_path, script)
+                display_action(script_file)
+                subprocess.run([script_file], shell=True, check=True)
+
+
+if __name__ == '__main__':
+    main()

.azure-pipelines/scripts/sqlserver/linux/55_install_odbc.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -ex
+
+sudo apt-get update
+sudo apt-get install -y --no-install-recommends tdsodbc unixodbc-dev
+
+set +ex

.azure-pipelines/scripts/sqlserver/windows/32_install_odbc.bat

@@ -0,0 +1,5 @@
+:: https://docs.microsoft.com/en-us/sql/connect/odbc/windows/system-requirements-installation-and-driver-files
+:: Finding the actual URL not gated by a form was a nightmare
+powershell -Command "Invoke-WebRequest https://download.microsoft.com/download/E/6/B/E6BFDC7A-5BCD-4C51-9912-635646DA801E/en-US/msodbcsql_17.3.1.1_x64.msi -OutFile msodbcsql.msi"
+msiexec /quiet /passive /qn /i msodbcsql.msi IACCEPTMSODBCSQLLICENSETERMS=YES
+del msodbcsql.msi

.azure-pipelines/scripts/sqlserver/windows/55_install_sqlserver.bat

@@ -0,0 +1,12 @@
+:: Install with TCP/IP enabled, see: https://chocolatey.org/packages/sql-server-2017
+choco install sql-server-2017 --params="'/TCPENABLED:1'"
+
+:: Set password
+sqlcmd -Q "ALTER LOGIN sa with PASSWORD = 'Password12!';ALTER LOGIN sa ENABLE;"
+
+:: Enable port
+powershell -Command "stop-service MSSQLSERVER"
+powershell -Command "set-itemproperty -path 'HKLM:\software\microsoft\microsoft sql server\mssql14.MSSQLSERVER\mssqlserver\supersocketnetlib\tcp\ipall' -name tcpdynamicports -value ''"
+powershell -Command "set-itemproperty -path 'HKLM:\software\microsoft\microsoft sql server\mssql14.MSSQLSERVER\mssqlserver\supersocketnetlib\tcp\ipall' -name tcpport -value 1433"
+powershell -Command "set-itemproperty -path 'HKLM:\software\microsoft\microsoft sql server\mssql14.MSSQLSERVER\mssqlserver\' -name LoginMode -value 2"
+powershell -Command "start-service MSSQLSERVER"

.azure-pipelines/scripts/tokumx/linux/55_disable_hugepages.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -ex
+
+# Turn off hugepages and defrag, see:
+# https://github.com/DataDog/integrations-core/pull/2134
+(echo never | sudo tee /sys/kernel/mm/transparent_hugepage/enabled || true)
+(echo never | sudo tee /sys/kernel/mm/transparent_hugepage/defrag || true)
+
+set +ex

.azure-pipelines/templates/checkout-code.yml

@@ -0,0 +1,17 @@
+steps:
+# https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema#checkout
+- checkout: self
+  fetchDepth: 3
+
+# Azure pulls a commit hash via git fetch rather than clone so
+# we checkout the branches we need which also gets us out of a
+# detached HEAD state. This may be fixed soon, see:
+# https://developercommunity.visualstudio.com/content/problem/373462/git-get-sources-shallow-fetch-is-broken-subject-to.html
+- script: git checkout master
+  displayName: 'Checkout master'
+
+# Switch to the pull request branch last. Also see:
+# https://docs.microsoft.com/en-us/azure/devops/pipelines/build/variables#system-variables
+- script: git checkout $(System.PullRequest.SourceBranch)
+  condition: eq(variables['Build.Reason'], 'PullRequest')
+  displayName: 'Checkout branch'

.azure-pipelines/templates/install-deps.yml

@@ -0,0 +1,25 @@
+steps:
+# Install the Python version with which to use globally last as
+# these tasks prepend to PATH instead of appending to it. See:
+# https://docs.microsoft.com/en-us/azure/devops/pipelines/tasks/tool/use-python-version
+- task: UsePythonVersion@0
+  inputs:
+    versionSpec: '2.7'
+  displayName: 'Use Python 2.7'
+
+- task: UsePythonVersion@0
+  inputs:
+    versionSpec: '3.7'
+  displayName: 'Use Python 3.7'
+
+- script: python -m pip install --disable-pip-version-check --upgrade pip setuptools
+  displayName: 'Upgrade Python packaging tools'
+
+- script: pip install --disable-pip-version-check --upgrade codecov
+  displayName: 'Install Codecov'
+
+- script: pip install ./datadog_checks_dev[cli]
+  displayName: 'Install ddev'
+
+- script: ddev config set core .
+  displayName: 'Configure ddev'

.azure-pipelines/templates/run-validations.yml

@@ -0,0 +1,21 @@
+steps:
+- script: ddev validate agent-reqs
+  displayName: 'Validate Agent requirements'
+
+- script: ddev validate config
+  displayName: 'Validate default configuration files'
+
+- script: ddev validate dep
+  displayName: 'Validate dependencies'
+
+- script: ddev validate logos
+  displayName: 'Validate logos'
+
+- script: ddev validate manifest --include-extras
+  displayName: 'Validate manifest files'
+
+- script: ddev validate metadata
+  displayName: 'Validate metric data'
+
+- script: ddev validate service-checks
+  displayName: 'Validate service check data'

.azure-pipelines/templates/set-up-integrations.yml

@@ -0,0 +1,7 @@
+parameters:
+  check: ''
+
+steps:
+# Need unbuffered IO, see: https://github.com/Microsoft/azure-pipelines-yaml/issues/106
+- script: python -u .azure-pipelines/scripts/run.py ${{ parameters.check }}
+  displayName: 'Set up integration requirements'

.azure-pipelines/templates/set-up-windows.yml

@@ -0,0 +1,5 @@
+steps:
+# Newer versions of Windows require this for full use of psutil.
+# See: https://github.com/giampaolo/psutil/issues/351
+- script: diskperf -y
+  displayName: 'Enable disk performance counters'