Add data files to the wheel package

[ofek]

What does this PR do?

This moves the yaml configs to a location that can be picked up by pip/wheel.

Motivation

They aren't being included currently.

Additional Notes

Script (Python 3 only)

import os
import re
from glob import iglob
from itertools import chain
from shutil import move

CORE_DIR = r'C:\Users\Ofek\Desktop\code\integrations-core'
BASE_CHECKS = {'datadog_checks_base', 'datadog_checks_tests_helper'}
URL_PATTERN = 'https?://github.com/DataDog/integrations-core/blob/master{}'
URL_ROOT = URL_PATTERN.replace('?', '', 1)
MANIFEST_JMX = """\
include README.md
include manifest.json
graft datadog_checks
"""
MANIFEST_TEMPLATE = """\
graft datadog_checks
graft {test_dir}

include MANIFEST.in
include README.md
include requirements.in
include requirements.txt
include requirements-dev.txt
include manifest.json

global-exclude *.py[cod] __pycache__
"""


def get_setup_file(check_name):
    f = os.path.join(CORE_DIR, check_name, 'setup.py')
    if os.path.isfile(f):
        return f


def get_data_dir(check_name):
    return os.path.join(CORE_DIR, check_name, 'datadog_checks', check_name, 'data')


def get_relative_url_path(path):
    return path.replace(CORE_DIR, '').replace(os.path.sep, '/')


def main():
    for check_name in os.listdir(CORE_DIR):
        setup_file = get_setup_file(check_name)
        if setup_file and check_name not in BASE_CHECKS:
            check_dir = os.path.join(CORE_DIR, check_name)

            # Create new data directory
            data_dir = get_data_dir(check_name)
            if not os.path.isdir(data_dir):
                os.makedirs(data_dir)

            # Move package data to new location
            data_files = [
                (old_data_file, move(old_data_file, data_dir))
                for old_data_file in chain(
                    iglob(os.path.join(check_dir, 'auto_conf.yaml')),
                    iglob(os.path.join(check_dir, 'conf.yaml.*')),
                    iglob(os.path.join(check_dir, 'metrics.yaml')),
                )
            ]

            # Ensure we specify package data
            if os.path.isdir(os.path.join(check_dir, 'test')):
                with open(os.path.join(check_dir, 'MANIFEST.in'), 'w') as f:
                    f.write(MANIFEST_TEMPLATE.format(test_dir='test'))
            elif os.path.isdir(os.path.join(check_dir, 'tests')):
                with open(os.path.join(check_dir, 'MANIFEST.in'), 'w') as f:
                    f.write(MANIFEST_TEMPLATE.format(test_dir='tests'))
            else:
                with open(os.path.join(check_dir, 'MANIFEST.in'), 'w') as f:
                    f.write(MANIFEST_JMX)

            # Update URLs in README.md
            readme_file = os.path.join(check_dir, 'README.md')
            if os.path.isfile(readme_file):
                with open(readme_file, 'r', encoding='utf-8') as f:
                    readme_contents = f.read()

                for old_data_file, new_data_file in data_files:
                    readme_contents = re.sub(
                        URL_PATTERN.format(get_relative_url_path(old_data_file)),
                        URL_ROOT.format(get_relative_url_path(new_data_file)),
                        readme_contents
                    )

                with open(readme_file, 'w', encoding='utf-8') as f:
                    f.write(readme_contents)

            # Ensure we include package data in build
            with open(setup_file, 'r', encoding='utf-8') as f:
                setup_lines = f.readlines()

            for i, line in enumerate(reversed(setup_lines), 1):
                if line.strip().startswith('package_data='):
                    del setup_lines[len(setup_lines) - i]
                    break

            with open(setup_file, 'w', encoding='utf-8') as f:
                f.writelines(setup_lines)

if __name__ == '__main__':
    main()

[masci]

🤞