diff --git a/.builders/build.py b/.builders/build.py
index 564d5e06e4e49e..f2166f07a4da2e 100644
--- a/.builders/build.py
+++ b/.builders/build.py
@@ -69,37 +69,59 @@ def build_macos():
parser = argparse.ArgumentParser(prog='builder', allow_abbrev=False)
parser.add_argument('output_dir')
parser.add_argument('--python', default='3')
- parser.add_argument('--cache-dir',
- help='Path to directory to load or save a build cache.')
+ parser.add_argument('--builder-root', required=True,
+ help='Path to a folder where things will be installed during builder setup.')
+ parser.add_argument('--skip-setup', default=False, action='store_true',
+ help='Skip builder setup, assuming it has already been set up.')
args = parser.parse_args()
context_path = HERE / 'images' / 'macos'
+ builder_root = Path(args.builder_root).absolute()
+ builder_root.mkdir(exist_ok=True)
with temporary_directory() as temp_dir:
- build_context_dir = shutil.copytree(context_path, temp_dir, dirs_exist_ok=True)
+ mount_dir = temp_dir / 'mnt'
+ mount_dir.mkdir()
+
+ build_context_dir = shutil.copytree(context_path, mount_dir / 'build_context', dirs_exist_ok=True)
# Copy utilities shared by multiple images
for entry in context_path.parent.iterdir():
if entry.is_file():
shutil.copy2(entry, build_context_dir)
- mount_dir = temp_dir / 'mnt'
- mount_dir.mkdir()
-
dependency_file = mount_dir / 'requirements.in'
dependency_file.write_text('\n'.join(chain.from_iterable(read_dependencies().values())))
shutil.copy(HERE.parent / '.deps' / 'build_dependencies.txt', mount_dir)
shutil.copytree(HERE / 'scripts', mount_dir / 'scripts')
shutil.copytree(HERE / 'patches', mount_dir / 'patches')
+ prefix_path = builder_root / 'prefix'
env = {
**os.environ,
- 'DD_PREFIX_CACHE': args.cache_dir or '',
'DD_MOUNT_DIR': mount_dir,
+ # Paths to pythons
+ 'DD_PY3_BUILDENV_PATH': builder_root / 'py3' / 'bin' / 'python',
+ 'DD_PY2_BUILDENV_PATH': builder_root / 'py2' / 'bin' / 'python',
+ # Path where we'll install libraries that we build
+ 'DD_PREFIX_PATH': prefix_path,
+ # Common compilation flags
+ 'LDFLAGS': f'-Wl,-rpath,{prefix_path}/lib -L{prefix_path}/lib',
+ 'CFLAGS': f'-I{prefix_path}/include -O2',
+ # Build command for extra platform-specific build steps
+ 'DD_BUILD_COMMAND': f'bash {build_context_dir}/extra_build.sh'
}
+
+ if not args.skip_setup:
+ check_process(
+ ['bash', str(HERE / 'images' / 'macos' / 'builder_setup.sh')],
+ env=env,
+ cwd=builder_root,
+ )
+
check_process(
- ['bash', str(HERE / 'images' / 'macos' / 'build.sh'), '--python', args.python],
+ [os.environ['DD_PYTHON3'], str(mount_dir / 'scripts' / 'build_wheels.py'), '--python', args.python],
env=env,
- cwd=build_context_dir,
+ cwd=builder_root,
)
output_dir = Path(args.output_dir)
diff --git a/.builders/images/macos/build.sh b/.builders/images/macos/build.sh
deleted file mode 100644
index a02b694db653a1..00000000000000
--- a/.builders/images/macos/build.sh
+++ /dev/null
@@ -1,130 +0,0 @@
-#!/usr/bin/env bash
-
-set -euxo pipefail
-
-export MACOSX_DEPLOYMENT_TARGET="10.12"
-
-"${DD_PYTHON3}" -m pip install --no-warn-script-location --upgrade pip
-"${DD_PYTHON3}" -m pip install --no-warn-script-location virtualenv
-"${DD_PYTHON3}" -m virtualenv py3
-export DD_PY3_BUILDENV_PATH="$(pwd)/py3/bin/python"
-
-"${DD_PYTHON2}" -m pip install --no-warn-script-location --upgrade pip
-"${DD_PYTHON2}" -m pip install --no-warn-script-location virtualenv
-"${DD_PYTHON2}" -m virtualenv py2
-export DD_PY2_BUILDENV_PATH="$(pwd)/py2/bin/python"
-
-# Path where we'll install libraries that we build
-export DD_PREFIX_PATH="$(pwd)/prefix"
-
-export LDFLAGS="-Wl,-rpath,${DD_PREFIX_PATH}/lib -L${DD_PREFIX_PATH}/lib"
-export CFLAGS="-I${DD_PREFIX_PATH}/include -O2"
-export PATH="${DD_PREFIX_PATH}/bin:${PATH}"
-
-"${DD_PYTHON3}" -m pip install --no-warn-script-location -r "runner_dependencies.txt"
-
-# Install always with our own prefix path
-install-from-source() {
- bash install-from-source.sh --prefix="${DD_PREFIX_PATH}" "$@"
-}
-
-# mqi
-IBM_MQ_VERSION=9.2.4.0-IBM-MQ-DevToolkit
-curl --retry 5 --fail "https://s3.amazonaws.com/dd-agent-omnibus/ibm-mq-backup/${IBM_MQ_VERSION}-MacX64.pkg" -o /tmp/mq_client.pkg
-sudo installer -pkg /tmp/mq_client.pkg -target /
-rm -rf /tmp/mq_client.pkg
-
-# Restore cache if it exists
-if [[ -n ${DD_PREFIX_CACHE:-} && -d ${DD_PREFIX_CACHE:-} ]]; then
- echo "Using provided cache for built libraries."
- cp -r "${DD_PREFIX_CACHE}" "${DD_PREFIX_PATH}"
-else
- # openssl
- DOWNLOAD_URL="https://www.openssl.org/source/openssl-{{version}}.tar.gz" \
- VERSION="3.0.12" \
- SHA256="f93c9e8edde5e9166119de31755fc87b4aa34863662f67ddfcba14d0b6b69b61" \
- RELATIVE_PATH="openssl-{{version}}" \
- CONFIGURE_SCRIPT="./config" \
- install-from-source \
- -fPIC shared \
- no-module \
- no-comp no-idea no-mdc2 no-rc5 no-ssl3 no-gost
-
- # libxml & libxslt for lxml
- DOWNLOAD_URL="https://download.gnome.org/sources/libxml2/2.10/libxml2-{{version}}.tar.xz" \
- VERSION="2.10.3" \
- SHA256="5d2cc3d78bec3dbe212a9d7fa629ada25a7da928af432c93060ff5c17ee28a9c" \
- RELATIVE_PATH="libxml2-{{version}}" \
- install-from-source \
- --without-iconv \
- --without-python \
- --without-icu \
- --without-debug \
- --without-mem-debug \
- --without-run-debug \
- --without-legacy \
- --without-catalog \
- --without-docbook \
- --disable-static
-
- DOWNLOAD_URL="https://download.gnome.org/sources/libxslt/1.1/libxslt-{{version}}.tar.xz" \
- VERSION="1.1.37" \
- SHA256="3a4b27dc8027ccd6146725950336f1ec520928f320f144eb5fa7990ae6123ab4" \
- RELATIVE_PATH="libxslt-{{version}}" \
- install-from-source \
- --with-libxml-prefix="${DD_PREFIX_PATH}" \
- --without-python \
- --without-crypto \
- --without-profiler \
- --without-debugger \
- --disable-static
-
- # curl
- DOWNLOAD_URL="https://curl.haxx.se/download/curl-{{version}}.tar.gz" \
- VERSION="8.4.0" \
- SHA256="816e41809c043ff285e8c0f06a75a1fa250211bbfb2dc0a037eeef39f1a9e427" \
- RELATIVE_PATH="curl-{{version}}" \
- install-from-source \
- --disable-manual \
- --disable-debug \
- --enable-optimize \
- --disable-static \
- --disable-ldap \
- --disable-ldaps \
- --disable-rtsp \
- --enable-proxy \
- --disable-dependency-tracking \
- --enable-ipv6 \
- --without-libidn \
- --without-gnutls \
- --without-librtmp \
- --without-libssh2 \
- --with-ssl="${DD_PREFIX_PATH}"
- # Remove the binary installed so that we consistenly use the same original `curl` binary
- rm "${DD_PREFIX_PATH}/bin/curl"
-
- # Dependencies needed to build librdkafka (and thus, confluent-kafka) with kerberos support
- # Note that we don't ship these but rely on the Agent providing a working cyrus-sasl installation
- # with kerberos support, therefore we only need to watch out for the version of cyrus-sasl being
- # compatible with that in the Agent, the rest shouldn't matter much
- DOWNLOAD_URL="https://github.com/LMDB/lmdb/archive/LMDB_{{version}}.tar.gz" \
- VERSION="0.9.29" \
- SHA256="22054926b426c66d8f2bc22071365df6e35f3aacf19ad943bc6167d4cae3bebb" \
- RELATIVE_PATH="lmdb-LMDB_{{version}}/libraries/liblmdb" \
- CONFIGURE_SCRIPT="true" \
- install-from-source
- DOWNLOAD_URL="https://github.com/cyrusimap/cyrus-sasl/releases/download/cyrus-sasl-{{version}}/cyrus-sasl-{{version}}.tar.gz" \
- VERSION="2.1.28" \
- SHA256="7ccfc6abd01ed67c1a0924b353e526f1b766b21f42d4562ee635a8ebfc5bb38c" \
- RELATIVE_PATH="cyrus-sasl-{{version}}" \
- install-from-source --with-dblib=lmdb --enable-gssapi="${DD_PREFIX_PATH}" --disable-macos-framework
-
- # Cache everything under prefix
- if [[ -n ${DD_PREFIX_CACHE:-} ]]; then
- cp -r "${DD_PREFIX_PATH}" "${DD_PREFIX_CACHE}"
- fi
-fi
-
-export DD_BUILD_COMMAND="bash $(pwd)/extra_build.sh"
-
-"${DD_PYTHON3}" "${DD_MOUNT_DIR}/scripts/build_wheels.py" "$@"
diff --git a/.builders/images/macos/builder_setup.sh b/.builders/images/macos/builder_setup.sh
new file mode 100644
index 00000000000000..97bd8dbbdab9eb
--- /dev/null
+++ b/.builders/images/macos/builder_setup.sh
@@ -0,0 +1,105 @@
+#!/usr/bin/env bash
+
+set -euxo pipefail
+
+"${DD_PYTHON3}" -m pip install --no-warn-script-location --upgrade pip
+"${DD_PYTHON3}" -m pip install --no-warn-script-location virtualenv
+"${DD_PYTHON3}" -m virtualenv py3
+
+"${DD_PYTHON2}" -m pip install --no-warn-script-location --upgrade pip
+"${DD_PYTHON2}" -m pip install --no-warn-script-location virtualenv
+"${DD_PYTHON2}" -m virtualenv py2
+
+"${DD_PYTHON3}" -m pip install --no-warn-script-location -r "${DD_MOUNT_DIR}/build_context/runner_dependencies.txt"
+
+# Install always with our own prefix path
+cp "${DD_MOUNT_DIR}/build_context/install-from-source.sh" .
+install-from-source() {
+ bash "install-from-source.sh" --prefix="${DD_PREFIX_PATH}" "$@"
+}
+
+# mqi
+IBM_MQ_VERSION=9.2.4.0-IBM-MQ-DevToolkit
+curl --retry 5 --fail "https://s3.amazonaws.com/dd-agent-omnibus/ibm-mq-backup/${IBM_MQ_VERSION}-MacX64.pkg" -o /tmp/mq_client.pkg
+sudo installer -pkg /tmp/mq_client.pkg -target /
+rm -rf /tmp/mq_client.pkg
+
+# openssl
+DOWNLOAD_URL="https://www.openssl.org/source/openssl-{{version}}.tar.gz" \
+VERSION="3.0.12" \
+SHA256="f93c9e8edde5e9166119de31755fc87b4aa34863662f67ddfcba14d0b6b69b61" \
+RELATIVE_PATH="openssl-{{version}}" \
+CONFIGURE_SCRIPT="./config" \
+ install-from-source \
+ -fPIC shared \
+ no-module \
+ no-comp no-idea no-mdc2 no-rc5 no-ssl3 no-gost
+
+# libxml & libxslt for lxml
+DOWNLOAD_URL="https://download.gnome.org/sources/libxml2/2.10/libxml2-{{version}}.tar.xz" \
+VERSION="2.10.3" \
+SHA256="5d2cc3d78bec3dbe212a9d7fa629ada25a7da928af432c93060ff5c17ee28a9c" \
+RELATIVE_PATH="libxml2-{{version}}" \
+ install-from-source \
+ --without-iconv \
+ --without-python \
+ --without-icu \
+ --without-debug \
+ --without-mem-debug \
+ --without-run-debug \
+ --without-legacy \
+ --without-catalog \
+ --without-docbook \
+ --disable-static
+
+DOWNLOAD_URL="https://download.gnome.org/sources/libxslt/1.1/libxslt-{{version}}.tar.xz" \
+VERSION="1.1.37" \
+SHA256="3a4b27dc8027ccd6146725950336f1ec520928f320f144eb5fa7990ae6123ab4" \
+RELATIVE_PATH="libxslt-{{version}}" \
+ install-from-source \
+ --with-libxml-prefix="${DD_PREFIX_PATH}" \
+ --without-python \
+ --without-crypto \
+ --without-profiler \
+ --without-debugger \
+ --disable-static
+
+# curl
+DOWNLOAD_URL="https://curl.haxx.se/download/curl-{{version}}.tar.gz" \
+VERSION="8.4.0" \
+SHA256="816e41809c043ff285e8c0f06a75a1fa250211bbfb2dc0a037eeef39f1a9e427" \
+RELATIVE_PATH="curl-{{version}}" \
+ install-from-source \
+ --disable-manual \
+ --disable-debug \
+ --enable-optimize \
+ --disable-static \
+ --disable-ldap \
+ --disable-ldaps \
+ --disable-rtsp \
+ --enable-proxy \
+ --disable-dependency-tracking \
+ --enable-ipv6 \
+ --without-libidn \
+ --without-gnutls \
+ --without-librtmp \
+ --without-libssh2 \
+ --with-ssl="${DD_PREFIX_PATH}"
+# Remove the binary installed so that we consistenly use the same original `curl` binary
+rm "${DD_PREFIX_PATH}/bin/curl"
+
+# Dependencies needed to build librdkafka (and thus, confluent-kafka) with kerberos support
+# Note that we don't ship these but rely on the Agent providing a working cyrus-sasl installation
+# with kerberos support, therefore we only need to watch out for the version of cyrus-sasl being
+# compatible with that in the Agent, the rest shouldn't matter much
+DOWNLOAD_URL="https://github.com/LMDB/lmdb/archive/LMDB_{{version}}.tar.gz" \
+VERSION="0.9.29" \
+SHA256="22054926b426c66d8f2bc22071365df6e35f3aacf19ad943bc6167d4cae3bebb" \
+RELATIVE_PATH="lmdb-LMDB_{{version}}/libraries/liblmdb" \
+CONFIGURE_SCRIPT="true" \
+ install-from-source
+DOWNLOAD_URL="https://github.com/cyrusimap/cyrus-sasl/releases/download/cyrus-sasl-{{version}}/cyrus-sasl-{{version}}.gz" \
+VERSION="2.1.28" \
+SHA256="7ccfc6abd01ed67c1a0924b353e526f1b766b21f42d4562ee635a8ebfc5bb38c" \
+RELATIVE_PATH="cyrus-sasl-{{version}}" \
+ install-from-source --with-dblib=lmdb --enable-gssapi="${DD_PREFIX_PATH}" --disable-macos-framework
diff --git a/.github/workflows/build-deps-macos.yml b/.github/workflows/build-deps-macos.yml
index 729753ea68356e..682c52e357cf5a 100644
--- a/.github/workflows/build-deps-macos.yml
+++ b/.github/workflows/build-deps-macos.yml
@@ -34,7 +34,11 @@ jobs:
env:
DD_PYTHON3: "/Library/Frameworks/Python.framework/Versions/3.11/bin/python3"
DD_PYTHON2: "/Library/Frameworks/Python.framework/Versions/2.7/bin/python"
+ # This sets the minimum mac os version compatible for all built artifacts
+ MACOSX_DEPLOYMENT_TARGET: "10.12"
run: |
${DD_PYTHON3} -m pip install packaging
- ${DD_PYTHON3} .builders/build.py --python 3 out_py3
- ${DD_PYTHON3} .builders/build.py --python 2 out_py2
+
+ mkdir builder_root
+ ${DD_PYTHON3} .builders/build.py --builder-root builder_root --python 3 out_py3
+ ${DD_PYTHON3} .builders/build.py --builder-root builder_root --skip-setup --python 2 out_py2