Merge branch 'master' into avast-v1.0.0

.builders/build.py

@@ -118,6 +118,7 @@ def build_macos():
             'DD_PY2_BUILDENV_PATH': builder_root / 'py2' / 'bin' / 'python',
             # Path where we'll install libraries that we build
             'DD_PREFIX_PATH': prefix_path,
+            'PATH': f'{prefix_path}/bin:{os.environ["PATH"]}',
             # Common compilation flags
             'LDFLAGS': f'-L{prefix_path}/lib',
             'CFLAGS': f'-I{prefix_path}/include -O2',

.builders/deps/build_dependencies.txt

@@ -1,13 +1,9 @@
-hatchling==1.21.1; python_version > '3.0'
-hatchling==0.25.1; python_version < '3.0'
-setuptools==66.1.1; python_version > '3.0'
-setuptools==40.9.0; python_version < '3.0'
-wheel==0.38.4; python_version > '3.0'
-wheel==0.37.1; python_version < '3.0'
-setuptools-scm; python_version > '3.0'
-setuptools-scm==5.0.2; python_version < '3.0'
-setuptools-rust>=1.7.0; python_version > '3.0'
-maturin; python_version > '3.0'
+hatchling==1.21.1
+setuptools==75.6.0
+wheel==0.38.4
+setuptools-scm
+setuptools-rust>=1.7.0
+maturin
 cffi>=1.12
 cython==3.0.11
-tomli>=2.0.1; python_version > '3.0'
+tomli>=2.0.1

.builders/images/linux-aarch64/Dockerfile

@@ -36,22 +36,22 @@ RUN yum install -y perl-IPC-Cmd perl-CPANPLUS && \
  ldconfig
 
 # Compile and install Python 3
-ENV PYTHON3_VERSION=3.12.6
+ENV PYTHON3_VERSION=3.12.8
 RUN yum install -y libffi-devel && \
  DOWNLOAD_URL="https://python.org/ftp/python/{{version}}/Python-{{version}}.tgz" \
  VERSION="${PYTHON3_VERSION}" \
- SHA256="85a4c1be906d20e5c5a69f2466b00da769c221d6a684acfd3a514dbf5bf10a66" \
+ SHA256="5978435c479a376648cb02854df3b892ace9ed7d32b1fead652712bee9d03a45" \
  RELATIVE_PATH="Python-{{version}}" \
  bash install-from-source.sh \
- --prefix=/opt/python/${PYTHON_VERSION} \
+ --prefix=/opt/python/${PYTHON3_VERSION} \
  --with-ensurepip=yes \
  --enable-ipv6 \
  --with-dbmliborder=
-ENV PATH="/opt/python/${PYTHON_VERSION}/bin:${PATH}"
+ENV PATH="/opt/python/${PYTHON3_VERSION}/bin:${PATH}"
 # Set up virtual environment for Python 3
-RUN /opt/python/${PYTHON_VERSION}/bin/python3 -m pip install --no-warn-script-location --upgrade pip \
- && /opt/python/${PYTHON_VERSION}/bin/python3 -m pip install --no-warn-script-location virtualenv \
- && /opt/python/${PYTHON_VERSION}/bin/python3 -m virtualenv /py3
+RUN /opt/python/${PYTHON3_VERSION}/bin/python3 -m pip install --no-warn-script-location --upgrade pip \
+ && /opt/python/${PYTHON3_VERSION}/bin/python3 -m pip install --no-warn-script-location virtualenv \
+ && /opt/python/${PYTHON3_VERSION}/bin/python3 -m virtualenv /py3
 
 # krb5 for dependencies that require kerberos support
 RUN \
@@ -141,8 +141,8 @@ RUN \
 # curl
 RUN \
  DOWNLOAD_URL="https://curl.haxx.se/download/curl-{{version}}.tar.gz" \
- VERSION="8.9.1" \
- SHA256="291124a007ee5111997825940b3876b3048f7d31e73e9caa681b80fe48b2dcd5" \
+ VERSION="8.11.1" \
+ SHA256="a889ac9dbba3644271bd9d1302b5c22a088893719b72be3487bc3d401e5c4e80" \
  RELATIVE_PATH="curl-{{version}}" \
   bash install-from-source.sh \
     --disable-manual \
@@ -159,6 +159,7 @@ RUN \
     --without-gnutls \
     --without-librtmp \
     --without-libssh2 \
+    --without-libpsl \
     --with-ssl=/usr/local \
  && rm /usr/local/bin/curl
 

.builders/images/linux-aarch64/build_script.sh

@@ -21,7 +21,7 @@ if [[ "${DD_BUILD_PYTHON_VERSION}" == "3" ]]; then
     LDFLAGS="${LDFLAGS} -L/usr/local/lib -lkrb5 -lgssapi_krb5 -llmdb" \
     DOWNLOAD_URL="https://github.com/confluentinc/librdkafka/archive/refs/tags/v{{version}}.tar.gz" \
         VERSION="${kafka_version}" \
-        SHA256="3dc62de731fd516dfb1032861d9a580d4d0b5b0856beb0f185d06df8e6c26259" \
+        SHA256="5bd1c46f63265f31c6bfcedcde78703f77d28238eadf23821c2b43fc30be3e25" \
         RELATIVE_PATH="librdkafka-{{version}}" \
         bash install-from-source.sh --enable-sasl --enable-curl
     always_build+=("confluent-kafka")

.builders/images/linux-x86_64/Dockerfile

@@ -35,18 +35,18 @@ RUN yum install -y perl-IPC-Cmd perl-CPANPLUS && \
  ldconfig
 
 # Compile and install Python 3
-ENV PYTHON3_VERSION=3.12.6
+ENV PYTHON3_VERSION=3.12.8
 RUN yum install -y libffi-devel && \
  DOWNLOAD_URL="https://python.org/ftp/python/{{version}}/Python-{{version}}.tgz" \
  VERSION="${PYTHON3_VERSION}" \
- SHA256="85a4c1be906d20e5c5a69f2466b00da769c221d6a684acfd3a514dbf5bf10a66" \
+ SHA256="5978435c479a376648cb02854df3b892ace9ed7d32b1fead652712bee9d03a45" \
  RELATIVE_PATH="Python-{{version}}" \
- bash install-from-source.sh --prefix=/opt/python/${PYTHON_VERSION} --with-ensurepip=yes --enable-ipv6 --with-dbmliborder=
-ENV PATH="/opt/python/${PYTHON_VERSION}/bin:${PATH}"
+ bash install-from-source.sh --prefix=/opt/python/${PYTHON3_VERSION} --with-ensurepip=yes --enable-ipv6 --with-dbmliborder=
+ENV PATH="/opt/python/${PYTHON3_VERSION}/bin:${PATH}"
 # Set up virtual environment for Python 3
-RUN /opt/python/${PYTHON_VERSION}/bin/python3 -m pip install --no-warn-script-location --upgrade pip \
- && /opt/python/${PYTHON_VERSION}/bin/python3 -m pip install --no-warn-script-location virtualenv \
- && /opt/python/${PYTHON_VERSION}/bin/python3 -m virtualenv /py3
+RUN /opt/python/${PYTHON3_VERSION}/bin/python3 -m pip install --no-warn-script-location --upgrade pip \
+ && /opt/python/${PYTHON3_VERSION}/bin/python3 -m pip install --no-warn-script-location virtualenv \
+ && /opt/python/${PYTHON3_VERSION}/bin/python3 -m virtualenv /py3
 
 # MQ Client library required by pymqi
 ENV IBM_MQ_VERSION="9.2.4.0"
@@ -145,8 +145,8 @@ RUN \
 # curl
 RUN \
  DOWNLOAD_URL="https://curl.haxx.se/download/curl-{{version}}.tar.gz" \
- VERSION="8.9.1" \
- SHA256="291124a007ee5111997825940b3876b3048f7d31e73e9caa681b80fe48b2dcd5" \
+ VERSION="8.11.1" \
+ SHA256="a889ac9dbba3644271bd9d1302b5c22a088893719b72be3487bc3d401e5c4e80" \
  RELATIVE_PATH="curl-{{version}}" \
   bash install-from-source.sh \
     --disable-manual \
@@ -163,6 +163,7 @@ RUN \
     --without-gnutls \
     --without-librtmp \
     --without-libssh2 \
+    --without-libpsl \
     --with-ssl=/usr/local \
  && rm /usr/local/bin/curl
 

.builders/images/linux-x86_64/build_script.sh

@@ -18,7 +18,7 @@ if [[ "${DD_BUILD_PYTHON_VERSION}" == "3" ]]; then
     LDFLAGS="${LDFLAGS} -L/usr/local/lib -lkrb5 -lgssapi_krb5 -llmdb" \
     DOWNLOAD_URL="https://github.com/confluentinc/librdkafka/archive/refs/tags/v{{version}}.tar.gz" \
         VERSION="${kafka_version}" \
-        SHA256="3dc62de731fd516dfb1032861d9a580d4d0b5b0856beb0f185d06df8e6c26259" \
+        SHA256="5bd1c46f63265f31c6bfcedcde78703f77d28238eadf23821c2b43fc30be3e25" \
         RELATIVE_PATH="librdkafka-{{version}}" \
         bash install-from-source.sh --enable-sasl --enable-curl
     always_build+=("confluent-kafka")

.builders/images/macos-x86_64/builder_setup.sh

@@ -71,8 +71,8 @@ RELATIVE_PATH="libxslt-{{version}}" \
 
 # curl
 DOWNLOAD_URL="https://curl.haxx.se/download/curl-{{version}}.tar.gz" \
-VERSION="8.9.1" \
-SHA256="291124a007ee5111997825940b3876b3048f7d31e73e9caa681b80fe48b2dcd5" \
+VERSION="8.11.1" \
+SHA256="a889ac9dbba3644271bd9d1302b5c22a088893719b72be3487bc3d401e5c4e80" \
 RELATIVE_PATH="curl-{{version}}" \
   install-from-source \
     --disable-manual \
@@ -89,6 +89,7 @@ RELATIVE_PATH="curl-{{version}}" \
     --without-gnutls \
     --without-librtmp \
     --without-libssh2 \
+    --without-libpsl \
     --with-ssl="${DD_PREFIX_PATH}"
 # Remove the binary installed so that we consistenly use the same original `curl` binary
 rm "${DD_PREFIX_PATH}/bin/curl"

.builders/images/macos-x86_64/extra_build.sh

@@ -13,7 +13,7 @@ if [[ "${DD_BUILD_PYTHON_VERSION}" == "3" ]]; then
     LDFLAGS="${LDFLAGS} -L${DD_PREFIX_PATH}/lib -lgssapi_krb5 -llmdb" \
     DOWNLOAD_URL="https://github.com/confluentinc/librdkafka/archive/refs/tags/v{{version}}.tar.gz" \
       VERSION="${kafka_version}" \
-      SHA256="3dc62de731fd516dfb1032861d9a580d4d0b5b0856beb0f185d06df8e6c26259" \
+      SHA256="5bd1c46f63265f31c6bfcedcde78703f77d28238eadf23821c2b43fc30be3e25" \
       RELATIVE_PATH="librdkafka-{{version}}" \
       bash install-from-source.sh --prefix="${DD_PREFIX_PATH}" --enable-sasl --enable-curl
 

.builders/images/windows-x86_64/Dockerfile

@@ -35,6 +35,15 @@ RUN curl -SL --output PowerShell-%POWERSHELL_VERSION%-win-x64.msi https://github
 COPY helpers.ps1 C:\helpers.ps1
 SHELL ["pwsh", "-Command", ". C:\\helpers.ps1;"]
 
+# Enable long paths
+# https://learn.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation?tabs=powershell#registry-setting-to-enable-long-paths
+RUN New-ItemProperty -Path "HKLM:\SYSTEM\CurrentControlSet\Control\FileSystem" `
+ -Name "LongPathsEnabled" -Value 1 -PropertyType DWORD -Force
+# Reduce the chance of hitting path limits (the MSVC compiler cl.exe doesn't seem to respect that optioin)
+# This variable is honored by pip
+ENV TMP="C:\tmp" `
+    TEMP="C:\tmp"
+
 # Install 7-Zip ZS
 ENV 7ZIP_VERSION="22.01" `
     7ZIP_ZS_VERSION="1.5.5-R3"
@@ -73,11 +82,11 @@ RUN Get-RemoteFile `
     Approve-File -Path $($Env:USERPROFILE + '\.cargo\bin\rustc.exe') -Hash $Env:RUSTC_HASH
 
 # Install Python 3
-ENV PYTHON_VERSION="3.12.6"
+ENV PYTHON_VERSION="3.12.8"
 RUN Get-RemoteFile `
       -Uri https://www.python.org/ftp/python/$Env:PYTHON_VERSION/python-$Env:PYTHON_VERSION-amd64.exe `
       -Path python-$Env:PYTHON_VERSION-amd64.exe `
-      -Hash '5914748e6580e70bedeb7c537a0832b3071de9e09a2e4e7e3d28060616045e0a'; `
+      -Hash '71bd44e6b0e91c17558963557e4cdb80b483de9b0a0a9717f06cf896f95ab598'; `
     Start-Process -Wait python-$Env:PYTHON_VERSION-amd64.exe -ArgumentList '/quiet', 'InstallAllUsers=1'; `
     Remove-Item python-$Env:PYTHON_VERSION-amd64.exe; `
     & 'C:\Program Files\Python312\python.exe' -m pip install --no-warn-script-location --upgrade pip; `
@@ -95,10 +104,38 @@ RUN Get-RemoteFile `
     Remove-Item $Env:IBM_MQ_VERSION-IBM-MQC-Redist-Win64.zip; `
     setx /M MQ_FILE_PATH 'C:\ibm_mq'
 
+# Perl
+ENV PERL_VERSION="5.40.0.1"
+RUN Get-RemoteFile `
+      -Uri https://github.com/StrawberryPerl/Perl-Dist-Strawberry/releases/download/SP_54001_64bit_UCRT/strawberry-perl-$Env:PERL_VERSION-64bit-portable.zip `
+      -Path "strawberry-perl-$Env:PERL_VERSION-64bit.zip" `
+    -Hash '754f3e2a8e473dc68d1540c7802fb166a025f35ef18960c4564a31f8b5933907' && `
+    7z x "strawberry-perl-$Env:PERL_VERSION-64bit.zip" -o"C:\perl" && `
+    Add-ToPath -Append "C:\perl\perl\bin" && `
+    Remove-Item "strawberry-perl-$Env:PERL_VERSION-64bit.zip"
+
+ENV OPENSSL_VERSION="3.3.2"
+
+ENV CURL_VERSION="8.11.1"
+
 # Set up runner
 COPY runner_dependencies.txt C:\runner_dependencies.txt
 RUN python -m pip install --no-warn-script-location -r C:\runner_dependencies.txt
 
+COPY build_script.ps1 C:\build_script.ps1
+COPY update_librdkafka_manifest.py C:\update_librdkafka_manifest.py
+ENV DD_BUILD_COMMAND="pwsh C:\build_script.ps1"
+
+# Python packages that we want to build regardless of whether prebuilt versions exist on PyPI
+ENV PIP_NO_BINARY="confluent_kafka"
+# Where to find native dependencies when building extensions and for wheel repairing
+RUN New-Item -Path "C:\include" -ItemType Directory
+RUN New-Item -Path "C:\lib" -ItemType Directory
+RUN New-Item -Path "C:\bin" -ItemType Directory
+ENV INCLUDE="C:\include"
+ENV LIB="C:\lib"
+RUN Add-ToPath -Append "C:\bin"
+
 # Restore the default Windows shell for correct batch processing.
 SHELL ["cmd", "/S", "/C"]
 

.builders/images/windows-x86_64/build_script.ps1

@@ -0,0 +1,70 @@
+$ErrorActionPreference = 'Stop'
+$PSNativeCommandUseErrorActionPreference = $true
+
+. C:\helpers.ps1
+
+# The librdkafka version needs to stay in sync with the confluent-kafka version,
+# thus we extract the version from the requirements file
+$kafka_version = Get-Content 'C:\mnt\requirements.in' | perl -nE 'say/^\D*(\d+\.\d+\.\d+)\D*$/ if /confluent-kafka==/'
+Write-Host "Will build librdkafka $kafka_version"
+
+# Download and unpack the source
+Get-RemoteFile `
+  -Uri "https://github.com/confluentinc/librdkafka/archive/refs/tags/v${kafka_version}.tar.gz" `
+  -Path "librdkafka-${kafka_version}.tar.gz" `
+  -Hash '5bd1c46f63265f31c6bfcedcde78703f77d28238eadf23821c2b43fc30be3e25'
+7z x "librdkafka-${kafka_version}.tar.gz" -o"C:\"
+7z x "C:\librdkafka-${kafka_version}.tar" -o"C:\librdkafka"
+Remove-Item "librdkafka-${kafka_version}.tar.gz"
+
+# Build librdkafka
+# Based on this job from upstream:
+# https://github.com/confluentinc/librdkafka/blob/cb8c19c43011b66c4b08b25e5150455a247e1ff3/.semaphore/semaphore.yml#L265
+# Install vcpkg
+$triplet = "x64-windows"
+$vcpkg_dir = "C:\vcpkg"
+$librdkafka_dir = "C:\librdkafka\librdkafka-${kafka_version}"
+# We set the desired tag to the latest release tag to ensure that we are building with the latest stable version.
+# The desired tag should be updated periodically or when critical fixes or features are released.
+$desired_tag = "2024.12.16"
+
+# Clone and configure vcpkg
+if (-Not (Test-Path -Path "$vcpkg_dir\.git")) {
+    git clone https://github.com/Microsoft/vcpkg.git $vcpkg_dir
+}
+
+Set-Location $vcpkg_dir
+git checkout $desired_tag
+
+Write-Host "Bootstrapping vcpkg..."
+.\bootstrap-vcpkg.bat
+
+# Get deps
+Set-Location "$librdkafka_dir"
+# Patch the the vcpkg manifest to to override the OpenSSL version and CURL version
+python C:\update_librdkafka_manifest.py vcpkg.json --set-version openssl:${Env:OPENSSL_VERSION} --set-version curl:${Env:CURL_VERSION}
+
+C:\vcpkg\vcpkg integrate install
+C:\vcpkg\vcpkg --feature-flags=versions install --triplet $triplet
+# Build
+& .\win32\msbuild.ps1 -platform x64
+
+# Copy outputs to where they can be found
+# This is partially inspired by
+# https://github.com/confluentinc/librdkafka/blob/cb8c19c43011b66c4b08b25e5150455a247e1ff3/win32/package-zip.ps1
+$toolset = "v142"
+$platform = "x64"
+$config = "Release"
+$srcdir = "win32\outdir\${toolset}\${platform}\$config"
+$bindir = "C:\bin"
+$libdir = "C:\lib"
+$includedir = "C:\include"
+
+Copy-Item "${srcdir}\librdkafka.dll","${srcdir}\librdkafkacpp.dll",
+"${srcdir}\libcrypto-3-x64.dll","${srcdir}\libssl-3-x64.dll",
+"${srcdir}\zlib1.dll","${srcdir}\zstd.dll","${srcdir}\libcurl.dll" -Destination $bindir
+Copy-Item "${srcdir}\librdkafka.lib","${srcdir}\librdkafkacpp.lib" -Destination $libdir
+
+New-Item -Path $includedir\librdkafka -ItemType Directory
+Copy-Item -Path ".\src\*" -Filter *.h -Destination $includedir\librdkafka
+

.builders/images/windows-x86_64/update_librdkafka_manifest.py

@@ -0,0 +1,25 @@
+import json
+from argparse import ArgumentParser
+
+
+def main(manifest_file, versions):
+    with open(manifest_file) as f:
+        manifest = json.load(f)
+
+    for dep, version in versions.items():
+        manifest.setdefault("overrides", []).append({
+            "name": dep,
+            "version": version,
+        })
+
+    with open(manifest_file, 'w') as f:
+        json.dump(manifest, f)
+
+
+if __name__ == '__main__':
+    ap = ArgumentParser()
+    ap.add_argument("file")
+    ap.add_argument("--set-version", action="append", required=True)
+
+    args = ap.parse_args()
+    main(args.file, dict(spec.split(':') for spec in args.set_version))