Add REST remote function

.github/workflows/experimental.yml

@@ -47,7 +47,7 @@ permissions:
 
 jobs:
   compile:
-    runs-on: 8-core-ubuntu
+    runs-on: 16-core-ubuntu
     timeout-minutes: 120
     env:
       CCACHE_DIR: "${{ github.workspace }}/.ccache/"
@@ -103,8 +103,20 @@ jobs:
           name: join
           path: velox/_build/debug/velox/exec/tests/velox_join_fuzzer_test
 
+      - name: Upload Presto expression fuzzer
+        uses: actions/upload-artifact@v3
+        with:
+          name: presto_expression_fuzzer
+          path: velox/_build/debug/velox/expression/fuzzer/velox_expression_fuzzer_test
+
+      - name: Upload Spark expression fuzzer
+        uses: actions/upload-artifact@v3
+        with:
+          name: spark_expression_fuzzer
+          path: velox/_build/debug/velox/expression/fuzzer/spark_expression_fuzzer_test
+
   presto-java-aggregation-fuzzer-run:
-    runs-on: 8-core-ubuntu
+    runs-on: 16-core-ubuntu
     container: ghcr.io/facebookincubator/velox-dev:presto-java
     timeout-minutes: 120
     env:
@@ -249,3 +261,96 @@ jobs:
           name: join-fuzzer-failure-artifacts
           path: |
             /tmp/join_fuzzer_repro
+
+  presto-expression-fuzzer-run:
+    runs-on: ubuntu-latest
+    needs: compile
+    timeout-minutes: 120
+    steps:
+
+      - name: "Checkout Repo"
+        uses: actions/checkout@v3
+        with:
+          ref: "${{ inputs.ref || 'main' }}"
+
+      - name: "Install dependencies"
+        run: source ./scripts/setup-ubuntu.sh && install_apt_deps
+
+      - name: Download presto fuzzer
+        uses: actions/download-artifact@v3
+        with:
+          name: presto_expression_fuzzer
+
+      - name: "Run Presto Fuzzer"
+        run: |
+          mkdir -p /tmp/presto_fuzzer_repro/
+          rm -rfv /tmp/presto_fuzzer_repro/*
+          chmod -R 777 /tmp/presto_fuzzer_repro
+          chmod +x velox_expression_fuzzer_test
+          ./velox_expression_fuzzer_test \
+                --seed ${RANDOM} \
+                --enable_variadic_signatures \
+                --velox_fuzzer_enable_complex_types \
+                --velox_fuzzer_enable_decimal_type \
+                --lazy_vector_generation_ratio 0.2 \
+                --velox_fuzzer_enable_column_reuse \
+                --velox_fuzzer_enable_expression_reuse \
+                --max_expression_trees_per_step 2 \
+                --retry_with_try \
+                --enable_dereference \
+                --duration_sec 1800 \
+                --logtostderr=1 \
+                --minloglevel=1 \
+                --repro_persist_path=/tmp/presto_fuzzer_repro \
+          && echo -e "\n\nFuzzer run finished successfully."
+
+      - name: Archive Presto expression production artifacts
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: presto-fuzzer-failure-artifacts
+          path: |
+            /tmp/presto_fuzzer_repro
+
+  spark-expression-fuzzer-run:
+    runs-on: ubuntu-latest
+    needs: compile
+    timeout-minutes: 120
+    steps:
+
+      - name: "Checkout Repo"
+        uses: actions/checkout@v3
+        with:
+          ref: "${{ inputs.ref || 'main' }}"
+
+      - name: "Install dependencies"
+        run: source ./scripts/setup-ubuntu.sh && install_apt_deps
+
+      - name: Download spark fuzzer
+        uses: actions/download-artifact@v3
+        with:
+          name: spark_expression_fuzzer
+
+      - name: "Run Spark Fuzzer"
+        run: |
+          mkdir -p /tmp/spark_fuzzer_repro/
+          rm -rfv /tmp/spark_fuzzer_repro/*
+          chmod -R 777 /tmp/spark_fuzzer_repro
+          chmod +x spark_expression_fuzzer_test
+          ./spark_expression_fuzzer_test \
+                --seed ${RANDOM} \
+                --duration_sec 1800 \
+                --logtostderr=1 \
+                --minloglevel=1 \
+                --repro_persist_path=/tmp/spark_fuzzer_repro \
+                --velox_fuzzer_enable_decimal_type \
+                --retry_with_try \
+            && echo -e "\n\nSpark Fuzzer run finished successfully."
+
+      - name: Archive Spark expression production artifacts
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: spark-fuzzer-failure-artifacts
+          path: |
+            /tmp/spark_fuzzer_repro

.github/workflows/linux-build.yml

@@ -88,6 +88,22 @@ jobs:
             mv ./${MINIO_BINARY} /usr/local/bin/
           fi
 
+      - name: Install Proxygen
+        run: |
+          FB_OS_VERSION="v2024.05.20.00"
+          PROXYGEN_BINARY="proxygen-${FB_OS_VERSION}.tar.gz"
+          if [ ! -f /usr/local/bin/${PROXYGEN_BINARY} ]; then
+            wget https://github.com/facebook/proxygen/archive/refs/tags/${FB_OS_VERSION}.tar.gz -O ${PROXYGEN_BINARY}
+            tar -xzf ${PROXYGEN_BINARY}
+            cd proxygen-${FB_OS_VERSION}
+            ./build.sh   # Assumes there's a build script or replace with the appropriate build command
+            chmod +x ./proxygen-${FB_OS_VERSION}
+            mv ./proxygen-${FB_OS_VERSION} /usr/local/bin/
+            cd ..
+            rm -rf proxygen-${FB_OS_VERSION} ${PROXYGEN_BINARY}
+          fi
+
+
       - uses: assignUser/stash/restore@v1
         with:
           path: '${{ env.CCACHE_DIR }}'

.github/workflows/macos.yml

@@ -22,6 +22,7 @@ on:
       - "CMakeLists.txt"
       - "CMake/**"
       - "third_party/**"
+      - "scripts/setup-macos.sh"
       - ".github/workflows/macos.yml"
 
   pull_request:
@@ -31,6 +32,7 @@ on:
       - "CMakeLists.txt"
       - "CMake/**"
       - "third_party/**"
+      - "scripts/setup-macos.sh"
       - ".github/workflows/macos.yml"
 
 permissions:

CMake/resolve_dependency_modules/README.md

@@ -32,12 +32,11 @@ by Velox. See details on bundling below.
 | re2               | 2021-04-01      | Yes      |
 | fmt               | 10.1.1          | Yes      |
 | simdjson          | 3.9.3           | Yes      |
-| fast_float        | v6.1.6          | Yes      |
-| folly             | v2024.09.16.00  | Yes      |
-| fizz              | v2024.09.16.00  | No       |
-| wangle            | v2024.09.16.00  | No       |
-| mvfst             | v2024.09.16.00  | No       |
-| fbthrift          | v2024.09.16.00  | No       |
+| folly             | v2024.05.20.00  | Yes      |
+| fizz              | v2024.05.20.00  | No       |
+| wangle            | v2024.05.20.00  | No       |
+| mvfst             | v2024.05.20.00  | No       |
+| fbthrift          | v2024.05.20.00  | No       |
 | libstemmer        | 2.2.0           | Yes      |
 | DuckDB (testing)  | 0.8.1           | Yes      |
 | cpr (testing)     | 1.10.15         | Yes      |

CMake/resolve_dependency_modules/folly/CMakeLists.txt

@@ -14,15 +14,13 @@
 project(Folly)
 cmake_minimum_required(VERSION 3.14)
 
-set(VELOX_FOLLY_BUILD_VERSION v2024.09.16.00)
+set(VELOX_FOLLY_BUILD_VERSION v2024.05.20.00)
 set(VELOX_FOLLY_BUILD_SHA256_CHECKSUM
-    0a375f2f3e15a2679b4d21fa1064986830a52f59c74d82b3bda1aeeea4e77da0)
+    f4a450e59f0b74d1b0b4e5c55ae38e820166c95d02f8a8a298e54a49a90aa057)
 set(VELOX_FOLLY_SOURCE_URL
     "https://github.com/facebook/folly/releases/download/${VELOX_FOLLY_BUILD_VERSION}/folly-${VELOX_FOLLY_BUILD_VERSION}.tar.gz"
 )
 
-set(fast_float_SOURCE BUNDLED)
-resolve_dependency(fast_float)
 resolve_dependency_url(FOLLY)
 
 message(STATUS "Building Folly from source")

CMake/resolve_dependency_modules/folly/folly-gflags-glog.patch

@@ -13,10 +13,10 @@
 # limitations under the License.
 --- a/CMake/folly-deps.cmake
 +++ b/CMake/folly-deps.cmake
-@@ -55,19 +55,23 @@ list(APPEND FOLLY_INCLUDE_DIRECTORIES ${DOUBLE_CONVERSION_INCLUDE_DIR})
- find_package(FastFloat MODULE REQUIRED)
- list(APPEND FOLLY_INCLUDE_DIRECTORIES ${FASTFLOAT_INCLUDE_DIR})
-
+@@ -52,19 +52,20 @@ find_package(DoubleConversion MODULE REQUIRED)
+ list(APPEND FOLLY_LINK_LIBRARIES ${DOUBLE_CONVERSION_LIBRARY})
+ list(APPEND FOLLY_INCLUDE_DIRECTORIES ${DOUBLE_CONVERSION_INCLUDE_DIR})
+ 
 -find_package(Gflags MODULE)
 -set(FOLLY_HAVE_LIBGFLAGS ${LIBGFLAGS_FOUND})
 -if(LIBGFLAGS_FOUND)
@@ -27,14 +27,12 @@
 +find_package(gflags)
 +set(FOLLY_HAVE_LIBGFLAGS ${gflags_FOUND})
 +if(gflags_FOUND)
-+ list(APPEND FOLLY_LINK_LIBRARIES ${gflags_LIBRARY})
-+ list(APPEND FOLLY_INCLUDE_DIRECTORIES ${gflags_INCLUDE_DIR})
-+ set(FOLLY_LIBGFLAGS_LIBRARY ${gflags_LIBRARY})
-+ set(FOLLY_LIBGFLAGS_INCLUDE ${gflags_INCLUDE_DIR})
-+ message(STATUS "gflags_INCLUDE_DIR: ${gflags_INCLUDE_DIR}")
-+ message(STATUS "gflags_LIBRARY: ${gflags_LIBRARY}")
++  list(APPEND FOLLY_LINK_LIBRARIES ${gflags_LIBRARY})
++  list(APPEND FOLLY_INCLUDE_DIRECTORIES ${gflags_INCLUDE_DIR})
++  set(FOLLY_LIBGFLAGS_LIBRARY ${gflags_LIBRARY})
++  set(FOLLY_LIBGFLAGS_INCLUDE ${gflags_INCLUDE_DIR})
  endif()
-
+ 
 -find_package(Glog MODULE)
 -set(FOLLY_HAVE_LIBGLOG ${GLOG_FOUND})
 -list(APPEND FOLLY_LINK_LIBRARIES ${GLOG_LIBRARY})
@@ -43,8 +41,7 @@
 +set(FOLLY_HAVE_LIBGLOG ${glog_FOUND})
 +list(APPEND FOLLY_LINK_LIBRARIES ${glog_LIBRARY})
 +list(APPEND FOLLY_INCLUDE_DIRECTORIES ${glog_INCLUDE_DIR})
-+message(STATUS "glog_INCLUDE_DIR: ${glog_INCLUDE_DIR}")
-+message(STATUS "glog_LIBRARY: ${glog_LIBRARY}")
-
++message(STATUS "glog_INCLUDE_DIR: ${gflags_LINRARY}")
+
  find_package(LibEvent MODULE REQUIRED)
  list(APPEND FOLLY_LINK_LIBRARIES ${LIBEVENT_LIB})

CMakeLists.txt

@@ -487,6 +487,12 @@ if(${VELOX_BUILD_TESTING})
   resolve_dependency(gRPC)
 endif()
 
+if(VELOX_ENABLE_REMOTE_FUNCTIONS)
+  find_package(fizz CONFIG REQUIRED)
+  find_package(wangle CONFIG REQUIRED)
+  find_package(proxygen CONFIG REQUIRED)
+endif()
+
 if(VELOX_ENABLE_REMOTE_FUNCTIONS)
   # TODO: Move this to use resolve_dependency(). For some reason, FBThrift
   # requires clients to explicitly install fizz and wangle.

scripts/setup-centos9.sh

@@ -39,11 +39,11 @@ USE_CLANG="${USE_CLANG:-false}"
 export INSTALL_PREFIX=${INSTALL_PREFIX:-"/usr/local"}
 DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)/deps-download}
 
-FB_OS_VERSION="v2024.09.16.00"
+FB_OS_VERSION="v2024.05.20.00"
 FMT_VERSION="10.1.1"
 BOOST_VERSION="boost-1.84.0"
 ARROW_VERSION="15.0.0"
-FAST_FLOAT_VERSION="v6.1.6"
+STEMMER_VERSION="2.2.0"
 
 function dnf_install {
   dnf install -y -q --setopt=install_weak_deps=False "$@"
@@ -178,6 +178,17 @@ function install_duckdb {
   fi
 }
 
+function install_stemmer {
+  wget_and_untar https://snowballstem.org/dist/libstemmer_c-${STEMMER_VERSION}.tar.gz stemmer
+  (
+    cd ${DEPENDENCY_DIR}/stemmer
+    sed -i '/CPPFLAGS=-Iinclude/ s/$/ -fPIC/' Makefile
+    make clean && make "-j${NPROC}"
+    ${SUDO} cp libstemmer.a ${INSTALL_PREFIX}/lib/
+    ${SUDO} cp include/libstemmer.h ${INSTALL_PREFIX}/include/
+  )
+}
+
 function install_arrow {
   wget_and_untar https://archive.apache.org/dist/arrow/arrow-${ARROW_VERSION}/apache-arrow-${ARROW_VERSION}.tar.gz arrow
   cmake_install_dir arrow/cpp \
@@ -210,12 +221,6 @@ function install_cuda {
   dnf install -y cuda-nvcc-$(echo $1 | tr '.' '-') cuda-cudart-devel-$(echo $1 | tr '.' '-')
 }
 
-function install_fast_float {
-  # Dependency of folly.
-  wget_and_untar https://github.com/fastfloat/fast_float/archive/refs/tags/${FAST_FLOAT_VERSION}.tar.gz fast_float
-  cmake_install_dir fast_float
-}
-
 function install_velox_deps {
   run_and_time install_velox_deps_from_dnf
   run_and_time install_conda
@@ -226,13 +231,13 @@ function install_velox_deps {
   run_and_time install_boost
   run_and_time install_protobuf
   run_and_time install_fmt
-  run_and_time install_fast_float
   run_and_time install_folly
   run_and_time install_fizz
   run_and_time install_wangle
   run_and_time install_mvfst
   run_and_time install_fbthrift
   run_and_time install_duckdb
+  run_and_time install_stemmer
   run_and_time install_arrow
 }