ARROW-14506: [C++] Conda support for google-cloud-cpp

This PR adds support for `google-cloud-cpp` to the Conda files. Probably the most difficult change to grok is the change to compile with C++17 when using Conda: - Conda defaults all its builds to C++17, [this bug](conda/conda-build#3375) goes into some detail as to why. - Arrow defaults to C++11 if no `CMAKE_CXX_STANDARD` argument is provided. - Abseil's ABI changes when used from C++11 vs. C++17, see abseil/abseil-cpp#696 - Therefore, one must compile with C++17 to use Abseil in Conda. - And because `google-cloud-cpp` has a direct dependency on Abseil, exposed through the headers, one must use C++17 to use `google-cloud-cpp` too. Closes #11916 from coryan/ARROW-14506-add-google-cloud-cpp-to-conda-files Lead-authored-by: Uwe L. Korn <uwe.korn@quantco.com> Co-authored-by: Carlos O'Ryan <coryan@google.com> Signed-off-by: Antoine Pitrou <antoine@python.org>
apache · Feb 8, 2022 · d78967e · d78967e
1 parent fa4d517
commit d78967e
Show file tree

Hide file tree

Showing 49 changed files with 923 additions and 272 deletions.
diff --git a/ci/conda_env_cpp.txt b/ci/conda_env_cpp.txt
@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-aws-sdk-cpp
+aws-sdk-cpp=1.8.186
 benchmark>=1.6.0
 boost-cpp>=1.68.0
 brotli
@@ -25,13 +25,17 @@ cmake
 gflags
 glog
 gmock>=1.10.0
+google-cloud-cpp>=1.34.0
 grpc-cpp>=1.27.3
 gtest>=1.10.0
 libprotobuf
 libutf8proc
 lz4-c
 make
 ninja
+# Required by google-cloud-cpp, the Conda package is missing the dependency:
+#    https://github.com/conda-forge/google-cloud-cpp-feedstock/issues/28
+nlohmann_json
 pkg-config
 python
 rapidjson

diff --git a/ci/docker/conda-cpp.dockerfile b/ci/docker/conda-cpp.dockerfile
@@ -22,9 +22,6 @@ FROM ${repo}:${arch}-conda
 COPY ci/scripts/install_minio.sh /arrow/ci/scripts
 RUN /arrow/ci/scripts/install_minio.sh latest /opt/conda
 
-COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts
-RUN /arrow/ci/scripts/install_gcs_testbench.sh default
-
 # install the required conda packages into the test environment
 COPY ci/conda_env_cpp.txt \
      ci/conda_env_gandiva.txt \
@@ -37,12 +34,17 @@ RUN mamba install \
         valgrind && \
     mamba clean --all
 
+# We want to install the GCS testbench using the same Python binary that the Conda code will use.
+COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts
+RUN /arrow/ci/scripts/install_gcs_testbench.sh default
+
 ENV ARROW_BUILD_TESTS=ON \
     ARROW_DATASET=ON \
     ARROW_DEPENDENCY_SOURCE=CONDA \
     ARROW_FLIGHT=ON \
     ARROW_FLIGHT_SQL=ON \
     ARROW_GANDIVA=ON \
+    ARROW_GCS=ON \
     ARROW_HOME=$CONDA_PREFIX \
     ARROW_ORC=ON \
     ARROW_PARQUET=ON \
@@ -57,6 +59,7 @@ ENV ARROW_BUILD_TESTS=ON \
     ARROW_WITH_SNAPPY=ON \
     ARROW_WITH_ZLIB=ON \
     ARROW_WITH_ZSTD=ON \
+    CMAKE_CXX_STANDARD=17 \
     GTest_SOURCE=BUNDLED \
     PARQUET_BUILD_EXAMPLES=ON \
     PARQUET_BUILD_EXECUTABLES=ON \

diff --git a/ci/scripts/cpp_build.sh b/ci/scripts/cpp_build.sh
@@ -137,6 +137,7 @@ cmake \
   -DCMAKE_VERBOSE_MAKEFILE=${CMAKE_VERBOSE_MAKEFILE:-OFF} \
   -DCMAKE_C_FLAGS="${CFLAGS:-}" \
   -DCMAKE_CXX_FLAGS="${CXXFLAGS:-}" \
+  -DCMAKE_CXX_STANDARD="${CMAKE_CXX_STANDARD:-11}" \
   -DCMAKE_INSTALL_LIBDIR=${CMAKE_INSTALL_LIBDIR:-lib} \
   -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX:-${ARROW_HOME}} \
   -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD:-OFF} \

diff --git a/cpp/src/arrow/filesystem/gcsfs.cc b/cpp/src/arrow/filesystem/gcsfs.cc
@@ -138,7 +138,7 @@ class GcsInputStream : public arrow::io::InputStream {
     stream_.read(reinterpret_cast<char*>(buffer->mutable_data()), nbytes);
     ARROW_GCS_RETURN_NOT_OK(stream_.status());
     RETURN_NOT_OK(buffer->Resize(stream_.gcount(), true));
-    return buffer;
+    return std::shared_ptr<Buffer>(std::move(buffer));
   }
   //@}
 

diff --git a/...on10.2numpy1.17python3.7.____cpython.yaml → ...rsion7numpy1.18python3.7.____cpython.yaml b/...on10.2numpy1.17python3.7.____cpython.yaml → ...rsion7numpy1.18python3.7.____cpython.yaml
@@ -1,15 +1,13 @@
-aws_sdk_cpp:
-- 1.8.186
 bzip2:
 - '1'
 c_compiler:
 - gcc
 c_compiler_version:
-- '9'
+- '7'
 cdt_name:
 - cos6
 channel_sources:
-- conda-forge,defaults
+- conda-forge
 channel_targets:
 - conda-forge main
 cuda_compiler:
@@ -19,23 +17,25 @@ cuda_compiler_version:
 cxx_compiler:
 - gxx
 cxx_compiler_version:
-- '9'
+- '7'
 docker_image:
-- quay.io/condaforge/linux-anvil-cuda:10.2
+- quay.io/condaforge/linux-anvil-cos7-cuda:10.2
 gflags:
 - '2.2'
 glog:
 - '0.5'
 grpc_cpp:
-- '1.38'
+- '1.42'
 libprotobuf:
-- '3.16'
+- '3.19'
 lz4_c:
 - 1.9.3
 numpy:
-- '1.17'
+- '1.18'
+openssl:
+- 1.1.1
 orc:
-- 1.6.8
+- 1.7.2
 pin_run_as_build:
   bzip2:
     max_pin: x
@@ -49,17 +49,17 @@ pin_run_as_build:
 python:
 - 3.7.* *_cpython
 re2:
-- 2021.06.01
+- 2021.11.01
 snappy:
 - '1'
 target_platform:
 - linux-64
 thrift_cpp:
-- 0.14.2
+- 0.15.0
 zip_keys:
 - - c_compiler_version
   - cxx_compiler_version
-- - cuda_compiler_version
+  - cuda_compiler_version
   - cdt_name
   - docker_image
 - - python

diff --git a/...on10.2numpy1.17python3.8.____cpython.yaml → ...rsion7numpy1.18python3.8.____cpython.yaml b/...on10.2numpy1.17python3.8.____cpython.yaml → ...rsion7numpy1.18python3.8.____cpython.yaml
@@ -1,15 +1,13 @@
-aws_sdk_cpp:
-- 1.8.186
 bzip2:
 - '1'
 c_compiler:
 - gcc
 c_compiler_version:
-- '9'
+- '7'
 cdt_name:
 - cos6
 channel_sources:
-- conda-forge,defaults
+- conda-forge
 channel_targets:
 - conda-forge main
 cuda_compiler:
@@ -19,23 +17,25 @@ cuda_compiler_version:
 cxx_compiler:
 - gxx
 cxx_compiler_version:
-- '9'
+- '7'
 docker_image:
-- quay.io/condaforge/linux-anvil-cuda:10.2
+- quay.io/condaforge/linux-anvil-cos7-cuda:10.2
 gflags:
 - '2.2'
 glog:
 - '0.5'
 grpc_cpp:
-- '1.38'
+- '1.42'
 libprotobuf:
-- '3.16'
+- '3.19'
 lz4_c:
 - 1.9.3
 numpy:
-- '1.17'
+- '1.18'
+openssl:
+- 1.1.1
 orc:
-- 1.6.8
+- 1.7.2
 pin_run_as_build:
   bzip2:
     max_pin: x
@@ -49,17 +49,17 @@ pin_run_as_build:
 python:
 - 3.8.* *_cpython
 re2:
-- 2021.06.01
+- 2021.11.01
 snappy:
 - '1'
 target_platform:
 - linux-64
 thrift_cpp:
-- 0.14.2
+- 0.15.0
 zip_keys:
 - - c_compiler_version
   - cxx_compiler_version
-- - cuda_compiler_version
+  - cuda_compiler_version
   - cdt_name
   - docker_image
 - - python

diff --git a/...on10.2numpy1.19python3.9.____cpython.yaml → ...rsion7numpy1.19python3.9.____cpython.yaml b/...on10.2numpy1.19python3.9.____cpython.yaml → ...rsion7numpy1.19python3.9.____cpython.yaml
@@ -1,15 +1,13 @@
-aws_sdk_cpp:
-- 1.8.186
 bzip2:
 - '1'
 c_compiler:
 - gcc
 c_compiler_version:
-- '9'
+- '7'
 cdt_name:
 - cos6
 channel_sources:
-- conda-forge,defaults
+- conda-forge
 channel_targets:
 - conda-forge main
 cuda_compiler:
@@ -19,23 +17,25 @@ cuda_compiler_version:
 cxx_compiler:
 - gxx
 cxx_compiler_version:
-- '9'
+- '7'
 docker_image:
-- quay.io/condaforge/linux-anvil-cuda:10.2
+- quay.io/condaforge/linux-anvil-cos7-cuda:10.2
 gflags:
 - '2.2'
 glog:
 - '0.5'
 grpc_cpp:
-- '1.38'
+- '1.42'
 libprotobuf:
-- '3.16'
+- '3.19'
 lz4_c:
 - 1.9.3
 numpy:
 - '1.19'
+openssl:
+- 1.1.1
 orc:
-- 1.6.8
+- 1.7.2
 pin_run_as_build:
   bzip2:
     max_pin: x
@@ -49,17 +49,17 @@ pin_run_as_build:
 python:
 - 3.9.* *_cpython
 re2:
-- 2021.06.01
+- 2021.11.01
 snappy:
 - '1'
 target_platform:
 - linux-64
 thrift_cpp:
-- 0.14.2
+- 0.15.0
 zip_keys:
 - - c_compiler_version
   - cxx_compiler_version
-- - cuda_compiler_version
+  - cuda_compiler_version
   - cdt_name
   - docker_image
 - - python

diff --git a/...on10.2numpy1.17python3.6.____cpython.yaml → ...sion7numpy1.21python3.10.____cpython.yaml b/...on10.2numpy1.17python3.6.____cpython.yaml → ...sion7numpy1.21python3.10.____cpython.yaml
@@ -1,15 +1,13 @@
-aws_sdk_cpp:
-- 1.8.186
 bzip2:
 - '1'
 c_compiler:
 - gcc
 c_compiler_version:
-- '9'
+- '7'
 cdt_name:
 - cos6
 channel_sources:
-- conda-forge,defaults
+- conda-forge
 channel_targets:
 - conda-forge main
 cuda_compiler:
@@ -19,23 +17,25 @@ cuda_compiler_version:
 cxx_compiler:
 - gxx
 cxx_compiler_version:
-- '9'
+- '7'
 docker_image:
-- quay.io/condaforge/linux-anvil-cuda:10.2
+- quay.io/condaforge/linux-anvil-cos7-cuda:10.2
 gflags:
 - '2.2'
 glog:
 - '0.5'
 grpc_cpp:
-- '1.38'
+- '1.42'
 libprotobuf:
-- '3.16'
+- '3.19'
 lz4_c:
 - 1.9.3
 numpy:
-- '1.17'
+- '1.21'
+openssl:
+- 1.1.1
 orc:
-- 1.6.8
+- 1.7.2
 pin_run_as_build:
   bzip2:
     max_pin: x
@@ -47,19 +47,19 @@ pin_run_as_build:
   zlib:
     max_pin: x.x
 python:
-- 3.6.* *_cpython
+- 3.10.* *_cpython
 re2:
-- 2021.06.01
+- 2021.11.01
 snappy:
 - '1'
 target_platform:
 - linux-64
 thrift_cpp:
-- 0.14.2
+- 0.15.0
 zip_keys:
 - - c_compiler_version
   - cxx_compiler_version
-- - cuda_compiler_version
+  - cuda_compiler_version
   - cdt_name
   - docker_image
 - - python