Merge remote-tracking branch 'origin/plasma-client-mutex' into plasma…

…-client-mutex

.clang-tidy

@@ -16,7 +16,9 @@
 # under the License.
 ---
 Checks:          'clang-diagnostic-*,clang-analyzer-*,-clang-analyzer-alpha*,google-*,modernize-*,readability-*'
-HeaderFilterRegex: 'arrow/.*'
+# produce HeaderFilterRegex from cpp/build-support/lint_exclusions.txt with:
+# echo -n '^('; sed -e 's/*/\.*/g' cpp/build-support/lint_exclusions.txt | tr '\n' '|'; echo ')$'
+HeaderFilterRegex: '^(.*codegen.*|.*_generated.*|.*windows_compatibility.h|.*pyarrow_api.h|.*pyarrow_lib.h|.*python/config.h|.*python/platform.h|.*thirdparty/ae/.*|.*vendored/.*|.*RcppExports.cpp.*|)$'
 AnalyzeTemporaryDtors: true
 CheckOptions:
   - key:             google-readability-braces-around-statements.ShortStatementLines

.github/ISSUE_TEMPLATE.md

@@ -1,16 +1,21 @@
-## GitHub Issues for Apache Arrow
+STOP! Are you reporting a bug, a possible bug, or requesting a
+feature? If so, please report under the ARROW project on the ASF JIRA
+server https://issues.apache.org/jira/browse/ARROW. This JIRA server
+is free to use and open to the public, but you must create an account
+if it is your first time.
 
-We support GitHub issues as a lightweight way to ask questions and engage with
-the Arrow developer community. We use the Apache Software Foundation's JIRA
-instance for maintaining a queue of development work and as the public record
-for work on the project. So, feel free to delete this text and open an issue
-here if you have a question or aren't sure whether something is a bug or
-feature request. Any bugs, suggestions, or feature requests will eventually
-need to end up in JIRA (https://issues.apache.org/jira/browse/ARROW),
-either before or after completing a pull request.
+See our contribution guidelines for more information:
+http://arrow.apache.org/docs/developers/contributing.html
 
-See our contribution guidelines for more information: http://arrow.apache.org/docs/developers/contributing.html
+We have GitHub issues available as a way for new contributors and
+passers-by who are unfamiliar with Apache Software Foundation projects
+to ask questions and interact with the project. Do not be surprised if
+the first response is to open a JIRA issue or to write an e-mail to
+one of the public mailing lists:
 
-For more in-depth development discussions, please use the public mailing list
-dev@arrow.apache.org (first subscribe by sending an e-mail to
-dev-subscribe@arrow.apache.org).
+* Development discussions: dev@arrow.apache.org (first subscribe by
+  sending an e-mail to dev-subscribe@arrow.apache.org).
+* User discussions: user@arrow.apache.org (first subscribe by
+  sending an e-mail to user-subscribe@arrow.apache.org).
+
+Thank you!

.travis.yml

@@ -336,10 +336,9 @@ matrix:
     after_success:
     - pushd ${TRAVIS_BUILD_DIR}/go/arrow
     - bash <(curl -s https://codecov.io/bash) || echo "Codecov did not collect coverage reports"
-  - name: R
+  - name: R (with and without libarrow)
     language: r
     cache: packages
-    latex: false
     dist: xenial
     env:
     - ARROW_TRAVIS_PARQUET=1
@@ -348,6 +347,14 @@ matrix:
     # Have to copy-paste this here because of how R's build steps work
     - eval `python $TRAVIS_BUILD_DIR/ci/detect-changes.py`
     - if [ $ARROW_CI_R_AFFECTED != "1" ]; then exit; fi
+    # First check that it builds without libarrow
+    - pushd ${TRAVIS_BUILD_DIR}/r
+    - R -e 'install.packages("remotes"); remotes::install_deps(dep = TRUE)'
+    - R CMD build .
+    - R CMD check arrow_*tar.gz
+    - rm arrow_*tar.gz
+    - popd
+    # Now, proceed to install the c++ lib and the rest of the job
     - |
         if [ $TRAVIS_OS_NAME == "linux" ]; then
           sudo bash -c "echo -e 'Acquire::Retries 10; Acquire::http::Timeout \"20\";' > /etc/apt/apt.conf.d/99-travis-retry"
@@ -363,25 +370,11 @@ matrix:
     - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$TRAVIS_BUILD_DIR/cpp-install/lib
     - export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:$TRAVIS_BUILD_DIR/cpp-install/lib/pkgconfig
     - export CXX11FLAGS=-Wall
+    # Add this env var so we can assert in the tests that the library is installed correctly
+    - export TEST_R_WITH_ARROW=TRUE
     - pushd ${TRAVIS_BUILD_DIR}/r
     after_success:
     - Rscript ../ci/travis_upload_r_coverage.R
-  - name: R_no_libarrow
-    language: r
-    cache: packages
-    latex: false
-    dist: xenial
-    before_install:
-    # Have to copy-paste this here because of how R's build steps work
-    - eval `python $TRAVIS_BUILD_DIR/ci/detect-changes.py`
-    - if [ $ARROW_CI_R_AFFECTED != "1" ]; then exit; fi
-    - |
-        if [ $TRAVIS_OS_NAME == "linux" ]; then
-          sudo bash -c "echo -e 'Acquire::Retries 10; Acquire::http::Timeout \"20\";' > /etc/apt/apt.conf.d/99-travis-retry"
-          sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
-          sudo apt-get update -qq
-        fi
-    - pushd ${TRAVIS_BUILD_DIR}/r
 
 after_failure:
 - |

c_glib/Brewfile

@@ -21,3 +21,4 @@ brew "gtk-doc"
 brew "libtool"
 brew "lua"
 brew "luarocks"
+brew "meson"

ci/appveyor-cpp-build-mingw.bat

@@ -42,17 +42,16 @@ pushd %CPP_BUILD_DIR%
 
 cmake ^
     -G "MSYS Makefiles" ^
-    -DCMAKE_INSTALL_PREFIX=%INSTALL_DIR% ^
-    -DCMAKE_BUILD_TYPE=%CMAKE_BUILD_TYPE% ^
     -DARROW_BUILD_STATIC=OFF ^
-    -DARROW_VERBOSE_THIRDPARTY_BUILD=OFF ^
+    -DARROW_BUILD_TESTS=ON ^
     -DARROW_PACKAGE_PREFIX=%MINGW_PREFIX% ^
-    -DARROW_USE_GLOG=OFF ^
     -DARROW_PARQUET=ON ^
     -DARROW_PYTHON=ON ^
+    -DARROW_USE_GLOG=OFF ^
+    -DCMAKE_BUILD_TYPE=%CMAKE_BUILD_TYPE% ^
+    -DCMAKE_INSTALL_PREFIX=%INSTALL_DIR% ^
     -DPythonInterp_FIND_VERSION=ON ^
     -DPythonInterp_FIND_VERSION_MAJOR=3 ^
-    -DARROW_BUILD_TESTS=ON ^
     .. || exit /B
 make -j4 || exit /B
 setlocal

ci/appveyor-cpp-setup-mingw.bat

@@ -43,6 +43,7 @@ pacman --sync --noconfirm ^
   %MINGW_PACKAGE_PREFIX%-flatbuffers ^
   %MINGW_PACKAGE_PREFIX%-gflags ^
   %MINGW_PACKAGE_PREFIX%-gobject-introspection ^
+  %MINGW_PACKAGE_PREFIX%-grpc ^
   %MINGW_PACKAGE_PREFIX%-gtest ^
   %MINGW_PACKAGE_PREFIX%-gtk-doc ^
   %MINGW_PACKAGE_PREFIX%-lz4 ^
@@ -52,6 +53,7 @@ pacman --sync --noconfirm ^
   %MINGW_PACKAGE_PREFIX%-rapidjson ^
   %MINGW_PACKAGE_PREFIX%-snappy ^
   %MINGW_PACKAGE_PREFIX%-thrift ^
+  %MINGW_PACKAGE_PREFIX%-uriparser ^
   %MINGW_PACKAGE_PREFIX%-zlib ^
   %MINGW_PACKAGE_PREFIX%-zstd || exit /B
 

ci/travis_before_script_c_glib.sh

@@ -21,17 +21,14 @@ set -ex
 
 source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
 
-source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh
-
-conda create -n meson -y -q python=3.6
-conda activate meson
-
-pip install meson
-
 if [ $TRAVIS_OS_NAME = "osx" ]; then
   export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:/usr/local/opt/libffi/lib/pkgconfig
   export XML_CATALOG_FILES=/usr/local/etc/xml/catalog
 else
+  source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh
+  conda create -n meson -y -q python=3.6
+  conda activate meson
+  pip install meson
   sudo apt-get install -y -q \
     autoconf-archive \
     gtk-doc-tools \

cpp/CMakeLists.txt

@@ -270,7 +270,12 @@ if(NOT ARROW_BUILD_TESTS)
   set(NO_TESTS 1)
 else()
   add_custom_target(all-tests)
-  add_custom_target(unittest ctest -L unittest)
+  add_custom_target(unittest
+                    ctest
+                    -j4
+                    -L
+                    unittest
+                    --output-on-failure)
   add_dependencies(unittest all-tests)
 endif()
 
@@ -555,7 +560,7 @@ if(ARROW_WITH_URIPARSER)
   list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS uriparser::uriparser)
 endif()
 
-if(PARQUET_BUILD_ENCRYPTION)
+if(ARROW_USE_OPENSSL)
   list(APPEND ARROW_LINK_LIBS OpenSSL::Crypto)
   list(APPEND ARROW_STATIC_LINK_LIBS OpenSSL::Crypto)
   list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS OpenSSL::Crypto)

cpp/Dockerfile.alpine

@@ -44,6 +44,7 @@ RUN apk add --no-cache -q \
 # ARROW-4917: ORC fails with compiler problems
 ENV CC=gcc \
     CXX=g++ \
+    ARROW_FLIGHT=OFF \
     ARROW_GANDIVA=OFF \
     ARROW_ORC=OFF \
     ARROW_PARQUET=ON \

cpp/build-support/get_apache_mirror.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# This script queries the ASF mirror system to obtain a suggested
+# mirror for downloading dependencies, e.g. in CMake
+
+import json
+try:
+    from urllib2 import urlopen
+except ImportError:
+    # py3
+    from urllib.request import urlopen
+
+suggested_mirror = urlopen('https://www.apache.org/dyn/'
+                           'closer.cgi?as_json=1').read()
+print(json.loads(suggested_mirror.decode('utf-8'))['preferred'])

cpp/build-support/lint_cpp_cli.py

@@ -98,7 +98,8 @@ def lint_files():
 
             # Only run on header files
             if filename.endswith('.h'):
-                yield from lint_file(full_path)
+                for _ in lint_file(full_path):
+                    yield _
 
 
 if __name__ == '__main__':

cpp/build-support/run_clang_tidy.py

@@ -94,8 +94,13 @@ def _check_all(cmd, filenames):
                         help="If specified, only print errors")
     arguments = parser.parse_args()
 
+    exclude_globs = []
+    if arguments.exclude_globs:
+        for line in open(arguments.exclude_globs):
+            exclude_globs.append(line.strip())
+
     linted_filenames = []
-    for path in lintutils.get_sources(arguments.source_dir):
+    for path in lintutils.get_sources(arguments.source_dir, exclude_globs):
         linted_filenames.append(path)
 
     if not arguments.quiet:
@@ -111,8 +116,9 @@ def _check_all(cmd, filenames):
         cmd.append('-fix')
         results = lintutils.run_parallel(
             [cmd + some for some in lintutils.chunk(linted_filenames, 16)])
-        for result in results:
-            result.check_returncode()
+        for returncode, stdout, stderr in results:
+            if returncode != 0:
+                sys.exit(returncode)
 
     else:
         _check_all(cmd, linted_filenames)

cpp/cmake_modules/BuildUtils.cmake

@@ -20,6 +20,7 @@
 # search there as well.
 set(LIB_PATH_SUFFIXES
     "${CMAKE_LIBRARY_ARCHITECTURE}"
+    "lib/${CMAKE_LIBRARY_ARCHITECTURE}"
     "lib64"
     "lib32"
     "lib"
@@ -720,9 +721,11 @@ function(ARROW_INSTALL_ALL_HEADERS PATH)
 
   set(PUBLIC_HEADERS)
   foreach(HEADER ${CURRENT_DIRECTORY_HEADERS})
-    if(NOT ((HEADER MATCHES "internal")))
-      list(APPEND PUBLIC_HEADERS ${HEADER})
+    get_filename_component(HEADER_BASENAME ${HEADER} NAME)
+    if(HEADER_BASENAME MATCHES "internal")
+      continue()
     endif()
+    list(APPEND PUBLIC_HEADERS ${HEADER})
   endforeach()
   install(FILES ${PUBLIC_HEADERS} DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/${PATH}")
 endfunction()

cpp/cmake_modules/DefineOptions.cmake

@@ -139,6 +139,8 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
 
   define_option(ARROW_COMPUTE "Build the Arrow Compute Modules" ON)
 
+  define_option(ARROW_DATASET "Build the Arrow Dataset Modules" ON)
+
   define_option(ARROW_FLIGHT
                 "Build the Arrow Flight RPC System (requires GRPC, Protocol Buffers)" OFF)
 
@@ -278,8 +280,6 @@ Note that this requires linking Boost statically" OFF)
   #----------------------------------------------------------------------
   set_option_category("Parquet")
 
-  define_option(PARQUET_BUILD_ENCRYPTION "Build Parquet with encryption support" ON)
-
   define_option(PARQUET_MINIMAL_DEPENDENCY
                 "Depend only on Thirdparty headers to build libparquet. \
 Always OFF if building binaries" OFF)
@@ -291,6 +291,9 @@ Always OFF if building binaries" OFF)
   define_option(PARQUET_BUILD_EXAMPLES
                 "Build the Parquet examples. Requires static libraries to be built." OFF)
 
+  define_option(PARQUET_REQUIRE_ENCRYPTION
+                "Build support for encryption. Fail if OpenSSL is not found" OFF)
+
   #----------------------------------------------------------------------
   set_option_category("Gandiva")