Merge pull request #3 from BonsaiAI/ms/fix-0.6.6-con-close

[fix][0.6.6] Raylet connection closed & RayOutOfMemoryError

.bazelrc

@@ -1,4 +1,37 @@
-# build config
-build --compilation_mode=opt
+# Must be first. Enables build:windows, build:linux, build:macos, build:freebsd, build:openbsd
+build --enable_platform_specific_config
+###############################################################################
+build --action_env=BAZEL_LLVM
+build --action_env=BAZEL_SH
 build --action_env=PATH
 build --action_env=PYTHON_BIN_PATH
+# Use Clang-Cl (Clang front-end with Visual C++ backend) on Windows
+build --action_env=USE_CLANG_CL=1
+build:linux   --compilation_mode=opt
+build:macos   --compilation_mode=opt
+build:windows --compilation_mode=fastbuild
+# TODO(mehrdadn): Revert the "-\\.(asm|S)$" exclusion when this Bazel bug
+#                 for compiling assembly files is fixed on Windows:
+#                 https://github.com/bazelbuild/bazel/issues/8924
+# Warnings should be errors
+build --per_file_copt="-\\.(asm|S)$,-.*/arrow/util/logging\\.cc@-Werror"
+# This workaround is required in order to compile hiredis
+build --per_file_copt="+.*/hiredis/dict.c@-Wno-error,-Wno-unused-function"
+# Ignore warnings for protobuf generated files and external projects.
+build --per_file_copt="-\\.(asm|S)$,\\.pb\\.cc$@-w"
+build --per_file_copt="-\\.(asm|S)$,external/.*@-w"
+# Ignore minor warnings for host tools, which we generally can't control
+build --host_copt="-Wno-builtin-macro-redefined"
+build --host_copt="-Wno-inconsistent-missing-override"
+build --host_copt="-Wno-microsoft-unqualified-friend"
+# This workaround is needed due to https://github.com/bazelbuild/bazel/issues/4341
+build --per_file_copt="-\\.(asm|S)$,external/com_github_grpc_grpc/.*@-DGRPC_BAZEL_BUILD"
+build --http_timeout_scaling=5.0
+
+# Thread sanitizer configuration:
+build:tsan --strip=never
+build:tsan --copt -fsanitize=thread
+build:tsan --copt -DTHREAD_SANITIZER
+build:tsan --copt -g
+build:tsan --copt -fno-omit-frame-pointer
+build:tsan --linkopt -fsanitize=thread

.gitignore

@@ -146,11 +146,6 @@ java/**/.classpath
 java/**/.project
 java/runtime/native_dependencies/
 
-# Java third-party dependencies.
-java/third_party/*/
-!java/third_party/workspace.bzl
-!java/third_party/BUILD
-
 # python virtual env
 venv
 

WORKSPACE

@@ -1,95 +1,9 @@
 workspace(name = "com_github_ray_project_ray")
 
-load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository", "new_git_repository")
-load("//java:repo.bzl", "java_repositories")
+load("//bazel:ray_deps_setup.bzl", "ray_deps_setup")
 
-java_repositories()
+ray_deps_setup()
 
-git_repository(
-    name = "com_github_checkstyle_java",
-    commit = "85f37871ca03b9d3fee63c69c8107f167e24e77b",
-    remote = "https://github.com/ruifangChen/checkstyle_java",
-)
+load("//bazel:ray_deps_build_all.bzl", "ray_deps_build_all")
 
-load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
-
-git_repository(
-    name = "com_github_nelhage_rules_boost",
-    commit = "6d6fd834281cb8f8e758dd9ad76df86304bf1869",
-    remote = "https://github.com/nelhage/rules_boost",
-)
-
-load("@com_github_nelhage_rules_boost//:boost/boost.bzl", "boost_deps")
-
-boost_deps()
-
-git_repository(
-    name = "com_github_google_flatbuffers",
-    commit = "63d51afd1196336a7d1f56a988091ef05deb1c62",
-    remote = "https://github.com/google/flatbuffers.git",
-)
-
-git_repository(
-    name = "com_google_googletest",
-    commit = "3306848f697568aacf4bcca330f6bdd5ce671899",
-    remote = "https://github.com/google/googletest",
-)
-
-git_repository(
-    name = "com_github_gflags_gflags",
-    remote = "https://github.com/gflags/gflags.git",
-    tag = "v2.2.2",
-)
-
-new_git_repository(
-    name = "com_github_google_glog",
-    build_file = "@//bazel:BUILD.glog",
-    commit = "5c576f78c49b28d89b23fbb1fc80f54c879ec02e",
-    remote = "https://github.com/google/glog",
-)
-
-new_git_repository(
-    name = "plasma",
-    build_file = "@//bazel:BUILD.plasma",
-    commit = "d00497b38be84fd77c40cbf77f3422f2a81c44f9",
-    remote = "https://github.com/apache/arrow",
-)
-
-new_git_repository(
-    name = "cython",
-    build_file = "@//bazel:BUILD.cython",
-    commit = "49414dbc7ddc2ca2979d6dbe1e44714b10d72e7e",
-    remote = "https://github.com/cython/cython",
-)
-
-load("@//bazel:python_configure.bzl", "python_configure")
-
-python_configure(name = "local_config_python")
-
-http_archive(
-    name = "io_opencensus_cpp",
-    strip_prefix = "opencensus-cpp-0.3.0",
-    urls = ["https://github.com/census-instrumentation/opencensus-cpp/archive/v0.3.0.zip"],
-)
-
-# OpenCensus depends on Abseil so we have to explicitly pull it in.
-# This is how diamond dependencies are prevented.
-git_repository(
-    name = "com_google_absl",
-    commit = "88a152ae747c3c42dc9167d46c590929b048d436",
-    remote = "https://github.com/abseil/abseil-cpp.git",
-)
-
-# OpenCensus depends on jupp0r/prometheus-cpp
-http_archive(
-    name = "com_github_jupp0r_prometheus_cpp",
-    strip_prefix = "prometheus-cpp-master",
-
-    # TODO(qwang): We should use the repository of `jupp0r` here when this PR
-    # `https://github.com/jupp0r/prometheus-cpp/pull/225` getting merged.
-    urls = ["https://github.com/jovany-wang/prometheus-cpp/archive/master.zip"],
-)
-
-load("@com_github_jupp0r_prometheus_cpp//:repositories.bzl", "prometheus_cpp_repositories")
-
-prometheus_cpp_repositories()
+ray_deps_build_all()

bazel/BUILD.plasma

@@ -6,7 +6,7 @@ java_library(
     data = [":plasma_client_java"],
     visibility = ["//visibility:public"],
     deps = [
-        "@org_slf4j_slf4j_api//jar",
+        "@maven//:org_slf4j_slf4j_api",
     ],
 )
 
@@ -17,7 +17,7 @@ java_binary(
     visibility = ["//visibility:public"],
     deps = [
         ":org_apache_arrow_arrow_plasma",
-        "@junit_junit//jar",
+        "@maven//:junit_junit",
     ],
 )
 

bazel/ray.bzl

@@ -1,5 +1,6 @@
 load("@com_github_google_flatbuffers//:build_defs.bzl", "flatbuffer_library_public")
 load("@com_github_checkstyle_java//checkstyle:checkstyle.bzl", "checkstyle_test")
+load("@bazel_common//tools/maven:pom_file.bzl", "pom_file")
 
 def flatbuffer_py_library(name, srcs, outs, out_prefix, includes = [], include_paths = []):
     flatbuffer_library_public(
@@ -52,3 +53,13 @@ def define_java_module(name, additional_srcs = [], additional_resources = [], de
             size = "small",
             tags = ["checkstyle"],
         )
+    pom_file(
+        name = "org_ray_ray_" + name + "_pom",
+        targets = [
+            ":org_ray_ray_" + name,
+        ],
+        template_file = name + "/pom_template.xml",
+        substitutions = {
+            "{auto_gen_header}": "<!-- This file is auto-generated by Bazel from pom_template.xml, do not modify it. -->",
+        },
+    )

bazel/ray_deps_build_all.bzl

@@ -0,0 +1,12 @@
+load("@com_github_ray_project_ray//java:dependencies.bzl", "gen_java_deps")
+load("@com_github_nelhage_rules_boost//:boost/boost.bzl", "boost_deps")
+load("@com_github_jupp0r_prometheus_cpp//:repositories.bzl", "prometheus_cpp_repositories")
+load("@com_github_ray_project_ray//bazel:python_configure.bzl", "python_configure")
+load("@com_github_checkstyle_java//:repo.bzl", "checkstyle_deps")
+
+def ray_deps_build_all():
+  gen_java_deps()
+  checkstyle_deps()
+  boost_deps()
+  prometheus_cpp_repositories()
+  python_configure(name = "local_config_python")

bazel/ray_deps_setup.bzl

@@ -0,0 +1,97 @@
+load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository", "new_git_repository")
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
+def ray_deps_setup():
+    git_repository(
+        name = "rules_jvm_external",
+        tag = "2.10",
+        remote = "https://github.com/bazelbuild/rules_jvm_external",
+    )
+
+    git_repository(
+        name = "bazel_common",
+        commit = "bf87eb1a4ddbfc95e215b0897f3edc89b2254a1a",
+        remote = "https://github.com/google/bazel-common",
+    )
+
+    BAZEL_SKYLIB_TAG = "0.6.0"
+
+    http_archive(
+        name = "bazel_skylib",
+        strip_prefix = "bazel-skylib-%s" % BAZEL_SKYLIB_TAG,
+        url = "https://github.com/bazelbuild/bazel-skylib/archive/%s.tar.gz" % BAZEL_SKYLIB_TAG,
+    )
+
+    git_repository(
+        name = "com_github_checkstyle_java",
+        commit = "ef367030d1433877a3360bbfceca18a5d0791bdd",
+        remote = "https://github.com/ray-project/checkstyle_java",
+    )
+
+    git_repository(
+        name = "com_github_nelhage_rules_boost",
+        commit = "5171b9724fbb39c5fdad37b9ca9b544e8858d8ac",
+        remote = "https://github.com/ray-project/rules_boost",
+    )
+
+    git_repository(
+        name = "com_github_google_flatbuffers",
+        commit = "63d51afd1196336a7d1f56a988091ef05deb1c62",
+        remote = "https://github.com/google/flatbuffers.git",
+    )
+
+    git_repository(
+        name = "com_google_googletest",
+        commit = "3306848f697568aacf4bcca330f6bdd5ce671899",
+        remote = "https://github.com/google/googletest",
+    )
+
+    git_repository(
+        name = "com_github_gflags_gflags",
+        remote = "https://github.com/gflags/gflags.git",
+        tag = "v2.2.2",
+    )
+
+    new_git_repository(
+        name = "com_github_google_glog",
+        build_file = "@//bazel:BUILD.glog",
+        commit = "5c576f78c49b28d89b23fbb1fc80f54c879ec02e",
+        remote = "https://github.com/google/glog",
+    )
+
+    new_git_repository(
+        name = "plasma",
+        build_file = "@//bazel:BUILD.plasma",
+        commit = "d00497b38be84fd77c40cbf77f3422f2a81c44f9",
+        remote = "https://github.com/apache/arrow",
+    )
+
+    new_git_repository(
+        name = "cython",
+        build_file = "@//bazel:BUILD.cython",
+        commit = "49414dbc7ddc2ca2979d6dbe1e44714b10d72e7e",
+        remote = "https://github.com/cython/cython",
+    )
+
+    http_archive(
+        name = "io_opencensus_cpp",
+        strip_prefix = "opencensus-cpp-3aa11f20dd610cb8d2f7c62e58d1e69196aadf11",
+        urls = ["https://github.com/census-instrumentation/opencensus-cpp/archive/3aa11f20dd610cb8d2f7c62e58d1e69196aadf11.zip"],
+    )
+
+    # OpenCensus depends on Abseil so we have to explicitly pull it in.
+    # This is how diamond dependencies are prevented.
+    git_repository(
+        name = "com_google_absl",
+        commit = "5b65c4af5107176555b23a638e5947686410ac1f",
+        remote = "https://github.com/abseil/abseil-cpp.git",
+    )
+
+    # OpenCensus depends on jupp0r/prometheus-cpp
+    git_repository(
+        name = "com_github_jupp0r_prometheus_cpp",
+        commit = "5c45ba7ddc0585d765a43d136764dd2a542bd495",
+        # TODO(qwang): We should use the repository of `jupp0r` here when this PR
+        # `https://github.com/jupp0r/prometheus-cpp/pull/225` getting merged.
+        remote = "https://github.com/ray-project/prometheus-cpp.git",
+    )

build.sh

@@ -101,7 +101,6 @@ $PYTHON_EXECUTABLE -m pip install \
 export PYTHON_BIN_PATH="$PYTHON_EXECUTABLE"
 
 if [ "$RAY_BUILD_JAVA" == "YES" ]; then
-  bazel run //java:bazel_deps -- generate -r $ROOT_DIR -s java/third_party/workspace.bzl -d java/dependencies.yaml
   bazel build //java:all --verbose_failures
 fi