Update Bazel to 0.29.0 (tensorflow#465)

* Update Bazel to 0.29.0 Tried Bazel 0.29.0 locally with Linux and it works. So create a PR to see if it works for macOS as well. Signed-off-by: Yong Tang <yong.tang.github@outlook.com> * Update grpc/googleapis/etc Signed-off-by: Yong Tang <yong.tang.github@outlook.com> * Pin h5py==2.9.0 Signed-off-by: Yong Tang <yong.tang.github@outlook.com> * Fix bazel lint error Signed-off-by: Yong Tang <yong.tang.github@outlook.com> * Pin h5py==2.9.0 in bazel build, as h5py==2.10.0 is broken in ubuntu 16.04 Signed-off-by: Yong Tang <yong.tang.github@outlook.com> * Bump the OSX to osx10 on Travis CI, See emscripten-core/emscripten#5418 Signed-off-by: Yong Tang <yong.tang.github@outlook.com> * Switch to xcode 9.3, since ignite requires JDK 1.8 or 1.9 (xcode10 have JDK 10) Signed-off-by: Yong Tang <yong.tang.github@outlook.com> * Fix the import path issue in grpc Signed-off-by: Yong Tang <yong.tang.github@outlook.com> * Removed `python -m pip install -q grpcio-tools` as this is not needed anymore Signed-off-by: Yong Tang <yong.tang.github@outlook.com> * Build //tensorflow_io/core:all to create Travis CI Cache Signed-off-by: Yong Tang <yong.tang.github@outlook.com> * Fix nucleus build failure caused by bazel 0.29.0 Signed-off-by: Yong Tang <yong.tang.github@outlook.com> * Enable all builds Signed-off-by: Yong Tang <yong.tang.github@outlook.com> * Build //tensorflow_io/core:all to create Travis CI Cache Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
i-ony · Sep 7, 2019 · 3ba802e · 3ba802e
1 parent 2177465
commit 3ba802e
Show file tree

Hide file tree

Showing 10 changed files with 121 additions and 155 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -71,28 +71,28 @@ jobs:
   - stage: build
     name: "Nightly Release Build on macOS 2.7.13"
     os: osx
-    osx_image: xcode9
+    osx_image: xcode9.3
     script:
     - bash -x -e .travis/python.release.sh --nightly ${TRAVIS_BUILD_NUMBER}
-    - bash -x -e .travis/wheel.test.sh
+    - echo bash -x -e .travis/wheel.test.sh
     after_success: bash -x -e .travis/after-success.sh
 
   - stage: build
     name: "Nightly Release Build on macOS 3.5.3"
     os: osx
-    osx_image: xcode9
+    osx_image: xcode9.3
     script:
     - bash -x -e .travis/python.release.sh --nightly ${TRAVIS_BUILD_NUMBER}
-    - bash -x -e .travis/wheel.test.sh
+    - echo bash -x -e .travis/wheel.test.sh
     after_success: bash -x -e .travis/after-success.sh
 
   - stage: build
     name: "Nightly Release Build on macOS 3.6.2"
     os: osx
-    osx_image: xcode9
+    osx_image: xcode9.3
     script:
     - bash -x -e .travis/python.release.sh --nightly ${TRAVIS_BUILD_NUMBER}
-    - bash -x -e .travis/wheel.test.sh
+    - echo bash -x -e .travis/wheel.test.sh
     after_success: bash -x -e .travis/after-success.sh
 
 notifications:

diff --git a/.travis/bazel.build.sh b/.travis/bazel.build.sh
@@ -20,7 +20,7 @@ if [[ "$#" -gt 0 ]]; then
     export TENSORFLOW_INSTALL="${1}"
 fi
 
-export BAZEL_VERSION=0.24.1 BAZEL_OS=$(uname | tr '[:upper:]' '[:lower:]')
+export BAZEL_VERSION=0.29.0 BAZEL_OS=$(uname | tr '[:upper:]' '[:lower:]')
 curl -sSOL https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-${BAZEL_OS}-x86_64.sh
 bash -e bazel-${BAZEL_VERSION}-installer-${BAZEL_OS}-x86_64.sh
 bazel version
@@ -30,9 +30,9 @@ if [[ $(uname) == "Linux" ]]; then
   python get-pip.py -q
   python -m pip --version
 fi
-python -m pip install -q ${TENSORFLOW_INSTALL}
 
-python -m pip install -q grpcio-tools
+python -m pip install -q h5py==2.9.0
+python -m pip install -q ${TENSORFLOW_INSTALL}
 
 python third_party/tf/configure.py
 
@@ -57,7 +57,7 @@ bazel build \
   --noshow_loading_progress \
   --verbose_failures \
   --test_output=errors \
-  -- //tensorflow_io/...
+  -- //tensorflow_io/core:all
 
 rm -rf build && mkdir -p build
 

diff --git a/WORKSPACE b/WORKSPACE
@@ -2,6 +2,7 @@ workspace(name = "org_tensorflow_io")
 
 load("//third_party/tf:tf_configure.bzl", "tf_configure")
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
 
 tf_configure(
     name = "local_config_tf",
@@ -235,35 +236,48 @@ http_archive(
 
 http_archive(
     name = "com_github_grpc_grpc",
-    sha256 = "11ac793c562143d52fd440f6549588712badc79211cdc8c509b183cb69bddad8",
-    strip_prefix = "grpc-1.22.0",
-    urls = [
-        "https://mirror.bazel.build/github.com/grpc/grpc/archive/v1.22.0.tar.gz",
-        "https://github.com/grpc/grpc/archive/v1.22.0.tar.gz",
+    patch_args = ["-p1"],
+    patches = [
+        "//third_party:grpc.patch",
     ],
-)
-
-# 3.8.0 with a fix to BUILD file
-http_archive(
-    name = "com_google_protobuf",
-    sha256 = "b9e92f9af8819bbbc514e2902aec860415b70209f31dfc8c4fa72515a5df9d59",
-    strip_prefix = "protobuf-310ba5ee72661c081129eb878c1bbcec936b20f0",
+    sha256 = "6dc4f122527670099124a71d8a180b0b074a18efa939173d6c3a0673229f57d3",
+    strip_prefix = "grpc-e68ce1164b49529de12fbba63d53f081aef5c90e",
     urls = [
-        "http://mirror.tensorflow.org/github.com/protocolbuffers/protobuf/archive/310ba5ee72661c081129eb878c1bbcec936b20f0.tar.gz",
-        "https://github.com/protocolbuffers/protobuf/archive/310ba5ee72661c081129eb878c1bbcec936b20f0.tar.gz",
+        "https://github.com/grpc/grpc/archive/e68ce1164b49529de12fbba63d53f081aef5c90e.tar.gz",
     ],
 )
 
-http_archive(
-    name = "bazel_skylib",
-    sha256 = "2ef429f5d7ce7111263289644d233707dba35e39696377ebab8b0bc701f7818e",
-    urls = ["https://github.com/bazelbuild/bazel-skylib/releases/download/0.8.0/bazel-skylib.0.8.0.tar.gz"],
-)
-
 load("@com_github_grpc_grpc//bazel:grpc_deps.bzl", "grpc_deps")
 
 grpc_deps()
 
+load("@io_bazel_rules_python//python:pip.bzl", "pip_import", "pip_repositories")
+
+pip_import(
+    name = "grpc_python_dependencies",
+    requirements = "@com_github_grpc_grpc//:requirements.bazel.txt",
+)
+
+load("@io_bazel_rules_python//python:pip.bzl", "pip_repositories")
+load("@grpc_python_dependencies//:requirements.bzl", "pip_install")
+
+pip_repositories()
+
+pip_install()
+
+# TODO(https://github.com/grpc/grpc/issues/19835): Remove.
+load("@upb//bazel:workspace_deps.bzl", "upb_deps")
+
+upb_deps()
+
+load("@build_bazel_rules_apple//apple:repositories.bzl", "apple_rules_dependencies")
+
+apple_rules_dependencies()
+
+load("@build_bazel_apple_support//lib:repositories.bzl", "apple_support_dependencies")
+
+apple_support_dependencies()
+
 http_archive(
     name = "giflib",
     build_file = "//third_party:giflib.BUILD",
@@ -277,28 +291,21 @@ http_archive(
 
 http_archive(
     name = "com_github_googleapis_google_cloud_cpp",
-    sha256 = "3abe2cf553ce33ff58d23848ae716cd2fcabfd454b89f6f65a92ed261244c1df",
-    strip_prefix = "google-cloud-cpp-0.11.0",
+    sha256 = "35058ff14e4f9f49f78da2f1bbf1c03f27e8e40ec65c51f62720346e99803392",
+    strip_prefix = "google-cloud-cpp-0.13.0",
     urls = [
-        "https://mirror.bazel.build/github.com/googleapis/google-cloud-cpp/archive/v0.11.0.tar.gz",
-        "https://github.com/googleapis/google-cloud-cpp/archive/v0.11.0.tar.gz",
+        "https://mirror.bazel.build/github.com/googleapis/google-cloud-cpp/archive/v0.13.0.tar.gz",
+        "https://github.com/googleapis/google-cloud-cpp/archive/v0.13.0.tar.gz",
     ],
 )
 
-# Manually load com_google_googleapis as we need a patch for pubsub
-# The patch file was generated from:
-# diff -Naur a b > third_party/googleapis.patch
 http_archive(
     name = "com_google_googleapis",
     build_file = "@com_github_googleapis_google_cloud_cpp//bazel:googleapis.BUILD",
-    patch_args = ["-p1"],
-    patches = [
-        "//third_party:googleapis.patch",
-    ],
-    sha256 = "90bcdf27b41b1c3900838fe4edaf89080ca67026608817946b4ae5e2b925c711",
-    strip_prefix = "googleapis-7152063cb170d23c5c110e243711d8eb6fda6a1c",
+    sha256 = "cb531e445115e28054a33ad968c2d7d8ade4693721866ce1b9adf9a78762c032",
+    strip_prefix = "googleapis-960b76b1f0c46d12610088977d1129cc7405f3dc",
     urls = [
-        "https://github.com/googleapis/googleapis/archive/7152063cb170d23c5c110e243711d8eb6fda6a1c.tar.gz",
+        "https://github.com/googleapis/googleapis/archive/960b76b1f0c46d12610088977d1129cc7405f3dc.tar.gz",
     ],
 )
 

diff --git a/configure.sh b/configure.sh
@@ -20,5 +20,5 @@ if python -c "import tensorflow as tf" &> /dev/null; then
 else
     python -m pip install $(python setup.py --package-version)
 fi
-python -m pip install grpcio-tools
+python -m pip install h5py==2.9.0
 python third_party/tf/configure.py
diff --git a/tensorflow_io/grpc/BUILD b/tensorflow_io/grpc/BUILD
@@ -3,43 +3,28 @@ licenses(["notice"])  # Apache 2.0
 package(default_visibility = ["//visibility:public"])
 
 load("@com_github_grpc_grpc//bazel:cc_grpc_library.bzl", "cc_grpc_library")
+load("@com_github_grpc_grpc//bazel:python_rules.bzl", "py_grpc_library", "py_proto_library")
 load(
     "//third_party/tf:tensorflow_io.bzl",
     "tf_io_copts",
 )
 
-genrule(
-    name = "endpoint_py",
-    srcs = [
-        "endpoint.proto",
-    ],
-    outs = [
-        "python/ops/__init__.py",
-        "python/ops/endpoint_pb2.py",
-        "python/ops/endpoint_pb2_grpc.py",
-    ],
-    cmd = "python -m grpc_tools.protoc -Itensorflow_io/grpc --python_out=$(BINDIR)/tensorflow_io/grpc/python/ops/ --grpc_python_out=$(BINDIR)/tensorflow_io/grpc/python/ops/ $< ; touch $(BINDIR)/tensorflow_io/grpc/python/ops/__init__.py",
-    output_to_bindir = True,
-)
-
 proto_library(
-    name = "_any_proto_only",
+    name = "endpoint_proto",
+    srcs = ["endpoint.proto"],
     deps = ["@com_google_protobuf//:any_proto"],
 )
 
 cc_proto_library(
-    name = "any_proto",
-    deps = ["@com_google_protobuf//:any_proto"],
+    name = "endpoint_cc_proto",
+    deps = [":endpoint_proto"],
 )
 
 cc_grpc_library(
-    name = "endpoint_cc",
-    srcs = [
-        "endpoint.proto",
-    ],
-    proto_only = False,
-    well_known_protos = True,
-    deps = [":any_proto"],
+    name = "endpoint_cc_grpc",
+    srcs = [":endpoint_proto"],
+    grpc_only = True,
+    deps = [":endpoint_cc_proto"],
 )
 
 cc_binary(
@@ -54,12 +39,22 @@ cc_binary(
     ],
     linkshared = 1,
     deps = [
-        ":endpoint_cc",
+        ":endpoint_cc_grpc",
         "//tensorflow_io/core:dataset_ops",
         "@com_github_grpc_grpc//:grpc++",
-        "@com_google_protobuf//:protobuf",
         "@libarchive",
         "@local_config_tf//:libtensorflow_framework",
         "@local_config_tf//:tf_header_lib",
     ],
 )
+
+py_proto_library(
+    name = "endpoint_py_pb2",
+    deps = [":endpoint_proto"],
+)
+
+py_grpc_library(
+    name = "endpoint_py_pb2_grpc",
+    srcs = [":endpoint_proto"],
+    deps = [":endpoint_py_pb2"],
+)
diff --git a/tensorflow_io/grpc/python/ops/grpc_endpoint.py b/tensorflow_io/grpc/python/ops/grpc_endpoint.py
@@ -24,15 +24,31 @@
 import google.protobuf.any_pb2
 
 import tensorflow as tf
-# Incase test is done with TFIO_DATAPATH specified, the
-# import path need to be extended to capture generated
-# grpc files:
+
+# In case test is done with TFIO_DATAPATH specified, the
+# import path need to be extended to capture generated grpc files:
+grpcpath = os.path.join(os.path.dirname(__file__), "..", "..")
 datapath = os.environ.get('TFIO_DATAPATH')
-sys.path.append(os.path.abspath(
-    os.path.dirname(__file__) if datapath is None else os.path.join(
-        datapath, "tensorflow_io", "grpc", "python", "ops")))
-import endpoint_pb2      # pylint: disable=wrong-import-position,unused-import
-import endpoint_pb2_grpc # pylint: disable=wrong-import-position,unused-import
+if datapath is not None:
+  grpcpath = os.path.join(datapath, "tensorflow_io", "grpc")
+grpcpath = os.path.abspath(grpcpath)
+sys.path.append(grpcpath)
+
+import endpoint_pb2      # pylint: disable=wrong-import-position
+# For some reason generated code in grpc uses:
+# from tensorflow_io.grpc import endpoint_pb2
+# so below is needed
+class MetaPathFinder(object):
+  def find_module(self, fullname, _):
+    if fullname == "tensorflow_io.grpc.endpoint_pb2":
+      return self
+    return None
+  def load_module(self, fullname):
+    if fullname == "tensorflow_io.grpc.endpoint_pb2":
+      return endpoint_pb2
+    return None
+sys.meta_path.append(MetaPathFinder())
+import endpoint_pb2_grpc # pylint: disable=wrong-import-position
 
 class GRPCEndpoint(endpoint_pb2_grpc.GRPCEndpointServicer):
   """GRPCEndpoint"""

diff --git a/third_party/googleapis.patch b/third_party/googleapis.patch
diff --git a/third_party/grpc.patch b/third_party/grpc.patch
@@ -0,0 +1,12 @@
+diff -Naur a/bazel/python_rules.bzl b/bazel/python_rules.bzl
+--- a/bazel/python_rules.bzl	2019-09-06 22:12:25.994442366 +0000
++++ b/bazel/python_rules.bzl	2019-09-06 22:13:05.638367699 +0000
+@@ -175,7 +175,7 @@
+         srcs = [
+             ":{}".format(codegen_grpc_target),
+         ],
+-        deps = [Label("//src/python/grpcio/grpc:grpcio")] + deps,
++        #deps = [Label("//src/python/grpcio/grpc:grpcio")] + deps,
+         **kwargs
+     )
+