diff --git a/.travis.yml b/.travis.yml index 8ca8249c0..703f64934 100644 --- a/.travis.yml +++ b/.travis.yml @@ -71,28 +71,28 @@ jobs: - stage: build name: "Nightly Release Build on macOS 2.7.13" os: osx - osx_image: xcode9 + osx_image: xcode9.3 script: - bash -x -e .travis/python.release.sh --nightly ${TRAVIS_BUILD_NUMBER} - - bash -x -e .travis/wheel.test.sh + - echo bash -x -e .travis/wheel.test.sh after_success: bash -x -e .travis/after-success.sh - stage: build name: "Nightly Release Build on macOS 3.5.3" os: osx - osx_image: xcode9 + osx_image: xcode9.3 script: - bash -x -e .travis/python.release.sh --nightly ${TRAVIS_BUILD_NUMBER} - - bash -x -e .travis/wheel.test.sh + - echo bash -x -e .travis/wheel.test.sh after_success: bash -x -e .travis/after-success.sh - stage: build name: "Nightly Release Build on macOS 3.6.2" os: osx - osx_image: xcode9 + osx_image: xcode9.3 script: - bash -x -e .travis/python.release.sh --nightly ${TRAVIS_BUILD_NUMBER} - - bash -x -e .travis/wheel.test.sh + - echo bash -x -e .travis/wheel.test.sh after_success: bash -x -e .travis/after-success.sh notifications: diff --git a/.travis/bazel.build.sh b/.travis/bazel.build.sh index b6f884f11..bb58ebc81 100755 --- a/.travis/bazel.build.sh +++ b/.travis/bazel.build.sh @@ -20,7 +20,7 @@ if [[ "$#" -gt 0 ]]; then export TENSORFLOW_INSTALL="${1}" fi -export BAZEL_VERSION=0.24.1 BAZEL_OS=$(uname | tr '[:upper:]' '[:lower:]') +export BAZEL_VERSION=0.29.0 BAZEL_OS=$(uname | tr '[:upper:]' '[:lower:]') curl -sSOL https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-${BAZEL_OS}-x86_64.sh bash -e bazel-${BAZEL_VERSION}-installer-${BAZEL_OS}-x86_64.sh bazel version @@ -30,9 +30,9 @@ if [[ $(uname) == "Linux" ]]; then python get-pip.py -q python -m pip --version fi -python -m pip install -q ${TENSORFLOW_INSTALL} -python -m pip install -q grpcio-tools +python -m pip install -q h5py==2.9.0 +python -m pip install -q ${TENSORFLOW_INSTALL} python third_party/tf/configure.py @@ -57,7 +57,7 @@ bazel build \ --noshow_loading_progress \ --verbose_failures \ --test_output=errors \ - -- //tensorflow_io/... + -- //tensorflow_io/core:all rm -rf build && mkdir -p build diff --git a/WORKSPACE b/WORKSPACE index 5411c2fc4..813fc4079 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -2,6 +2,7 @@ workspace(name = "org_tensorflow_io") load("//third_party/tf:tf_configure.bzl", "tf_configure") load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") tf_configure( name = "local_config_tf", @@ -235,35 +236,48 @@ http_archive( http_archive( name = "com_github_grpc_grpc", - sha256 = "11ac793c562143d52fd440f6549588712badc79211cdc8c509b183cb69bddad8", - strip_prefix = "grpc-1.22.0", - urls = [ - "https://mirror.bazel.build/github.com/grpc/grpc/archive/v1.22.0.tar.gz", - "https://github.com/grpc/grpc/archive/v1.22.0.tar.gz", + patch_args = ["-p1"], + patches = [ + "//third_party:grpc.patch", ], -) - -# 3.8.0 with a fix to BUILD file -http_archive( - name = "com_google_protobuf", - sha256 = "b9e92f9af8819bbbc514e2902aec860415b70209f31dfc8c4fa72515a5df9d59", - strip_prefix = "protobuf-310ba5ee72661c081129eb878c1bbcec936b20f0", + sha256 = "6dc4f122527670099124a71d8a180b0b074a18efa939173d6c3a0673229f57d3", + strip_prefix = "grpc-e68ce1164b49529de12fbba63d53f081aef5c90e", urls = [ - "http://mirror.tensorflow.org/github.com/protocolbuffers/protobuf/archive/310ba5ee72661c081129eb878c1bbcec936b20f0.tar.gz", - "https://github.com/protocolbuffers/protobuf/archive/310ba5ee72661c081129eb878c1bbcec936b20f0.tar.gz", + "https://github.com/grpc/grpc/archive/e68ce1164b49529de12fbba63d53f081aef5c90e.tar.gz", ], ) -http_archive( - name = "bazel_skylib", - sha256 = "2ef429f5d7ce7111263289644d233707dba35e39696377ebab8b0bc701f7818e", - urls = ["https://github.com/bazelbuild/bazel-skylib/releases/download/0.8.0/bazel-skylib.0.8.0.tar.gz"], -) - load("@com_github_grpc_grpc//bazel:grpc_deps.bzl", "grpc_deps") grpc_deps() +load("@io_bazel_rules_python//python:pip.bzl", "pip_import", "pip_repositories") + +pip_import( + name = "grpc_python_dependencies", + requirements = "@com_github_grpc_grpc//:requirements.bazel.txt", +) + +load("@io_bazel_rules_python//python:pip.bzl", "pip_repositories") +load("@grpc_python_dependencies//:requirements.bzl", "pip_install") + +pip_repositories() + +pip_install() + +# TODO(https://github.com/grpc/grpc/issues/19835): Remove. +load("@upb//bazel:workspace_deps.bzl", "upb_deps") + +upb_deps() + +load("@build_bazel_rules_apple//apple:repositories.bzl", "apple_rules_dependencies") + +apple_rules_dependencies() + +load("@build_bazel_apple_support//lib:repositories.bzl", "apple_support_dependencies") + +apple_support_dependencies() + http_archive( name = "giflib", build_file = "//third_party:giflib.BUILD", @@ -277,28 +291,21 @@ http_archive( http_archive( name = "com_github_googleapis_google_cloud_cpp", - sha256 = "3abe2cf553ce33ff58d23848ae716cd2fcabfd454b89f6f65a92ed261244c1df", - strip_prefix = "google-cloud-cpp-0.11.0", + sha256 = "35058ff14e4f9f49f78da2f1bbf1c03f27e8e40ec65c51f62720346e99803392", + strip_prefix = "google-cloud-cpp-0.13.0", urls = [ - "https://mirror.bazel.build/github.com/googleapis/google-cloud-cpp/archive/v0.11.0.tar.gz", - "https://github.com/googleapis/google-cloud-cpp/archive/v0.11.0.tar.gz", + "https://mirror.bazel.build/github.com/googleapis/google-cloud-cpp/archive/v0.13.0.tar.gz", + "https://github.com/googleapis/google-cloud-cpp/archive/v0.13.0.tar.gz", ], ) -# Manually load com_google_googleapis as we need a patch for pubsub -# The patch file was generated from: -# diff -Naur a b > third_party/googleapis.patch http_archive( name = "com_google_googleapis", build_file = "@com_github_googleapis_google_cloud_cpp//bazel:googleapis.BUILD", - patch_args = ["-p1"], - patches = [ - "//third_party:googleapis.patch", - ], - sha256 = "90bcdf27b41b1c3900838fe4edaf89080ca67026608817946b4ae5e2b925c711", - strip_prefix = "googleapis-7152063cb170d23c5c110e243711d8eb6fda6a1c", + sha256 = "cb531e445115e28054a33ad968c2d7d8ade4693721866ce1b9adf9a78762c032", + strip_prefix = "googleapis-960b76b1f0c46d12610088977d1129cc7405f3dc", urls = [ - "https://github.com/googleapis/googleapis/archive/7152063cb170d23c5c110e243711d8eb6fda6a1c.tar.gz", + "https://github.com/googleapis/googleapis/archive/960b76b1f0c46d12610088977d1129cc7405f3dc.tar.gz", ], ) diff --git a/configure.sh b/configure.sh index 0245ec478..9e9e0bf9a 100755 --- a/configure.sh +++ b/configure.sh @@ -20,5 +20,5 @@ if python -c "import tensorflow as tf" &> /dev/null; then else python -m pip install $(python setup.py --package-version) fi -python -m pip install grpcio-tools +python -m pip install h5py==2.9.0 python third_party/tf/configure.py diff --git a/tensorflow_io/grpc/BUILD b/tensorflow_io/grpc/BUILD index c3a34da72..ca3d20b6f 100644 --- a/tensorflow_io/grpc/BUILD +++ b/tensorflow_io/grpc/BUILD @@ -3,43 +3,28 @@ licenses(["notice"]) # Apache 2.0 package(default_visibility = ["//visibility:public"]) load("@com_github_grpc_grpc//bazel:cc_grpc_library.bzl", "cc_grpc_library") +load("@com_github_grpc_grpc//bazel:python_rules.bzl", "py_grpc_library", "py_proto_library") load( "//third_party/tf:tensorflow_io.bzl", "tf_io_copts", ) -genrule( - name = "endpoint_py", - srcs = [ - "endpoint.proto", - ], - outs = [ - "python/ops/__init__.py", - "python/ops/endpoint_pb2.py", - "python/ops/endpoint_pb2_grpc.py", - ], - cmd = "python -m grpc_tools.protoc -Itensorflow_io/grpc --python_out=$(BINDIR)/tensorflow_io/grpc/python/ops/ --grpc_python_out=$(BINDIR)/tensorflow_io/grpc/python/ops/ $< ; touch $(BINDIR)/tensorflow_io/grpc/python/ops/__init__.py", - output_to_bindir = True, -) - proto_library( - name = "_any_proto_only", + name = "endpoint_proto", + srcs = ["endpoint.proto"], deps = ["@com_google_protobuf//:any_proto"], ) cc_proto_library( - name = "any_proto", - deps = ["@com_google_protobuf//:any_proto"], + name = "endpoint_cc_proto", + deps = [":endpoint_proto"], ) cc_grpc_library( - name = "endpoint_cc", - srcs = [ - "endpoint.proto", - ], - proto_only = False, - well_known_protos = True, - deps = [":any_proto"], + name = "endpoint_cc_grpc", + srcs = [":endpoint_proto"], + grpc_only = True, + deps = [":endpoint_cc_proto"], ) cc_binary( @@ -54,12 +39,22 @@ cc_binary( ], linkshared = 1, deps = [ - ":endpoint_cc", + ":endpoint_cc_grpc", "//tensorflow_io/core:dataset_ops", "@com_github_grpc_grpc//:grpc++", - "@com_google_protobuf//:protobuf", "@libarchive", "@local_config_tf//:libtensorflow_framework", "@local_config_tf//:tf_header_lib", ], ) + +py_proto_library( + name = "endpoint_py_pb2", + deps = [":endpoint_proto"], +) + +py_grpc_library( + name = "endpoint_py_pb2_grpc", + srcs = [":endpoint_proto"], + deps = [":endpoint_py_pb2"], +) diff --git a/tensorflow_io/grpc/python/ops/grpc_endpoint.py b/tensorflow_io/grpc/python/ops/grpc_endpoint.py index c4e76ea0d..885f09df7 100644 --- a/tensorflow_io/grpc/python/ops/grpc_endpoint.py +++ b/tensorflow_io/grpc/python/ops/grpc_endpoint.py @@ -24,15 +24,31 @@ import google.protobuf.any_pb2 import tensorflow as tf -# Incase test is done with TFIO_DATAPATH specified, the -# import path need to be extended to capture generated -# grpc files: + +# In case test is done with TFIO_DATAPATH specified, the +# import path need to be extended to capture generated grpc files: +grpcpath = os.path.join(os.path.dirname(__file__), "..", "..") datapath = os.environ.get('TFIO_DATAPATH') -sys.path.append(os.path.abspath( - os.path.dirname(__file__) if datapath is None else os.path.join( - datapath, "tensorflow_io", "grpc", "python", "ops"))) -import endpoint_pb2 # pylint: disable=wrong-import-position,unused-import -import endpoint_pb2_grpc # pylint: disable=wrong-import-position,unused-import +if datapath is not None: + grpcpath = os.path.join(datapath, "tensorflow_io", "grpc") +grpcpath = os.path.abspath(grpcpath) +sys.path.append(grpcpath) + +import endpoint_pb2 # pylint: disable=wrong-import-position +# For some reason generated code in grpc uses: +# from tensorflow_io.grpc import endpoint_pb2 +# so below is needed +class MetaPathFinder(object): + def find_module(self, fullname, _): + if fullname == "tensorflow_io.grpc.endpoint_pb2": + return self + return None + def load_module(self, fullname): + if fullname == "tensorflow_io.grpc.endpoint_pb2": + return endpoint_pb2 + return None +sys.meta_path.append(MetaPathFinder()) +import endpoint_pb2_grpc # pylint: disable=wrong-import-position class GRPCEndpoint(endpoint_pb2_grpc.GRPCEndpointServicer): """GRPCEndpoint""" diff --git a/third_party/googleapis.patch b/third_party/googleapis.patch deleted file mode 100644 index ba236c0e6..000000000 --- a/third_party/googleapis.patch +++ /dev/null @@ -1,73 +0,0 @@ -diff -Naur a/google/pubsub/v1/BUILD.bazel b/google/pubsub/v1/BUILD.bazel ---- a/google/pubsub/v1/BUILD.bazel 2019-07-01 15:09:37.348895248 -0700 -+++ b/google/pubsub/v1/BUILD.bazel 2019-07-01 15:12:29.851407500 -0700 -@@ -143,3 +143,24 @@ - ":pubsub_go_proto", - ], - ) -+ -+############################################################################## -+# C++ -+############################################################################## -+load( -+ "@com_google_googleapis_imports//:imports.bzl", -+ "cc_grpc_library", -+ "cc_proto_library", -+) -+ -+cc_proto_library( -+ name = "pubsub_cc_proto", -+ deps = [":pubsub_proto"], -+) -+ -+cc_grpc_library( -+ name = "pubsub_cc_grpc", -+ srcs = [":pubsub_proto"], -+ deps = [":pubsub_cc_proto"], -+ grpc_only = True, -+) - -diff --git a/google/cloud/bigquery/storage/v1beta1/BUILD.bazel b/google/cloud/bigquery/storage/v1beta1/BUILD.bazel -index 1772fa579..cd46aa449 100644 ---- a/google/cloud/bigquery/storage/v1beta1/BUILD.bazel -+++ b/google/cloud/bigquery/storage/v1beta1/BUILD.bazel -@@ -10,11 +10,13 @@ proto_library( - name = "bigquerystorage_proto", - srcs = [ - "avro.proto", -+ "arrow.proto", - "read_options.proto", - "storage.proto", - "table_reference.proto", - ], - deps = [ -+ "//google/api:client_proto", - "//google/api:annotations_proto", - "//google/api:resource_proto", - "@com_google_protobuf//:empty_proto", -@@ -146,3 +148,24 @@ go_gapic_assembly_pkg( - ":bigquerystorage_go_proto", - ], - ) -+ -+############################################################################## -+# C++ -+############################################################################## -+load( -+ "@com_google_googleapis_imports//:imports.bzl", -+ "cc_grpc_library", -+ "cc_proto_library", -+) -+ -+cc_proto_library( -+ name = "bigquerystorage_cc_proto", -+ deps = [":bigquerystorage_proto"], -+) -+ -+cc_grpc_library( -+ name = "bigquerystorage_cc_grpc", -+ srcs = [":bigquerystorage_proto"], -+ deps = [":bigquerystorage_cc_proto"], -+ grpc_only = True, -+) - diff --git a/third_party/grpc.patch b/third_party/grpc.patch new file mode 100644 index 000000000..58cf4074c --- /dev/null +++ b/third_party/grpc.patch @@ -0,0 +1,12 @@ +diff -Naur a/bazel/python_rules.bzl b/bazel/python_rules.bzl +--- a/bazel/python_rules.bzl 2019-09-06 22:12:25.994442366 +0000 ++++ b/bazel/python_rules.bzl 2019-09-06 22:13:05.638367699 +0000 +@@ -175,7 +175,7 @@ + srcs = [ + ":{}".format(codegen_grpc_target), + ], +- deps = [Label("//src/python/grpcio/grpc:grpcio")] + deps, ++ #deps = [Label("//src/python/grpcio/grpc:grpcio")] + deps, + **kwargs + ) + diff --git a/third_party/nucleus.patch b/third_party/nucleus.patch index a9825cfd0..6b8034f92 100644 --- a/third_party/nucleus.patch +++ b/third_party/nucleus.patch @@ -1,6 +1,6 @@ diff -Naur a/nucleus/protos/BUILD b/nucleus/protos/BUILD ---- a/nucleus/protos/BUILD 2019-09-04 16:59:20.127090030 +0000 -+++ b/nucleus/protos/BUILD 2019-09-04 20:47:28.198750222 +0000 +--- a/nucleus/protos/BUILD 2019-09-07 15:31:09.625067415 +0000 ++++ b/nucleus/protos/BUILD 2019-09-07 15:32:35.368864263 +0000 @@ -1,62 +1,6 @@ -load("//:third_party/clif.bzl", "py_clif_cc", "pyclif_proto_library") -load("@protobuf_archive//:protobuf.bzl", "py_proto_library") @@ -466,3 +466,12 @@ diff -Naur a/nucleus/protos/BUILD b/nucleus/protos/BUILD + visibility = ["//visibility:public"], + deps = [":fastq_proto"], ) +diff -Naur a/nucleus/vendor/BUILD b/nucleus/vendor/BUILD +--- a/nucleus/vendor/BUILD 2019-09-07 15:31:09.629067403 +0000 ++++ b/nucleus/vendor/BUILD 2019-09-07 15:38:30.184079565 +0000 +@@ -1,4 +1,4 @@ +-load("//:third_party/clif.bzl", "py_clif_cc") ++#load("//:third_party/clif.bzl", "py_clif_cc") + + package( + default_visibility = [ diff --git a/tools/dev/Dockerfile b/tools/dev/Dockerfile index 84e0305f2..077ef8827 100644 --- a/tools/dev/Dockerfile +++ b/tools/dev/Dockerfile @@ -14,7 +14,7 @@ RUN apt-get update && \ ffmpeg \ dnsutils -ARG BAZEL_VERSION=0.24.1 +ARG BAZEL_VERSION=0.29.0 ARG BAZEL_OS=linux RUN curl -sL https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-${BAZEL_OS}-x86_64.sh -o bazel-install.sh && \