diff --git a/WORKSPACE b/WORKSPACE index f03dae798a6..58a1edb471b 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -205,12 +205,15 @@ http_archive( url = "https://github.com/bazelbuild/rules_python/releases/download/0.1.0/rules_python-0.1.0.tar.gz", ) -load("@rules_python//python:repositories.bzl", "py_repositories") -py_repositories() -# Only needed if using the packaging rules. -# load("@rules_python//python:pip.bzl", "pip_repositories") -# pip_repositories() +load("@rules_python//python:pip.bzl", "pip_install") +# Create a central external repo, @heron_py_deps, that contains Bazel targets for all the +# third-party packages specified in the requirements.txt file. +pip_install( + name = "heron_py_deps", + requirements = "//tools/python:requirements.txt", +) + # for pex repos PEX_PKG = "https://files.pythonhosted.org/packages/d4/73/4c76e06824baadba81b39125721c97fb22e201b35fcd17b32b5a5fa77c59/pex-2.1.62-py2.py3-none-any.whl" @@ -515,3 +518,30 @@ load("@io_bazel_rules_scala//testing:scalatest.bzl", "scalatest_repositories", " scalatest_repositories() scalatest_toolchain() + +# Protocol buffers in Java and CC. +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +http_archive( + name = "rules_proto", + sha256 = "66bfdf8782796239d3875d37e7de19b1d94301e8972b3cbd2446b332429b4df1", + strip_prefix = "rules_proto-4.0.0", + urls = [ + "https://mirror.bazel.build/github.com/bazelbuild/rules_proto/archive/refs/tags/4.0.0.tar.gz", + "https://github.com/bazelbuild/rules_proto/archive/refs/tags/4.0.0.tar.gz", + ], +) +load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains") +rules_proto_dependencies() +rules_proto_toolchains() + +# Protocol buffer for Python. +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") + +git_repository( + name = "com_google_protobuf", + remote = "https://github.com/protocolbuffers/protobuf", + tag = "v3.10.0", +) +load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps") +protobuf_deps() diff --git a/examples/src/python/BUILD b/examples/src/python/BUILD index 9c0b77959bf..71d042d2c21 100644 --- a/examples/src/python/BUILD +++ b/examples/src/python/BUILD @@ -1,6 +1,9 @@ package(default_visibility = ["//visibility:public"]) -pex_library( +load("@rules_python//python:defs.bzl", "py_library") +load("@rules_python//python:defs.bzl", "py_binary") + +py_library( name = "example-py", srcs = glob([ "bolt/*.py", @@ -13,45 +16,45 @@ pex_library( ) # with main method -pex_binary( +py_binary( name = "word_count", srcs = ["word_count_topology.py"], - deps = [":example-py"], + deps = ["example-py"], ) -pex_binary( +py_binary( name = "half_acking", srcs = ["half_acking_topology.py"], - deps = [":example-py"], + deps = ["example-py"], ) -pex_binary( +py_binary( name = "window_size", srcs = ["window_size_topology.py"], - deps = [":example-py"], + deps = ["example-py"], ) -pex_binary( +py_binary( name = "stateful_word_count", srcs = ["stateful_word_count_topology.py"], - deps = [":example-py"], + deps = ["example-py"], ) # without main method -pex_binary( +py_binary( name = "custom_grouping", srcs = ["custom_grouping_topology.py"], - deps = [":example-py"], + deps = ["example-py"], ) -pex_binary( +py_binary( name = "multi_stream", srcs = ["multi_stream_topology.py"], - deps = [":example-py"], + deps = ["example-py"], ) # streamlet API examples -pex_binary( +py_binary( name = "word_count_streamlet", srcs = ["word_count_streamlet.py"], deps = [ @@ -61,7 +64,7 @@ pex_binary( ], ) -pex_binary( +py_binary( name = "pulsar_word_count_streamlet", srcs = ["pulsar_word_count_streamlet.py"], deps = [ @@ -71,7 +74,7 @@ pex_binary( ], ) -pex_binary( +py_binary( name = "join_streamlet_topology", srcs = ["join_streamlet_topology.py"], deps = [ diff --git a/heron/common/tests/python/pex_loader/testdata/src/BUILD b/heron/common/tests/python/pex_loader/testdata/src/BUILD index 4c0ab78d04b..b33935c2dfc 100644 --- a/heron/common/tests/python/pex_loader/testdata/src/BUILD +++ b/heron/common/tests/python/pex_loader/testdata/src/BUILD @@ -1,6 +1,9 @@ package(default_visibility = ["//visibility:public"]) -pex_binary( +load("@rules_python//python:defs.bzl", "py_binary") + +py_binary( name = "sample_pex", + main = "sample.py", srcs = ["sample.py"], ) diff --git a/tools/python/requirements.txt b/tools/python/requirements.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tools/rules/newgenproto.bzl b/tools/rules/newgenproto.bzl deleted file mode 100644 index 348e9a0ef01..00000000000 --- a/tools/rules/newgenproto.bzl +++ /dev/null @@ -1,185 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -load("@rules_java//java:defs.bzl", "java_library") -load("@rules_cc//cc:defs.bzl", "cc_library") -load("pex_rules", "pex_library") - -standard_proto_path = "heron/proto" - -def _genproto_impl(ctx): - proto_src_deps = [src.proto_src for src in ctx.attr.deps] - inputs, outputs, arguments = [ctx.file.src] + proto_src_deps, [], ["--proto_path=."] - for src in proto_src_deps: - if src.path.startswith(standard_proto_path): - arguments.append("--proto_path=" + standard_proto_path) - break - - if ctx.attr.gen_cc: - outputs += [ctx.outputs.cc_hdr, ctx.outputs.cc_src] - arguments.append("--cpp_out=" + ctx.configuration.genfiles_dir.path) - - if ctx.attr.gen_java: - if ctx.outputs.java_src.path.endswith(".srcjar"): - srcjar = ctx.actions.declare_file(ctx.outputs.java_src.basename[:-6] + "jar") - else: - srcjar = ctx.outputs.java_src - outputs.append(srcjar) - arguments.append("--java_out=" + srcjar.path) - - if ctx.attr.gen_py: - outputs.append(ctx.outputs.py_src) - arguments.append("--python_out=" + ctx.configuration.genfiles_dir.path) - - ctx.actions.run( - mnemonic = "GenProto", - inputs = inputs, - outputs = outputs, - arguments = arguments + [ctx.file.src.path], - executable = ctx.executable._protoc, - ) - - # This is required because protoc only understands .jar extensions, but Bazel - # requires source JAR files end in .srcjar. - if ctx.attr.gen_java and srcjar != ctx.outputs.java_src: - ctx.actions.run( - mnemonic = "FixProtoSrcJar", - inputs = [srcjar], - outputs = [ctx.outputs.java_src], - arguments = [srcjar.path, ctx.outputs.java_src.path], - command = "cp $1 $2", - ) - - # Fixup the resulting outputs to keep the source-only .jar out of the result. - outputs.append(ctx.outputs.java_src) - outputs = [e for e in outputs if e != srcjar] - - return struct( - files = set(outputs), - proto_src = ctx.file.src, - ) - -_genproto_attrs = { - "src": attr.label( - allow_files = [".proto"], - allow_single_file = True, - ), - "deps": attr.label_list( - allow_files = False, - providers = ["proto_src"], - ), - "_protoc": attr.label( - default = Label("//third_party/protobuf:protoc"), - executable = True, - ), - "gen_cc": attr.bool(), - "gen_java": attr.bool(), - "gen_py": attr.bool(), -} - -def _genproto_outputs(attrs): - outputs = {} - if attrs.gen_cc: - outputs.update( - { - "cc_hdr": "%{src}.pb.h", - "cc_src": "%{src}.pb.cc", - }, - ) - if attrs.gen_java: - outputs.update( - { - "java_src": "%{src}.srcjar", - }, - ) - if attrs.gen_py: - outputs.update( - { - "py_src": "%{src}_pb2.py", - }, - ) - return outputs - -genproto = rule( - _genproto_impl, - attrs = _genproto_attrs, - output_to_genfiles = True, - outputs = _genproto_outputs, -) - -def proto_library( - name, - src = None, - deps = [], - visibility = None, - gen_java = False, - gen_cc = False, - gen_py = False): - if not src: - if name.endswith("_proto"): - src = name[:-6] + ".proto" - else: - src = name + ".proto" - - proto_pkg = genproto( - name = name, - src = src, - deps = deps, - gen_java = gen_java, - gen_cc = gen_cc, - gen_py = gen_py, - ) - - # TODO(shahms): These should probably not be separate libraries, but - # allowing upstream *_library and *_binary targets to depend on the - # proto_library() directly is a challenge. We'd also need a different - # workaround for the non-generated any.pb.{h,cc} from the upstream protocol - # buffer library. - if gen_java: - java_deps = ["@com_google_protobuf//:protobuf_java"] - for dep in deps: - java_deps.append(dep + "_java") - java_library( - name = name + "_java", - srcs = [proto_pkg.label()], - deps = java_deps, - visibility = visibility, - ) - - if gen_cc: - cc_deps = ["//third_party/protobuf:protobuf-cxx"] - for dep in deps: - cc_deps.append(dep + "_cc") - cc_library( - name = name + "_cc", - visibility = visibility, - hdrs = [proto_pkg.label()], - srcs = [proto_pkg.label()], - defines = ["GOOGLE_PROTOBUF_NO_RTTI"], - deps = cc_deps, - ) - - if gen_py: - py_deps = [] - for dep in deps: - py_deps.append(dep + "_py") - pex_library( - name = name + "_py", - visibility = visibility, - srcs = [proto_pkg.label()], - deps = py_deps, - )