diff --git a/.github/workflows/test_upb.yml b/.github/workflows/test_upb.yml index 21ec30887658e..f2c1cfa2e4d71 100644 --- a/.github/workflows/test_upb.yml +++ b/.github/workflows/test_upb.yml @@ -76,7 +76,7 @@ jobs: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} bazel-cache: "upb-bazel-windows" bazel: test --cxxopt=/std:c++17 --host_cxxopt=/std:c++17 //upb/... //upb_generator/... //python/... //protos/... //protos_generator/... - exclude-targets: -//python:conformance_test -//upb:def_builder_test + exclude-targets: -//python:conformance_test -//upb/reflection:def_builder_test macos: strategy: diff --git a/python/dist/BUILD.bazel b/python/dist/BUILD.bazel index 4330fe1c4f640..ba549acb251a5 100644 --- a/python/dist/BUILD.bazel +++ b/python/dist/BUILD.bazel @@ -246,6 +246,7 @@ pkg_files( "//upb/mini_descriptor:source_files", "//upb/mini_table:source_files", "//upb/port:source_files", + "//upb/reflection:source_files", "//upb/text:source_files", "//upb/util:source_files", "//upb/wire:source_files", diff --git a/python/google/protobuf/internal/api_implementation.py b/python/google/protobuf/internal/api_implementation.py index 65caf63f8b558..09af96e67c614 100755 --- a/python/google/protobuf/internal/api_implementation.py +++ b/python/google/protobuf/internal/api_implementation.py @@ -13,6 +13,8 @@ import sys import warnings +_GOOGLE3_PYTHON_UPB_DEFAULT = False + def _ApiVersionToImplementationType(api_version): if api_version == 2: diff --git a/src/google/protobuf/extension_set.cc b/src/google/protobuf/extension_set.cc index 4b54d7a3b6fed..7fde2cf8de0b1 100644 --- a/src/google/protobuf/extension_set.cc +++ b/src/google/protobuf/extension_set.cc @@ -11,6 +11,7 @@ #include "google/protobuf/extension_set.h" +#include #include #include #include @@ -1933,6 +1934,8 @@ LazyEagerVerifyFnType FindExtensionLazyEagerVerifyFn( return nullptr; } +std::atomic + ExtensionSet::maybe_create_lazy_extension_; } // namespace internal } // namespace protobuf diff --git a/src/google/protobuf/extension_set.h b/src/google/protobuf/extension_set.h index f45a1c858ca25..dd4a473ebc718 100644 --- a/src/google/protobuf/extension_set.h +++ b/src/google/protobuf/extension_set.h @@ -16,6 +16,7 @@ #define GOOGLE_PROTOBUF_EXTENSION_SET_H__ #include +#include #include #include #include @@ -57,6 +58,7 @@ class FeatureSet; namespace internal { class FieldSkipper; // wire_format_lite.h class WireFormat; +void InitializeLazyExtensionSet(); } // namespace internal } // namespace protobuf } // namespace google @@ -508,6 +510,8 @@ class PROTOBUF_EXPORT ExtensionSet { friend class google::protobuf::Reflection; friend class google::protobuf::internal::WireFormat; + friend void internal::InitializeLazyExtensionSet(); + const int32_t& GetRefInt32(int number, const int32_t& default_value) const; const int64_t& GetRefInt64(int number, const int64_t& default_value) const; const uint32_t& GetRefUInt32(int number, const uint32_t& default_value) const; @@ -579,7 +583,13 @@ class PROTOBUF_EXPORT ExtensionSet { virtual void UnusedKeyMethod(); // Dummy key method to avoid weak vtable. }; // Give access to function defined below to see LazyMessageExtension. - friend LazyMessageExtension* MaybeCreateLazyExtension(Arena* arena); + static LazyMessageExtension* MaybeCreateLazyExtensionImpl(Arena* arena); + static LazyMessageExtension* MaybeCreateLazyExtension(Arena* arena) { + auto* f = maybe_create_lazy_extension_.load(std::memory_order_relaxed); + return f != nullptr ? f(arena) : nullptr; + } + static std::atomic + maybe_create_lazy_extension_; struct Extension { // The order of these fields packs Extension into 24 bytes when using 8 // byte alignment. Consider this when adding or removing fields here. @@ -1529,10 +1539,6 @@ class ExtensionIdentifier { // Generated accessors -// Used to retrieve a lazy extension, may return nullptr in some environments. -extern PROTOBUF_ATTRIBUTE_WEAK ExtensionSet::LazyMessageExtension* -MaybeCreateLazyExtension(Arena* arena); - // Define a specialization of ExtensionIdentifier for bootstrapped extensions // that we need to register lazily. template <> diff --git a/src/google/protobuf/generated_message_reflection.cc b/src/google/protobuf/generated_message_reflection.cc index 452fd5240d17a..84272163fece5 100644 --- a/src/google/protobuf/generated_message_reflection.cc +++ b/src/google/protobuf/generated_message_reflection.cc @@ -16,6 +16,7 @@ #include #include #include +#include #include "absl/base/call_once.h" #include "absl/base/casts.h" @@ -93,6 +94,9 @@ void InitializeFileDescriptorDefaultInstances() { #endif // !defined(PROTOBUF_CONSTINIT_DEFAULT_INSTANCES) } +void InitializeLazyExtensionSet() { +} + bool ParseNamedEnum(const EnumDescriptor* descriptor, absl::string_view name, int* value) { const EnumValueDescriptor* d = descriptor->FindValueByName(name); @@ -3660,6 +3664,7 @@ void AddDescriptorsImpl(const DescriptorTable* table) { // Reflection refers to the default fields so make sure they are initialized. internal::InitProtobufDefaults(); internal::InitializeFileDescriptorDefaultInstances(); + internal::InitializeLazyExtensionSet(); // Ensure all dependent descriptors are registered to the generated descriptor // pool and message factory. diff --git a/upb/BUILD b/upb/BUILD index 02d65bdee245c..1e52a8092b14a 100644 --- a/upb/BUILD +++ b/upb/BUILD @@ -8,14 +8,7 @@ load("@rules_python//python:defs.bzl", "py_binary") load("@bazel_skylib//rules:common_settings.bzl", "bool_flag") load("//bazel:build_defs.bzl", "UPB_DEFAULT_COPTS") -load("//bazel:upb_minitable_proto_library.bzl", "upb_minitable_proto_library") -load("//bazel:upb_proto_library.bzl", "upb_proto_reflection_library") load("//bazel:upb_proto_library_internal/copts.bzl", "upb_proto_library_copts") -load( - "//upb_generator:bootstrap_compiler.bzl", - "bootstrap_cc_library", - "bootstrap_upb_proto_library", -) # begin:google_only # load("//tools/build_defs/kotlin/native:rules.bzl", "kt_native_interop_hint") @@ -104,14 +97,14 @@ cc_library( # implementation, but depends on :upb and exposes a few more hdrs. # # This is public only because we have no way of visibility-limiting it to -# upb_proto_library() only. This interface is not stable and by using it you +# upb_c_proto_library() only. This interface is not stable and by using it you # give up any backward compatibility guarantees. cc_library( name = "generated_code_support__only_for_generated_code_do_not_use__i_give_permission_to_break_me", hdrs = ["generated_code_support.h"], copts = UPB_DEFAULT_COPTS, textual_hdrs = [ - "//upb/port:inc", + ":port_inc", ], visibility = ["//visibility:public"], deps = [ @@ -133,176 +126,54 @@ cc_library( name = "generated_cpp_support__only_for_generated_code_do_not_use__i_give_permission_to_break_me", copts = UPB_DEFAULT_COPTS, textual_hdrs = [ - "//upb/port:inc", + ":port_inc", ], visibility = ["//visibility:public"], ) -cc_library( - name = "generated_reflection_support__only_for_generated_code_do_not_use__i_give_permission_to_break_me", - hdrs = [ - "reflection/def.h", - "reflection/internal/def_pool.h", - ], - copts = UPB_DEFAULT_COPTS, - textual_hdrs = [ - "//upb/port:inc", - ], - visibility = ["//visibility:public"], - deps = [ - ":mem", - ":mini_descriptor", - ":reflection_internal", - ], -) +# Aliases ###################################################################### +# TODO: Remove these. -bootstrap_upb_proto_library( - name = "descriptor_upb_proto", - base_dir = "reflection/", - # TODO: Export 'net/proto2/proto/descriptor.upb.h' and remove "-layering_check". - features = ["-layering_check"], - google3_src_files = ["net/proto2/proto/descriptor.proto"], - google3_src_rules = ["//net/proto2/proto:descriptor_proto_source"], - oss_src_files = ["google/protobuf/descriptor.proto"], - oss_src_rules = ["//:descriptor_proto_srcs"], - oss_strip_prefix = "third_party/protobuf/github/bootstrap/src", - proto_lib_deps = ["//:descriptor_proto"], +alias( + name = "base", + actual = "//upb/base", visibility = ["//visibility:public"], ) -upb_proto_reflection_library( - name = "descriptor_upb_proto_reflection", +alias( + name = "base_internal", + actual = "//upb/base:internal", visibility = ["//visibility:public"], - deps = ["//:descriptor_proto"], ) -upb_minitable_proto_library( - name = "descriptor_upb_minitable_proto", - visibility = [ - "//upb:__subpackages__", - # begin:github_only - "//python:__subpackages__", - # end:github_only - ], - deps = ["//:descriptor_proto"], -) - -# TODO: Once we can delete the deprecated forwarding headers -# (= everything in upb/) we can move this build target down into reflection/ -bootstrap_cc_library( - name = "reflection", - hdrs = [ - "reflection/def.h", - "reflection/def.hpp", - "reflection/message.h", - "reflection/message.hpp", - ], - bootstrap_deps = [":reflection_internal"], - copts = UPB_DEFAULT_COPTS, +alias( + name = "collections", + actual = "//upb/collections", + deprecation = "use upb:message instead", visibility = ["//visibility:public"], - deps = [ - ":base", - ":mem", - ":message", - ":message_types", - ":message_value", - ":port", - ], ) -bootstrap_cc_library( - name = "reflection_internal", - srcs = [ - "reflection/def_pool.c", - "reflection/def_type.c", - "reflection/desc_state.c", - "reflection/enum_def.c", - "reflection/enum_reserved_range.c", - "reflection/enum_value_def.c", - "reflection/extension_range.c", - "reflection/field_def.c", - "reflection/file_def.c", - "reflection/internal/def_builder.c", - "reflection/internal/def_builder.h", - "reflection/internal/strdup2.c", - "reflection/internal/strdup2.h", - "reflection/message.c", - "reflection/message_def.c", - "reflection/message_reserved_range.c", - "reflection/method_def.c", - "reflection/oneof_def.c", - "reflection/service_def.c", - ], - hdrs = [ - "reflection/common.h", - "reflection/def.h", - "reflection/def.hpp", - "reflection/def_pool.h", - "reflection/def_type.h", - "reflection/enum_def.h", - "reflection/enum_reserved_range.h", - "reflection/enum_value_def.h", - "reflection/extension_range.h", - "reflection/field_def.h", - "reflection/file_def.h", - "reflection/internal/def_pool.h", - "reflection/internal/desc_state.h", - "reflection/internal/enum_def.h", - "reflection/internal/enum_reserved_range.h", - "reflection/internal/enum_value_def.h", - "reflection/internal/extension_range.h", - "reflection/internal/field_def.h", - "reflection/internal/file_def.h", - "reflection/internal/message_def.h", - "reflection/internal/message_reserved_range.h", - "reflection/internal/method_def.h", - "reflection/internal/oneof_def.h", - "reflection/internal/service_def.h", - "reflection/message.h", - "reflection/message.hpp", - "reflection/message_def.h", - "reflection/message_reserved_range.h", - "reflection/method_def.h", - "reflection/oneof_def.h", - "reflection/service_def.h", - ], - bootstrap_deps = [":descriptor_upb_proto"], - copts = UPB_DEFAULT_COPTS, +alias( + name = "descriptor_upb_proto", + actual = "//upb/reflection:descriptor_upb_proto", visibility = ["//visibility:public"], - deps = [ - ":base", - ":hash", - ":mem", - ":message", - ":message_accessors", - ":message_types", - ":message_value", - ":mini_descriptor", - ":mini_descriptor_internal", - ":mini_table", - ":port", - ], ) -# Aliases ###################################################################### -# TODO: Remove these. - alias( - name = "base", - actual = "//upb/base", + name = "descriptor_upb_minitable_proto", + actual = "//upb/reflection:descriptor_upb_minitable_proto", visibility = ["//visibility:public"], ) alias( - name = "base_internal", - actual = "//upb/base:internal", + name = "descriptor_upb_proto_reflection", + actual = "//upb/reflection:descriptor_upb_proto_reflection", visibility = ["//visibility:public"], ) alias( - name = "collections", - actual = "//upb/collections", - deprecation = "use upb:message instead", + name = "generated_reflection_support__only_for_generated_code_do_not_use__i_give_permission_to_break_me", + actual = "//upb/reflection:generated_reflection_support__only_for_generated_code_do_not_use__i_give_permission_to_break_me", visibility = ["//visibility:public"], ) @@ -438,6 +309,24 @@ alias( visibility = ["//visibility:public"], ) +alias( + name = "port_inc", + actual = "//upb/port:inc", + visibility = ["//visibility:public"], +) + +alias( + name = "reflection", + actual = "//upb/reflection", + visibility = ["//visibility:public"], +) + +alias( + name = "reflection_internal", + actual = "//upb/reflection:internal", + visibility = ["//visibility:public"], +) + alias( name = "text", actual = "//upb/text", @@ -474,28 +363,6 @@ alias( visibility = ["//visibility:public"], ) -# Tests ######################################################################## - -cc_test( - name = "def_builder_test", - srcs = [ - "reflection/common.h", - "reflection/def_type.h", - "reflection/internal/def_builder.h", - "reflection/internal/def_builder_test.cc", - ], - deps = [ - ":descriptor_upb_proto", - ":hash", - ":mem", - ":port", - ":reflection", - ":reflection_internal", - "@com_google_googletest//:gtest_main", - "@com_google_absl//absl/strings", - ], -) - # Internal C/C++ libraries ##################################################### cc_binary( diff --git a/upb/cmake/BUILD.bazel b/upb/cmake/BUILD.bazel index b2a5e5a7f8abd..3e40380e8bfbe 100644 --- a/upb/cmake/BUILD.bazel +++ b/upb/cmake/BUILD.bazel @@ -96,6 +96,7 @@ sh_test( "//upb/mini_descriptor:source_files", "//upb/mini_table:source_files", "//upb/port:source_files", + "//upb/reflection:source_files", "//upb/text:source_files", "//upb/wire:source_files", "@utf8_range//:utf8_range_srcs", diff --git a/upb/reflection/BUILD b/upb/reflection/BUILD new file mode 100644 index 0000000000000..0e92276c444bf --- /dev/null +++ b/upb/reflection/BUILD @@ -0,0 +1,192 @@ +# Copyright (c) 2009-2021, Google LLC +# All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +load("//bazel:build_defs.bzl", "UPB_DEFAULT_COPTS") +load("//bazel:upb_minitable_proto_library.bzl", "upb_minitable_proto_library") +load("//bazel:upb_proto_library.bzl", "upb_proto_reflection_library") +load( + "//upb_generator:bootstrap_compiler.bzl", + "bootstrap_cc_library", + "bootstrap_upb_proto_library", +) + +bootstrap_upb_proto_library( + name = "descriptor_upb_proto", + base_dir = "", + # TODO: Export 'net/proto2/proto/descriptor.upb.h' and remove "-layering_check". + features = ["-layering_check"], + google3_src_files = ["net/proto2/proto/descriptor.proto"], + google3_src_rules = ["//net/proto2/proto:descriptor_proto_source"], + oss_src_files = ["google/protobuf/descriptor.proto"], + oss_src_rules = ["//:descriptor_proto_srcs"], + oss_strip_prefix = "third_party/protobuf/github/bootstrap/src", + proto_lib_deps = ["//:descriptor_proto"], + visibility = ["//visibility:public"], +) + +upb_proto_reflection_library( + name = "descriptor_upb_proto_reflection", + visibility = ["//visibility:public"], + deps = ["//:descriptor_proto"], +) + +upb_minitable_proto_library( + name = "descriptor_upb_minitable_proto", + visibility = ["//visibility:public"], + deps = ["//:descriptor_proto"], +) + +bootstrap_cc_library( + name = "reflection", + hdrs = [ + "def.h", + "def.hpp", + "message.h", + "message.hpp", + ], + bootstrap_deps = [":internal"], + copts = UPB_DEFAULT_COPTS, + visibility = ["//visibility:public"], + deps = [ + "//upb:base", + "//upb:mem", + "//upb:message", + "//upb:message_types", + "//upb:message_value", + "//upb:port", + ], +) + +bootstrap_cc_library( + name = "internal", + srcs = [ + "def_pool.c", + "def_type.c", + "desc_state.c", + "enum_def.c", + "enum_reserved_range.c", + "enum_value_def.c", + "extension_range.c", + "field_def.c", + "file_def.c", + "internal/def_builder.c", + "internal/def_builder.h", + "internal/strdup2.c", + "internal/strdup2.h", + "message.c", + "message_def.c", + "message_reserved_range.c", + "method_def.c", + "oneof_def.c", + "service_def.c", + ], + hdrs = [ + "common.h", + "def.h", + "def.hpp", + "def_pool.h", + "def_type.h", + "enum_def.h", + "enum_reserved_range.h", + "enum_value_def.h", + "extension_range.h", + "field_def.h", + "file_def.h", + "internal/def_pool.h", + "internal/desc_state.h", + "internal/enum_def.h", + "internal/enum_reserved_range.h", + "internal/enum_value_def.h", + "internal/extension_range.h", + "internal/field_def.h", + "internal/file_def.h", + "internal/message_def.h", + "internal/message_reserved_range.h", + "internal/method_def.h", + "internal/oneof_def.h", + "internal/service_def.h", + "message.h", + "message.hpp", + "message_def.h", + "message_reserved_range.h", + "method_def.h", + "oneof_def.h", + "service_def.h", + ], + bootstrap_deps = [":descriptor_upb_proto"], + copts = UPB_DEFAULT_COPTS, + visibility = ["//visibility:public"], + deps = [ + "//upb:base", + "//upb:hash", + "//upb:mem", + "//upb:message", + "//upb:message_accessors", + "//upb:message_types", + "//upb:message_value", + "//upb:mini_descriptor", + "//upb:mini_descriptor_internal", + "//upb:mini_table", + "//upb:port", + ], +) + +cc_library( + name = "generated_reflection_support__only_for_generated_code_do_not_use__i_give_permission_to_break_me", + hdrs = [ + "def.h", + "internal/def_pool.h", + ], + copts = UPB_DEFAULT_COPTS, + textual_hdrs = [ + "//upb:port_inc", + ], + visibility = ["//visibility:public"], + deps = [ + "//upb:mem", + "//upb:mini_descriptor", + "//upb:reflection_internal", + ], +) + +cc_test( + name = "def_builder_test", + srcs = [ + "common.h", + "def_type.h", + "internal/def_builder.h", + "internal/def_builder_test.cc", + ], + deps = [ + ":descriptor_upb_proto", + "@com_google_googletest//:gtest_main", + "//upb:hash", + "//upb:mem", + "//upb:port", + "//upb:reflection", + "//upb:reflection_internal", + "@com_google_absl//absl/strings", + ], +) + +# begin:github_only +filegroup( + name = "source_files", + srcs = glob( + [ + "**/*.c", + "**/*.h", + "**/*.hpp", + ], + exclude = ["stage0/**"], + ), + visibility = [ + "//upb/cmake:__pkg__", + "//python/dist:__pkg__", + ] +) +# end:github_only diff --git a/upb_generator/BUILD b/upb_generator/BUILD index 79f2ac36493ea..723d490402c49 100644 --- a/upb_generator/BUILD +++ b/upb_generator/BUILD @@ -79,7 +79,7 @@ bootstrap_upb_proto_library( oss_strip_prefix = "third_party/protobuf/github/bootstrap/src", proto_lib_deps = ["//:compiler_plugin_proto"], visibility = ["//upb:friends"], - deps = ["//upb:descriptor_upb_proto"], + deps = ["//upb/reflection:descriptor_upb_proto"], ) upb_proto_reflection_library( @@ -97,7 +97,7 @@ bootstrap_cc_library( "common.h", ], bootstrap_deps = [ - "//upb:reflection", + "//upb/reflection:reflection", ], copts = UPB_DEFAULT_CPPOPTS, visibility = ["//protos_generator:__pkg__"], @@ -119,8 +119,8 @@ bootstrap_cc_library( ], bootstrap_deps = [ ":common", - "//upb:reflection", - "//upb:descriptor_upb_proto", + "//upb/reflection:reflection", + "//upb/reflection:descriptor_upb_proto", ], copts = UPB_DEFAULT_CPPOPTS, visibility = ["//visibility:public"], @@ -154,8 +154,8 @@ bootstrap_cc_library( ], bootstrap_deps = [ ":plugin_upb_proto", - "//upb:descriptor_upb_proto", - "//upb:reflection", + "//upb/reflection:descriptor_upb_proto", + "//upb/reflection:reflection", ], copts = UPB_DEFAULT_CPPOPTS, visibility = ["//protos_generator:__pkg__"], @@ -177,7 +177,7 @@ bootstrap_cc_library( "names.h", ], bootstrap_deps = [ - "//upb:reflection", + "//upb/reflection:reflection", ], copts = UPB_DEFAULT_CPPOPTS, visibility = ["//protos_generator:__pkg__"], @@ -244,8 +244,8 @@ bootstrap_cc_binary( ":names", ":plugin", ":plugin_upb_proto", - "//upb:descriptor_upb_proto", - "//upb:reflection", + "//upb/reflection:descriptor_upb_proto", + "//upb/reflection:reflection", ], copts = UPB_DEFAULT_CPPOPTS, visibility = ["//visibility:public"], @@ -283,8 +283,8 @@ bootstrap_cc_binary( ":names", ":plugin", ":plugin_upb_proto", - "//upb:descriptor_upb_proto", - "//upb:reflection", + "//upb/reflection:descriptor_upb_proto", + "//upb/reflection:reflection", ], copts = UPB_DEFAULT_CPPOPTS, visibility = ["//visibility:public"], diff --git a/upb_generator/protoc-gen-upbdefs.cc b/upb_generator/protoc-gen-upbdefs.cc index 69682c4e78bb8..58234150a2ce5 100644 --- a/upb_generator/protoc-gen-upbdefs.cc +++ b/upb_generator/protoc-gen-upbdefs.cc @@ -70,17 +70,13 @@ void WriteDefHeader(upb::FileDefPtr file, Output& output) { "#define $0_UPBDEFS_H_\n\n" "#include \"upb/reflection/def.h\"\n" "#include \"upb/reflection/internal/def_pool.h\"\n" - "#include \"upb/port/def.inc\"\n" + "\n" + "#include \"upb/port/def.inc\" // Must be last.\n" "#ifdef __cplusplus\n" "extern \"C\" {\n" "#endif\n\n", ToPreproc(file.name())); - output("#include \"upb/reflection/def.h\"\n"); - output("\n"); - output("#include \"upb/port/def.inc\"\n"); - output("\n"); - output("extern _upb_DefPool_Init $0;\n", DefInitSymbol(file)); output("\n");