From 2257232676e60f1d06f8c9545c88d1a73a3a8f54 Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Thu, 2 May 2024 10:09:15 -0700 Subject: [PATCH] Split bootstrapped java_features.proto to keep it from leaking out. This can cause ODR violations in downstream users who link against both the bootstrapped proto and transitive C++ gencode of java_features.proto. Once protoc is split up, we can turn the bootstrapped proto into a real cc_proto_library target and avoid this problem altogether. PiperOrigin-RevId: 630099889 --- src/google/protobuf/compiler/java/BUILD.bazel | 21 +++- src/google/protobuf/compiler/java/helpers.cc | 80 ------------ src/google/protobuf/compiler/java/helpers.h | 36 ------ .../compiler/java/immutable/BUILD.bazel | 1 + .../compiler/java/immutable/enum_field.cc | 1 + .../compiler/java/immutable/map_field.cc | 1 + .../compiler/java/immutable/string_field.cc | 1 + .../compiler/java/internal_helpers.cc | 114 ++++++++++++++++++ .../protobuf/compiler/java/internal_helpers.h | 69 +++++++++++ .../protobuf/compiler/java/lite/BUILD.bazel | 2 + .../protobuf/compiler/java/lite/enum_field.cc | 2 +- .../protobuf/compiler/java/lite/map_field.cc | 1 + .../compiler/java/lite/message_field.cc | 1 + .../compiler/java/lite/primitive_field.cc | 1 + .../compiler/java/lite/string_field.cc | 1 + 15 files changed, 213 insertions(+), 119 deletions(-) create mode 100644 src/google/protobuf/compiler/java/internal_helpers.cc create mode 100644 src/google/protobuf/compiler/java/internal_helpers.h diff --git a/src/google/protobuf/compiler/java/BUILD.bazel b/src/google/protobuf/compiler/java/BUILD.bazel index 915b3703096b..ad25fecb27a1 100644 --- a/src/google/protobuf/compiler/java/BUILD.bazel +++ b/src/google/protobuf/compiler/java/BUILD.bazel @@ -35,7 +35,6 @@ cc_library( hdrs = [ "context.h", "doc_comment.h", - "generator.h", "helpers.h", "name_resolver.h", "names.h", @@ -48,7 +47,6 @@ cc_library( "//src/google/protobuf/compiler/java:__subpackages__", ], deps = [ - ":java_features_bootstrap", "//src/google/protobuf", "//src/google/protobuf:port", "//src/google/protobuf/compiler:code_generator", @@ -64,6 +62,25 @@ cc_library( ], ) +cc_library( + name = "internal_helpers", + srcs = ["internal_helpers.cc"], + hdrs = [ + "generator.h", + "internal_helpers.h", + ], + strip_include_prefix = "/src", + visibility = ["//src/google/protobuf/compiler/java:__subpackages__"], + deps = [ + ":helpers", + ":java_features_bootstrap", + "//src/google/protobuf", + "//src/google/protobuf:port", + "//src/google/protobuf/compiler:code_generator", + "@com_google_absl//absl/log:absl_log", + ], +) + cc_library( name = "java_features_bootstrap", srcs = ["java_features.pb.cc"], diff --git a/src/google/protobuf/compiler/java/helpers.cc b/src/google/protobuf/compiler/java/helpers.cc index 826d9f834405..8deb2dcec302 100644 --- a/src/google/protobuf/compiler/java/helpers.cc +++ b/src/google/protobuf/compiler/java/helpers.cc @@ -878,86 +878,6 @@ void WriteUInt32ToUtf16CharSequence(uint32_t number, output->push_back(static_cast(number)); } -int GetExperimentalJavaFieldTypeForSingular(const FieldDescriptor* field) { - // j/c/g/protobuf/FieldType.java lists field types in a slightly different - // order from FieldDescriptor::Type so we can't do a simple cast. - // - // TODO: Make j/c/g/protobuf/FieldType.java follow the same order. - int result = field->type(); - if (result == FieldDescriptor::TYPE_GROUP) { - return 17; - } else if (result < FieldDescriptor::TYPE_GROUP) { - return result - 1; - } else { - return result - 2; - } -} - -int GetExperimentalJavaFieldTypeForRepeated(const FieldDescriptor* field) { - if (field->type() == FieldDescriptor::TYPE_GROUP) { - return 49; - } else { - return GetExperimentalJavaFieldTypeForSingular(field) + 18; - } -} - -int GetExperimentalJavaFieldTypeForPacked(const FieldDescriptor* field) { - int result = field->type(); - if (result < FieldDescriptor::TYPE_STRING) { - return result + 34; - } else if (result > FieldDescriptor::TYPE_BYTES) { - return result + 30; - } else { - ABSL_LOG(FATAL) << field->full_name() << " can't be packed."; - return 0; - } -} - -int GetExperimentalJavaFieldType(const FieldDescriptor* field) { - static const int kMapFieldType = 50; - static const int kOneofFieldTypeOffset = 51; - - static const int kRequiredBit = 0x100; - static const int kUtf8CheckBit = 0x200; - static const int kCheckInitialized = 0x400; - static const int kLegacyEnumIsClosedBit = 0x800; - static const int kHasHasBit = 0x1000; - int extra_bits = field->is_required() ? kRequiredBit : 0; - if (field->type() == FieldDescriptor::TYPE_STRING && CheckUtf8(field)) { - extra_bits |= kUtf8CheckBit; - } - if (field->is_required() || (GetJavaType(field) == JAVATYPE_MESSAGE && - HasRequiredFields(field->message_type()))) { - extra_bits |= kCheckInitialized; - } - if (HasHasbit(field)) { - extra_bits |= kHasHasBit; - } - if (GetJavaType(field) == JAVATYPE_ENUM && !SupportUnknownEnumValue(field)) { - extra_bits |= kLegacyEnumIsClosedBit; - } - - if (field->is_map()) { - if (!SupportUnknownEnumValue(MapValueField(field))) { - const FieldDescriptor* value = field->message_type()->map_value(); - if (GetJavaType(value) == JAVATYPE_ENUM) { - extra_bits |= kLegacyEnumIsClosedBit; - } - } - return kMapFieldType | extra_bits; - } else if (field->is_packed()) { - return GetExperimentalJavaFieldTypeForPacked(field) | extra_bits; - } else if (field->is_repeated()) { - return GetExperimentalJavaFieldTypeForRepeated(field) | extra_bits; - } else if (IsRealOneof(field)) { - return (GetExperimentalJavaFieldTypeForSingular(field) + - kOneofFieldTypeOffset) | - extra_bits; - } else { - return GetExperimentalJavaFieldTypeForSingular(field) | extra_bits; - } -} - // Escape a UTF-16 character to be embedded in a Java string. void EscapeUtf16ToString(uint16_t code, std::string* output) { if (code == '\t') { diff --git a/src/google/protobuf/compiler/java/helpers.h b/src/google/protobuf/compiler/java/helpers.h index ed27eae1d555..52b4017cf562 100644 --- a/src/google/protobuf/compiler/java/helpers.h +++ b/src/google/protobuf/compiler/java/helpers.h @@ -16,8 +16,6 @@ #include #include "absl/strings/string_view.h" -#include "google/protobuf/compiler/java/generator.h" -#include "google/protobuf/compiler/java/java_features.pb.h" #include "google/protobuf/compiler/java/names.h" #include "google/protobuf/compiler/java/options.h" #include "google/protobuf/descriptor.h" @@ -344,18 +342,6 @@ inline bool HasHasbit(const FieldDescriptor* descriptor) { return internal::cpp::HasHasbit(descriptor); } -// Whether unknown enum values are kept (i.e., not stored in UnknownFieldSet -// but in the message and can be queried using additional getters that return -// ints. -inline bool SupportUnknownEnumValue(const FieldDescriptor* field) { - if (JavaGenerator::GetResolvedSourceFeatures(*field) - .GetExtension(pb::java) - .legacy_closed_enum()) { - return false; - } - return field->enum_type() != nullptr && !field->enum_type()->is_closed(); -} - // Check whether a message has repeated fields. bool HasRepeatedFields(const Descriptor* descriptor); @@ -375,18 +361,6 @@ inline bool IsWrappersProtoFile(const FileDescriptor* descriptor) { return descriptor->name() == "google/protobuf/wrappers.proto"; } -inline bool CheckUtf8(const FieldDescriptor* descriptor) { - if (JavaGenerator::GetResolvedSourceFeatures(*descriptor) - .GetExtension(pb::java) - .utf8_validation() == pb::JavaFeatures::VERIFY) { - return true; - } - return JavaGenerator::GetResolvedSourceFeatures(*descriptor) - .utf8_validation() == FeatureSet::VERIFY || - // For legacy syntax. This is not allowed under Editions. - descriptor->file()->options().java_string_check_utf8(); -} - void WriteUInt32ToUtf16CharSequence(uint32_t number, std::vector* output); @@ -398,16 +372,6 @@ inline void WriteIntToUtf16CharSequence(int value, // Escape a UTF-16 character so it can be embedded in a Java string literal. void EscapeUtf16ToString(uint16_t code, std::string* output); -// Only the lowest two bytes of the return value are used. The lowest byte -// is the integer value of a j/c/g/protobuf/FieldType enum. For the other -// byte: -// bit 0: whether the field is required. -// bit 1: whether the field requires UTF-8 validation. -// bit 2: whether the field needs isInitialized check. -// bit 3: whether the field is a map field with proto2 enum value. -// bits 4-7: unused -int GetExperimentalJavaFieldType(const FieldDescriptor* field); - // To get the total number of entries need to be built for experimental runtime // and the first field number that are not in the table part std::pair GetTableDrivenNumberOfEntriesAndLookUpStartFieldNumber( diff --git a/src/google/protobuf/compiler/java/immutable/BUILD.bazel b/src/google/protobuf/compiler/java/immutable/BUILD.bazel index a65a03acbdea..baa679ef42c9 100644 --- a/src/google/protobuf/compiler/java/immutable/BUILD.bazel +++ b/src/google/protobuf/compiler/java/immutable/BUILD.bazel @@ -35,6 +35,7 @@ cc_library( "//src/google/protobuf:port", "//src/google/protobuf/compiler/java:generator_common", "//src/google/protobuf/compiler/java:helpers", + "//src/google/protobuf/compiler/java:internal_helpers", "//src/google/protobuf/io:printer", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/log:absl_check", diff --git a/src/google/protobuf/compiler/java/immutable/enum_field.cc b/src/google/protobuf/compiler/java/immutable/enum_field.cc index 4a64460fb5cb..4d0f65539d5f 100644 --- a/src/google/protobuf/compiler/java/immutable/enum_field.cc +++ b/src/google/protobuf/compiler/java/immutable/enum_field.cc @@ -22,6 +22,7 @@ #include "google/protobuf/compiler/java/doc_comment.h" #include "google/protobuf/compiler/java/field_common.h" #include "google/protobuf/compiler/java/helpers.h" +#include "google/protobuf/compiler/java/internal_helpers.h" #include "google/protobuf/compiler/java/name_resolver.h" #include "google/protobuf/io/printer.h" #include "google/protobuf/wire_format.h" diff --git a/src/google/protobuf/compiler/java/immutable/map_field.cc b/src/google/protobuf/compiler/java/immutable/map_field.cc index 13e6319b79f8..12818629589b 100644 --- a/src/google/protobuf/compiler/java/immutable/map_field.cc +++ b/src/google/protobuf/compiler/java/immutable/map_field.cc @@ -15,6 +15,7 @@ #include "google/protobuf/compiler/java/doc_comment.h" #include "google/protobuf/compiler/java/field_common.h" #include "google/protobuf/compiler/java/helpers.h" +#include "google/protobuf/compiler/java/internal_helpers.h" #include "google/protobuf/compiler/java/name_resolver.h" #include "google/protobuf/io/printer.h" diff --git a/src/google/protobuf/compiler/java/immutable/string_field.cc b/src/google/protobuf/compiler/java/immutable/string_field.cc index df12b05f073e..1321f51675ba 100644 --- a/src/google/protobuf/compiler/java/immutable/string_field.cc +++ b/src/google/protobuf/compiler/java/immutable/string_field.cc @@ -22,6 +22,7 @@ #include "google/protobuf/compiler/java/doc_comment.h" #include "google/protobuf/compiler/java/field_common.h" #include "google/protobuf/compiler/java/helpers.h" +#include "google/protobuf/compiler/java/internal_helpers.h" #include "google/protobuf/compiler/java/name_resolver.h" #include "google/protobuf/io/printer.h" #include "google/protobuf/wire_format.h" diff --git a/src/google/protobuf/compiler/java/internal_helpers.cc b/src/google/protobuf/compiler/java/internal_helpers.cc new file mode 100644 index 000000000000..06dae866c46b --- /dev/null +++ b/src/google/protobuf/compiler/java/internal_helpers.cc @@ -0,0 +1,114 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// Author: kenton@google.com (Kenton Varda) +// Based on original Protocol Buffers design by +// Sanjay Ghemawat, Jeff Dean, and others. + +#include "google/protobuf/compiler/java/internal_helpers.h" + +#include "absl/log/absl_log.h" +#include "google/protobuf/compiler/java/helpers.h" +#include "google/protobuf/compiler/java/name_resolver.h" +#include "google/protobuf/descriptor.pb.h" + +// Must be last. +#include "google/protobuf/port_def.inc" + +namespace google { +namespace protobuf { +namespace compiler { +namespace java { +namespace { + +int GetExperimentalJavaFieldTypeForSingular(const FieldDescriptor* field) { + // j/c/g/protobuf/FieldType.java lists field types in a slightly different + // order from FieldDescriptor::Type so we can't do a simple cast. + // + // TODO: Make j/c/g/protobuf/FieldType.java follow the same order. + int result = field->type(); + if (result == FieldDescriptor::TYPE_GROUP) { + return 17; + } else if (result < FieldDescriptor::TYPE_GROUP) { + return result - 1; + } else { + return result - 2; + } +} + +int GetExperimentalJavaFieldTypeForRepeated(const FieldDescriptor* field) { + if (field->type() == FieldDescriptor::TYPE_GROUP) { + return 49; + } else { + return GetExperimentalJavaFieldTypeForSingular(field) + 18; + } +} + +int GetExperimentalJavaFieldTypeForPacked(const FieldDescriptor* field) { + int result = field->type(); + if (result < FieldDescriptor::TYPE_STRING) { + return result + 34; + } else if (result > FieldDescriptor::TYPE_BYTES) { + return result + 30; + } else { + ABSL_LOG(FATAL) << field->full_name() << " can't be packed."; + return 0; + } +} +} // namespace + +int GetExperimentalJavaFieldType(const FieldDescriptor* field) { + static const int kMapFieldType = 50; + static const int kOneofFieldTypeOffset = 51; + + static const int kRequiredBit = 0x100; + static const int kUtf8CheckBit = 0x200; + static const int kCheckInitialized = 0x400; + static const int kLegacyEnumIsClosedBit = 0x800; + static const int kHasHasBit = 0x1000; + int extra_bits = field->is_required() ? kRequiredBit : 0; + if (field->type() == FieldDescriptor::TYPE_STRING && CheckUtf8(field)) { + extra_bits |= kUtf8CheckBit; + } + if (field->is_required() || (GetJavaType(field) == JAVATYPE_MESSAGE && + HasRequiredFields(field->message_type()))) { + extra_bits |= kCheckInitialized; + } + if (HasHasbit(field)) { + extra_bits |= kHasHasBit; + } + if (GetJavaType(field) == JAVATYPE_ENUM && !SupportUnknownEnumValue(field)) { + extra_bits |= kLegacyEnumIsClosedBit; + } + + if (field->is_map()) { + if (!SupportUnknownEnumValue(MapValueField(field))) { + const FieldDescriptor* value = field->message_type()->map_value(); + if (GetJavaType(value) == JAVATYPE_ENUM) { + extra_bits |= kLegacyEnumIsClosedBit; + } + } + return kMapFieldType | extra_bits; + } else if (field->is_packed()) { + return GetExperimentalJavaFieldTypeForPacked(field) | extra_bits; + } else if (field->is_repeated()) { + return GetExperimentalJavaFieldTypeForRepeated(field) | extra_bits; + } else if (IsRealOneof(field)) { + return (GetExperimentalJavaFieldTypeForSingular(field) + + kOneofFieldTypeOffset) | + extra_bits; + } else { + return GetExperimentalJavaFieldTypeForSingular(field) | extra_bits; + } +} + +} // namespace java +} // namespace compiler +} // namespace protobuf +} // namespace google + +#include "google/protobuf/port_undef.inc" diff --git a/src/google/protobuf/compiler/java/internal_helpers.h b/src/google/protobuf/compiler/java/internal_helpers.h new file mode 100644 index 000000000000..e2534b95bf11 --- /dev/null +++ b/src/google/protobuf/compiler/java/internal_helpers.h @@ -0,0 +1,69 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// Author: kenton@google.com (Kenton Varda) +// Based on original Protocol Buffers design by +// Sanjay Ghemawat, Jeff Dean, and others. + +#ifndef GOOGLE_PROTOBUF_COMPILER_JAVA_INTERNAL_HELPERS_H__ +#define GOOGLE_PROTOBUF_COMPILER_JAVA_INTERNAL_HELPERS_H__ + +#include "google/protobuf/compiler/java/generator.h" +#include "google/protobuf/compiler/java/java_features.pb.h" +#include "google/protobuf/compiler/java/names.h" +#include "google/protobuf/descriptor.h" +#include "google/protobuf/descriptor.pb.h" + +// Must be last. +#include "google/protobuf/port_def.inc" + +namespace google { +namespace protobuf { +namespace compiler { +namespace java { + +// Whether unknown enum values are kept (i.e., not stored in UnknownFieldSet +// but in the message and can be queried using additional getters that return +// ints. +inline bool SupportUnknownEnumValue(const FieldDescriptor* field) { + if (JavaGenerator::GetResolvedSourceFeatures(*field) + .GetExtension(pb::java) + .legacy_closed_enum()) { + return false; + } + return field->enum_type() != nullptr && !field->enum_type()->is_closed(); +} + +inline bool CheckUtf8(const FieldDescriptor* descriptor) { + if (JavaGenerator::GetResolvedSourceFeatures(*descriptor) + .GetExtension(pb::java) + .utf8_validation() == pb::JavaFeatures::VERIFY) { + return true; + } + return JavaGenerator::GetResolvedSourceFeatures(*descriptor) + .utf8_validation() == FeatureSet::VERIFY || + // For legacy syntax. This is not allowed under Editions. + descriptor->file()->options().java_string_check_utf8(); +} + +// Only the lowest two bytes of the return value are used. The lowest byte +// is the integer value of a j/c/g/protobuf/FieldType enum. For the other +// byte: +// bit 0: whether the field is required. +// bit 1: whether the field requires UTF-8 validation. +// bit 2: whether the field needs isInitialized check. +// bit 3: whether the field is a map field with proto2 enum value. +// bits 4-7: unused +int GetExperimentalJavaFieldType(const FieldDescriptor* field); + +} // namespace java +} // namespace compiler +} // namespace protobuf +} // namespace google + +#include "google/protobuf/port_undef.inc" +#endif // GOOGLE_PROTOBUF_COMPILER_JAVA_INTERNAL_HELPERS_H__ diff --git a/src/google/protobuf/compiler/java/lite/BUILD.bazel b/src/google/protobuf/compiler/java/lite/BUILD.bazel index 7904d4b42e50..a26f8f9da663 100644 --- a/src/google/protobuf/compiler/java/lite/BUILD.bazel +++ b/src/google/protobuf/compiler/java/lite/BUILD.bazel @@ -30,6 +30,7 @@ cc_library( "//src/google/protobuf:port", "//src/google/protobuf/compiler/java:generator_common", "//src/google/protobuf/compiler/java:helpers", + "//src/google/protobuf/compiler/java:internal_helpers", "//src/google/protobuf/io:printer", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/log:absl_check", @@ -66,6 +67,7 @@ cc_library( "//src/google/protobuf:port", "//src/google/protobuf/compiler/java:generator_common", "//src/google/protobuf/compiler/java:helpers", + "//src/google/protobuf/compiler/java:internal_helpers", "//src/google/protobuf/compiler/java/immutable:service", "//src/google/protobuf/io", "//src/google/protobuf/io:printer", diff --git a/src/google/protobuf/compiler/java/lite/enum_field.cc b/src/google/protobuf/compiler/java/lite/enum_field.cc index c26797739eab..bffec0634745 100644 --- a/src/google/protobuf/compiler/java/lite/enum_field.cc +++ b/src/google/protobuf/compiler/java/lite/enum_field.cc @@ -17,11 +17,11 @@ #include "absl/container/flat_hash_map.h" #include "absl/log/absl_check.h" #include "absl/strings/str_cat.h" -#include "absl/strings/str_join.h" #include "google/protobuf/compiler/java/context.h" #include "google/protobuf/compiler/java/doc_comment.h" #include "google/protobuf/compiler/java/field_common.h" #include "google/protobuf/compiler/java/helpers.h" +#include "google/protobuf/compiler/java/internal_helpers.h" #include "google/protobuf/compiler/java/name_resolver.h" #include "google/protobuf/io/printer.h" #include "google/protobuf/wire_format.h" diff --git a/src/google/protobuf/compiler/java/lite/map_field.cc b/src/google/protobuf/compiler/java/lite/map_field.cc index 491523df9148..7ab58c50ba16 100644 --- a/src/google/protobuf/compiler/java/lite/map_field.cc +++ b/src/google/protobuf/compiler/java/lite/map_field.cc @@ -14,6 +14,7 @@ #include "google/protobuf/compiler/java/doc_comment.h" #include "google/protobuf/compiler/java/field_common.h" #include "google/protobuf/compiler/java/helpers.h" +#include "google/protobuf/compiler/java/internal_helpers.h" #include "google/protobuf/compiler/java/name_resolver.h" #include "google/protobuf/io/printer.h" diff --git a/src/google/protobuf/compiler/java/lite/message_field.cc b/src/google/protobuf/compiler/java/lite/message_field.cc index c46407bf516c..84a10b98887c 100644 --- a/src/google/protobuf/compiler/java/lite/message_field.cc +++ b/src/google/protobuf/compiler/java/lite/message_field.cc @@ -19,6 +19,7 @@ #include "google/protobuf/compiler/java/doc_comment.h" #include "google/protobuf/compiler/java/field_common.h" #include "google/protobuf/compiler/java/helpers.h" +#include "google/protobuf/compiler/java/internal_helpers.h" #include "google/protobuf/compiler/java/name_resolver.h" #include "google/protobuf/io/printer.h" #include "google/protobuf/wire_format.h" diff --git a/src/google/protobuf/compiler/java/lite/primitive_field.cc b/src/google/protobuf/compiler/java/lite/primitive_field.cc index c9b3a2947aae..838df002ce53 100644 --- a/src/google/protobuf/compiler/java/lite/primitive_field.cc +++ b/src/google/protobuf/compiler/java/lite/primitive_field.cc @@ -21,6 +21,7 @@ #include "google/protobuf/compiler/java/doc_comment.h" #include "google/protobuf/compiler/java/field_common.h" #include "google/protobuf/compiler/java/helpers.h" +#include "google/protobuf/compiler/java/internal_helpers.h" #include "google/protobuf/compiler/java/name_resolver.h" #include "google/protobuf/wire_format.h" diff --git a/src/google/protobuf/compiler/java/lite/string_field.cc b/src/google/protobuf/compiler/java/lite/string_field.cc index a08bae50b648..5e926422bc7c 100644 --- a/src/google/protobuf/compiler/java/lite/string_field.cc +++ b/src/google/protobuf/compiler/java/lite/string_field.cc @@ -22,6 +22,7 @@ #include "google/protobuf/compiler/java/doc_comment.h" #include "google/protobuf/compiler/java/field_common.h" #include "google/protobuf/compiler/java/helpers.h" +#include "google/protobuf/compiler/java/internal_helpers.h" #include "google/protobuf/compiler/java/name_resolver.h" #include "google/protobuf/wire_format.h"