Editions: Provide an API for C++ generators to specify their features.

This works for both built-in generators and plugins written in C++.  Any feature extensions specified by this virtual method will end up fully resolved in the descriptors.

PiperOrigin-RevId: 561446773

cmake/tests.cmake

@@ -107,11 +107,13 @@ set(fake_plugin_files
   ${fake_plugin_files}
   ${common_test_hdrs}
   ${common_test_srcs}
+  ${tests_proto_files}
 )
 set(test_plugin_files
   ${test_plugin_files}
   ${common_test_hdrs}
   ${common_test_srcs}
+  ${tests_proto_files}
 )
 
 add_executable(fake_plugin ${fake_plugin_files})

src/google/protobuf/compiler/BUILD.bazel

@@ -221,6 +221,9 @@ cc_library(
     visibility = ["//pkg:__pkg__"],
     deps = [
         ":code_generator",
+        "//src/google/protobuf:cc_test_protos",
+        "//src/google/protobuf:descriptor_legacy",
+        "//src/google/protobuf:descriptor_visitor",
         "//src/google/protobuf/io",
         "//src/google/protobuf/stubs",
         "//src/google/protobuf/testing",

src/google/protobuf/compiler/code_generator.cc

@@ -37,12 +37,14 @@
 #include <utility>
 
 #include "absl/log/absl_log.h"
+#include "absl/status/statusor.h"
 #include "absl/strings/str_cat.h"
 #include "absl/strings/str_split.h"
 #include "absl/strings/string_view.h"
 #include "absl/strings/strip.h"
 #include "google/protobuf/compiler/plugin.pb.h"
 #include "google/protobuf/descriptor.h"
+#include "google/protobuf/feature_resolver.h"
 
 namespace google {
 namespace protobuf {
@@ -76,6 +78,13 @@ bool CodeGenerator::GenerateAll(const std::vector<const FileDescriptor*>& files,
   return succeeded;
 }
 
+absl::StatusOr<FeatureSetDefaults> CodeGenerator::BuildFeatureSetDefaults()
+    const {
+  return FeatureResolver::CompileDefaults(
+      FeatureSet::descriptor(), GetFeatureExtensions(), GetMinimumEdition(),
+      GetMaximumEdition());
+}
+
 GeneratorContext::~GeneratorContext() {}
 
 io::ZeroCopyOutputStream* GeneratorContext::OpenForAppend(

src/google/protobuf/compiler/code_generator.h

@@ -43,6 +43,7 @@
 #include <utility>
 #include <vector>
 
+#include "absl/status/statusor.h"
 #include "absl/strings/string_view.h"
 #include "google/protobuf/compiler/retention.h"
 #include "google/protobuf/descriptor.h"
@@ -131,6 +132,34 @@ class PROTOC_EXPORT CodeGenerator {
   // method can be removed.
   virtual bool HasGenerateAll() const { return true; }
 
+  // Returns all the feature extensions used by this generator.  This must be in
+  // the generated pool, meaning that the extensions should be linked into this
+  // binary.  Any generator features not included here will not get properly
+  // resolved and GetResolvedSourceFeatures will not provide useful values.
+  virtual std::vector<const FieldDescriptor*> GetFeatureExtensions() const {
+    return {};
+  }
+
+  // Returns the minimum edition (inclusive) supported by this generator.  Any
+  // proto files with an edition before this will result in an error.
+  virtual absl::string_view GetMinimumEdition() const {
+    return PROTOBUF_MINIMUM_EDITION;
+  }
+
+  // Returns the maximum edition (inclusive) supported by this generator.  Any
+  // proto files with an edition after this will result in an error.
+  virtual absl::string_view GetMaximumEdition() const {
+    return PROTOBUF_MAXIMUM_EDITION;
+  }
+
+  // Builds a default feature set mapping for this generator.
+  //
+  // This will use the extensions specified by GetFeatureExtensions(), with the
+  // supported edition range [GetMinimumEdition(), GetMaximumEdition].  It has
+  // no side-effects, and code generators only need to call this if they want to
+  // embed the defaults into the generated code.
+  absl::StatusOr<FeatureSetDefaults> BuildFeatureSetDefaults() const;
+
  protected:
   // Retrieves the resolved source features for a given descriptor.  All the
   // features that are imported (from the proto file) and linked in (from the

src/google/protobuf/compiler/code_generator_unittest.cc

@@ -31,11 +31,13 @@
 #include "google/protobuf/compiler/code_generator.h"
 
 #include <string>
+#include <vector>
 
 #include "google/protobuf/descriptor.pb.h"
 #include <gmock/gmock.h>
 #include <gtest/gtest.h>
 #include "absl/log/absl_log.h"
+#include "absl/status/status.h"
 #include "absl/strings/str_format.h"
 #include "absl/strings/str_replace.h"
 #include "absl/strings/string_view.h"
@@ -45,11 +47,15 @@
 #include "google/protobuf/test_textproto.h"
 #include "google/protobuf/unittest_features.pb.h"
 
+// Must be included last.
+#include "google/protobuf/port_def.inc"
+
 namespace google {
 namespace protobuf {
 namespace compiler {
 namespace {
 
+using ::testing::HasSubstr;
 using ::testing::NotNull;
 
 class TestGenerator : public CodeGenerator {
@@ -60,9 +66,37 @@ class TestGenerator : public CodeGenerator {
     return true;
   }
 
+  std::vector<const FieldDescriptor*> GetFeatureExtensions() const override {
+    return feature_extensions_;
+  }
+  void set_feature_extensions(std::vector<const FieldDescriptor*> extensions) {
+    feature_extensions_ = extensions;
+  }
+
+  absl::string_view GetMinimumEdition() const override {
+    return minimum_edition_;
+  }
+  void set_minimum_edition(absl::string_view minimum_edition) {
+    minimum_edition_ = minimum_edition;
+  }
+
+  absl::string_view GetMaximumEdition() const override {
+    return maximum_edition_;
+  }
+  void set_maximum_edition(absl::string_view maximum_edition) {
+    maximum_edition_ = maximum_edition;
+  }
+
   // Expose the protected methods for testing.
   using CodeGenerator::GetResolvedSourceFeatures;
   using CodeGenerator::GetUnresolvedSourceFeatures;
+
+ private:
+  absl::string_view minimum_edition_ = PROTOBUF_MINIMUM_EDITION;
+  absl::string_view maximum_edition_ = PROTOBUF_MAXIMUM_EDITION;
+  std::vector<const FieldDescriptor*> feature_extensions_ = {
+      DescriptorPool::generated_pool()->FindExtensionByNumber(
+          FeatureSet::descriptor(), pb::test.number())};
 };
 
 class SimpleErrorCollector : public io::ErrorCollector {
@@ -74,11 +108,6 @@ class SimpleErrorCollector : public io::ErrorCollector {
 
 class CodeGeneratorTest : public ::testing::Test {
  protected:
-  void SetUp() override {
-    ASSERT_THAT(BuildFile(DescriptorProto::descriptor()->file()), NotNull());
-    ASSERT_THAT(BuildFile(pb::TestMessage::descriptor()->file()), NotNull());
-  }
-
   const FileDescriptor* BuildFile(absl::string_view schema) {
     io::ArrayInputStream input_stream(schema.data(),
                                       static_cast<int>(schema.size()));
@@ -102,6 +131,8 @@ class CodeGeneratorTest : public ::testing::Test {
 };
 
 TEST_F(CodeGeneratorTest, GetUnresolvedSourceFeaturesRoot) {
+  ASSERT_THAT(BuildFile(DescriptorProto::descriptor()->file()), NotNull());
+  ASSERT_THAT(BuildFile(pb::TestMessage::descriptor()->file()), NotNull());
   auto file = BuildFile(R"schema(
     edition = "2023";
     package protobuf_unittest;
@@ -123,6 +154,8 @@ TEST_F(CodeGeneratorTest, GetUnresolvedSourceFeaturesRoot) {
 }
 
 TEST_F(CodeGeneratorTest, GetUnresolvedSourceFeaturesInherited) {
+  ASSERT_THAT(BuildFile(DescriptorProto::descriptor()->file()), NotNull());
+  ASSERT_THAT(BuildFile(pb::TestMessage::descriptor()->file()), NotNull());
   auto file = BuildFile(R"schema(
     edition = "2023";
     package protobuf_unittest;
@@ -156,6 +189,14 @@ TEST_F(CodeGeneratorTest, GetUnresolvedSourceFeaturesInherited) {
 }
 
 TEST_F(CodeGeneratorTest, GetResolvedSourceFeaturesRoot) {
+  TestGenerator generator;
+  generator.set_feature_extensions(
+      {DescriptorPool::generated_pool()->FindExtensionByNumber(
+          FeatureSet::descriptor(), pb::test.number())});
+  pool_.SetFeatureSetDefaults(*generator.BuildFeatureSetDefaults());
+
+  ASSERT_THAT(BuildFile(DescriptorProto::descriptor()->file()), NotNull());
+  ASSERT_THAT(BuildFile(pb::TestMessage::descriptor()->file()), NotNull());
   auto file = BuildFile(R"schema(
     edition = "2023";
     package protobuf_unittest;
@@ -177,13 +218,20 @@ TEST_F(CodeGeneratorTest, GetResolvedSourceFeaturesRoot) {
   EXPECT_EQ(features.field_presence(), FeatureSet::EXPLICIT);
   EXPECT_EQ(features.enum_type(), FeatureSet::CLOSED);
 
-  // TODO(b/296638633) Flip this once generators can specify their feature sets.
-  EXPECT_FALSE(ext.has_int_message_feature());
+  EXPECT_TRUE(ext.has_int_message_feature());
   EXPECT_EQ(ext.int_file_feature(), 8);
   EXPECT_EQ(ext.string_source_feature(), "file");
 }
 
 TEST_F(CodeGeneratorTest, GetResolvedSourceFeaturesInherited) {
+  TestGenerator generator;
+  generator.set_feature_extensions(
+      {DescriptorPool::generated_pool()->FindExtensionByNumber(
+          FeatureSet::descriptor(), pb::test.number())});
+  pool_.SetFeatureSetDefaults(*generator.BuildFeatureSetDefaults());
+
+  ASSERT_THAT(BuildFile(DescriptorProto::descriptor()->file()), NotNull());
+  ASSERT_THAT(BuildFile(pb::TestMessage::descriptor()->file()), NotNull());
   auto file = BuildFile(R"schema(
     edition = "2023";
     package protobuf_unittest;
@@ -223,6 +271,47 @@ TEST_F(CodeGeneratorTest, GetResolvedSourceFeaturesInherited) {
   EXPECT_EQ(ext.string_source_feature(), "field");
 }
 
+// TODO(b/234474291): Use the gtest versions once that's available in OSS.
+MATCHER_P(HasError, msg_matcher, "") {
+  return arg.status().code() == absl::StatusCode::kFailedPrecondition &&
+         ExplainMatchResult(msg_matcher, arg.status().message(),
+                            result_listener);
+}
+MATCHER_P(IsOkAndHolds, matcher, "") {
+  return arg.ok() && ExplainMatchResult(matcher, *arg, result_listener);
+}
+
+TEST_F(CodeGeneratorTest, BuildFeatureSetDefaultsInvalidExtension) {
+  TestGenerator generator;
+  generator.set_feature_extensions({nullptr});
+  EXPECT_THAT(generator.BuildFeatureSetDefaults(),
+              HasError(HasSubstr("Unknown extension")));
+}
+
+TEST_F(CodeGeneratorTest, BuildFeatureSetDefaults) {
+  TestGenerator generator;
+  generator.set_feature_extensions({});
+  generator.set_minimum_edition("2020");
+  generator.set_maximum_edition("2024");
+  EXPECT_THAT(generator.BuildFeatureSetDefaults(),
+              IsOkAndHolds(EqualsProto(R"pb(
+                defaults {
+                  edition: "2023"
+                  features {
+                    field_presence: EXPLICIT
+                    enum_type: OPEN
+                    repeated_field_encoding: PACKED
+                    message_encoding: LENGTH_PREFIXED
+                    json_format: ALLOW
+                  }
+                }
+                minimum_edition: "2020"
+                maximum_edition: "2024"
+              )pb")));
+}
+
+#include "google/protobuf/port_undef.inc"
+
 }  // namespace
 }  // namespace compiler
 }  // namespace protobuf

src/google/protobuf/compiler/command_line_interface.cc

@@ -34,13 +34,17 @@
 
 #include "google/protobuf/compiler/command_line_interface.h"
 
+#include <cstdlib>
+
 #include "absl/algorithm/container.h"
 #include "absl/container/btree_set.h"
 #include "absl/container/flat_hash_map.h"
+#include "absl/status/statusor.h"
 #include "absl/types/span.h"
 #include "google/protobuf/compiler/allowlists/allowlists.h"
 #include "google/protobuf/descriptor_legacy.h"
 #include "google/protobuf/descriptor_visitor.h"
+#include "google/protobuf/feature_resolver.h"
 
 #include "google/protobuf/stubs/platform_macros.h"
 
@@ -1270,8 +1274,13 @@ int CommandLineInterface::Run(int argc, const char* const argv[]) {
 
   descriptor_pool->EnforceWeakDependencies(true);
 
-  // Enforce extension declarations only when compiling. We want to skip this
-  // enforcement when protoc is just being invoked to encode or decode protos.
+  if (!SetupFeatureResolution(*descriptor_pool)) {
+    return EXIT_FAILURE;
+  }
+
+  // Enforce extension declarations only when compiling. We want to skip
+  // this enforcement when protoc is just being invoked to encode or decode
+  // protos.
   if (mode_ == MODE_COMPILE) {
     descriptor_pool->EnforceExtensionDeclarations(true);
   }
@@ -1525,6 +1534,56 @@ bool CommandLineInterface::VerifyInputFilesInDescriptors(
   return true;
 }
 
+bool CommandLineInterface::SetupFeatureResolution(DescriptorPool& pool) {
+  // Calculate the feature defaults for each built-in generator.  All generators
+  // that support editions must agree on the supported edition range.
+  std::vector<const FieldDescriptor*> feature_extensions;
+  absl::string_view minimum_edition = PROTOBUF_MINIMUM_EDITION;
+  absl::string_view maximum_edition = PROTOBUF_MAXIMUM_EDITION;
+  for (const auto& output : output_directives_) {
+    if (output.generator == nullptr) continue;
+    if ((output.generator->GetSupportedFeatures() &
+         CodeGenerator::FEATURE_SUPPORTS_EDITIONS) == 0) {
+      continue;
+    }
+    if (output.generator->GetMinimumEdition() != PROTOBUF_MINIMUM_EDITION) {
+      ABSL_LOG(ERROR) << "Built-in generator " << output.name
+                      << " specifies a minimum edition "
+                      << output.generator->GetMinimumEdition()
+                      << " which is not the protoc minimum "
+                      << PROTOBUF_MINIMUM_EDITION << ".";
+      return false;
+    }
+    if (output.generator->GetMaximumEdition() != PROTOBUF_MAXIMUM_EDITION) {
+      ABSL_LOG(ERROR) << "Built-in generator " << output.name
+                      << " specifies a maximum edition "
+                      << output.generator->GetMaximumEdition()
+                      << " which is not the protoc maximum "
+                      << PROTOBUF_MINIMUM_EDITION << ".";
+      return false;
+    }
+    for (const FieldDescriptor* ext :
+         output.generator->GetFeatureExtensions()) {
+      if (ext == nullptr) {
+        ABSL_LOG(ERROR) << "Built-in generator " << output.name
+                        << " specifies an unknown feature extension.";
+        return false;
+      }
+      feature_extensions.push_back(ext);
+    }
+  }
+  absl::StatusOr<FeatureSetDefaults> defaults =
+      FeatureResolver::CompileDefaults(FeatureSet::descriptor(),
+                                       feature_extensions, minimum_edition,
+                                       maximum_edition);
+  if (!defaults.ok()) {
+    ABSL_LOG(ERROR) << defaults.status();
+    return false;
+  }
+  pool.SetFeatureSetDefaults(std::move(defaults).value());
+  return true;
+}
+
 bool CommandLineInterface::ParseInputFiles(
     DescriptorPool* descriptor_pool, DiskSourceTree* source_tree,
     std::vector<const FileDescriptor*>* parsed_files) {

src/google/protobuf/compiler/command_line_interface.h

@@ -292,6 +292,8 @@ class PROTOC_EXPORT CommandLineInterface {
                        DiskSourceTree* source_tree,
                        std::vector<const FileDescriptor*>* parsed_files);
 
+  bool SetupFeatureResolution(DescriptorPool& pool);
+
   // Generate the given output file from the given input.
   struct OutputDirective;  // see below
   bool GenerateOutput(const std::vector<const FileDescriptor*>& parsed_files,