Create Protobuf.Text, create Protobuf.field_presence/2

conformance/exemptions.txt

@@ -1,4 +1,6 @@
 Recommended.Proto2.JsonInput.FieldNameExtension.Validator
+
+# We raise if find unknown enum values
 Recommended.Proto3.JsonInput.IgnoreUnknownEnumStringValueInMapValue.ProtobufOutput
 Recommended.Proto3.JsonInput.IgnoreUnknownEnumStringValueInOptionalField.ProtobufOutput
 Recommended.Proto3.JsonInput.IgnoreUnknownEnumStringValueInRepeatedField.ProtobufOutput
@@ -9,3 +11,8 @@ Recommended.Proto2.JsonInput.IgnoreUnknownEnumStringValueInRepeatedField.Protobu
 Recommended.Proto2.JsonInput.IgnoreUnknownEnumStringValueInRepeatedPart.ProtobufOutput
 Recommended.Proto3.JsonInput.IgnoreUnknownEnumStringValueInMapPart.ProtobufOutput
 Recommended.Proto3.JsonInput.IgnoreUnknownEnumStringValueInRepeatedPart.ProtobufOutput
+
+# Any
+Required.Proto3.JsonInput.AnyWithNoType.JsonOutput
+Required.Proto3.JsonInput.AnyWktRepresentationWithBadType
+Required.Proto3.JsonInput.AnyWktRepresentationWithEmptyTypeAndValue

conformance/protobuf/runner.ex

@@ -77,10 +77,11 @@ defmodule Conformance.Protobuf.Runner do
   defp handle_conformance_request(%mod{
          requested_output_format: requested_output_format,
          message_type: message_type,
-         payload: {payload_kind, msg}
+         payload: {payload_kind, msg},
+         print_unknown_fields: print_unknown_fields
        })
        when mod == Conformance.ConformanceRequest and
-              requested_output_format in [:PROTOBUF, :JSON] and
+              requested_output_format in [:PROTOBUF, :JSON, :TEXT_FORMAT] and
               payload_kind in [:protobuf_payload, :json_payload] do
     test_proto_type = to_test_proto_type(message_type)
 
@@ -94,6 +95,7 @@ defmodule Conformance.Protobuf.Runner do
       case requested_output_format do
         :PROTOBUF -> {&safe_encode/1, :protobuf_payload}
         :JSON -> {&Protobuf.JSON.encode/1, :json_payload}
+        :TEXT_FORMAT -> {&safe_text_encode(&1, print_unknown_fields), :text_payload}
       end
 
     with {:decode, {:ok, decoded_msg}} <- {:decode, decode_fun.(msg, test_proto_type)},
@@ -138,4 +140,10 @@ defmodule Conformance.Protobuf.Runner do
   rescue
     exception -> {:error, exception, __STACKTRACE__}
   end
+
+  defp safe_text_encode(struct, print_unknown_fields?) do
+    {:ok, Protobuf.Text.encode(struct, print_unknown_fields?: print_unknown_fields?)}
+  rescue
+    exception -> {:error, exception, __STACKTRACE__}
+  end
 end

conformance/text-exemptions.txt

@@ -0,0 +1,4 @@
+# We do a best effort for printing unknown values but don't try to expand them
+Recommended.Proto3.ProtobufInput.GroupUnknownFields_Print.TextFormatOutput
+Recommended.Proto3.ProtobufInput.MessageUnknownFields_Print.TextFormatOutput
+Recommended.Proto3.ProtobufInput.RepeatedUnknownFields_Print.TextFormatOutput

lib/google/protobuf/descriptor.pb.ex

@@ -189,6 +189,16 @@ defmodule Google.Protobuf.FeatureSet.JsonFormat do
   field :LEGACY_BEST_EFFORT, 2
 end
 
+defmodule Google.Protobuf.FeatureSet.EnforceNamingStyle do
+  @moduledoc false
+
+  use Protobuf, enum: true, protoc_gen_elixir_version: "0.13.0", syntax: :proto2
+
+  field :ENFORCE_NAMING_STYLE_UNKNOWN, 0
+  field :STYLE2024, 1
+  field :STYLE_LEGACY, 2
+end
+
 defmodule Google.Protobuf.GeneratedCodeInfo.Annotation.Semantic do
   @moduledoc false
 
@@ -822,6 +832,13 @@ defmodule Google.Protobuf.FeatureSet do
     enum: true,
     deprecated: false
 
+  field :enforce_naming_style, 7,
+    optional: true,
+    type: Google.Protobuf.FeatureSet.EnforceNamingStyle,
+    json_name: "enforceNamingStyle",
+    enum: true,
+    deprecated: false
+
   extensions [{1000, 9995}, {9995, 10000}, {10000, 10001}]
 end
 

lib/protobuf.ex

@@ -285,6 +285,56 @@ defmodule Protobuf do
             "version that introduced implicit struct generation"
   end
 
+  @doc """
+  Returns whether a field or oneof is present, not present, or maybe present
+
+  `:present` and `:not present` mean that a field is **explicitly** present or not,
+  respectively.
+
+  Some values may be implicitly present. For example, lists in `repeated` fields
+  always have implicit presence. In these cases, if the presence is ambiguous,
+  returns `:maybe`.
+
+  For more information about field presence tracking rules, refer to the official
+  [Field Presence docs](https://protobuf.dev/programming-guides/field_presence/).
+
+
+  ## Examples
+
+      # Non-optional proto3 field:
+      Protobuf.field_presence(%MyMessage{foo: 42}, :foo)
+      #=> :present
+
+      Protobuf.field_presence(%MyMessage{foo: 0}, :foo)
+      #=> :maybe
+
+      Protobuf.field_presence(%MyMessage{}, :foo)
+      #=> :maybe
+
+      # Optional proto3 field:
+      Protobuf.field_presence(%MyMessage{bar: 42}, :bar)
+      #=> :present
+
+      Protobuf.field_presence(%MyMessage{bar: 0}, :bar)
+      #=> :present
+
+      Protobuf.field_presence(%MyMessage{}, :bar)
+      #=> :not_present
+
+      # Repeated
+      Protobuf.field_presence(%MyMessage{repeated_field: []}, :repeated_field)
+      #=> :maybe
+
+      Protobuf.field_presence(%MyMessage{repeated_field: [1}, :repeated_field)
+      #=> :present
+
+  """
+  @doc since: "0.15.0"
+  @spec field_presence(message :: struct(), field :: atom()) :: :present | :not_present | :maybe
+  def field_presence(message, field) do
+    Protobuf.Presence.field_presence(message, field)
+  end
+
   @doc """
   Loads extensions modules.
 

lib/protobuf/encoder.ex

@@ -83,28 +83,22 @@ defmodule Protobuf.Encoder do
           "Got error when encoding #{inspect(struct_mod)}##{prop.name_atom}: #{Exception.format(:error, error)}"
   end
 
-  defp skip_field?(_syntax, [], _prop), do: true
-  defp skip_field?(_syntax, val, _prop) when is_map(val), do: map_size(val) == 0
-  defp skip_field?(:proto2, nil, %FieldProps{optional?: optional?}), do: optional?
-  defp skip_field?(:proto2, value, %FieldProps{default: value, oneof: nil}), do: true
-
-  defp skip_field?(:proto3, val, %FieldProps{proto3_optional?: true}),
-    do: is_nil(val)
-
-  defp skip_field?(:proto3, nil, _prop), do: true
-  defp skip_field?(:proto3, 0, %FieldProps{oneof: nil}), do: true
-  defp skip_field?(:proto3, +0.0, %FieldProps{oneof: nil}), do: true
-  defp skip_field?(:proto3, "", %FieldProps{oneof: nil}), do: true
-  defp skip_field?(:proto3, false, %FieldProps{oneof: nil}), do: true
-  defp skip_field?(_syntax, _val, _prop), do: false
+  defp skip_field?(syntax, value, field_prop) do
+    case Protobuf.Presence.get_field_presence(syntax, value, field_prop) do
+      :present -> false
+      # Proto2 required isn't skipped even if not present
+      :maybe -> not (syntax == :proto2 && field_prop.required?)
+      :not_present -> not (syntax == :proto2 && field_prop.required?)
+    end
+  end
 
   defp do_encode_field(
          :normal,
          val,
          syntax,
          %FieldProps{encoded_fnum: fnum, type: type, repeated?: repeated?} = prop
        ) do
-    if skip_field?(syntax, val, prop) or skip_enum?(syntax, val, prop) do
+    if skip_field?(syntax, val, prop) do
       :skip
     else
       iodata = apply_or_map(val, repeated?, &[fnum | Wire.encode(type, &1)])
@@ -142,7 +136,7 @@ defmodule Protobuf.Encoder do
   end
 
   defp do_encode_field(:packed, val, syntax, %FieldProps{type: type, encoded_fnum: fnum} = prop) do
-    if skip_field?(syntax, val, prop) or skip_enum?(syntax, val, prop) do
+    if skip_field?(syntax, val, prop) do
       :skip
     else
       encoded = Enum.map(val, &Wire.encode(type, &1))
@@ -197,20 +191,6 @@ defmodule Protobuf.Encoder do
   defp apply_or_map(val, _repeated? = true, func), do: Enum.map(val, func)
   defp apply_or_map(val, _repeated? = false, func), do: func.(val)
 
-  defp skip_enum?(:proto2, _value, _prop), do: false
-  defp skip_enum?(:proto3, _value, %FieldProps{proto3_optional?: true}), do: false
-  defp skip_enum?(_syntax, _value, %FieldProps{enum?: false}), do: false
-
-  defp skip_enum?(_syntax, _value, %FieldProps{enum?: true, oneof: oneof}) when not is_nil(oneof),
-    do: false
-
-  defp skip_enum?(_syntax, _value, %FieldProps{required?: true}), do: false
-  defp skip_enum?(_syntax, value, %FieldProps{type: type}), do: enum_default?(type, value)
-
-  defp enum_default?({:enum, enum_mod}, val) when is_atom(val), do: enum_mod.value(val) == 0
-  defp enum_default?({:enum, _enum_mod}, val) when is_integer(val), do: val == 0
-  defp enum_default?({:enum, _enum_mod}, list) when is_list(list), do: false
-
   # Returns a map of %{field_name => field_value} from oneofs. For example, if you have:
   # oneof body {
   #   string a = 1;

lib/protobuf/json/encode.ex

@@ -162,7 +162,7 @@ defmodule Protobuf.JSON.Encode do
   defp encode_regular_fields(struct, %{field_props: field_props, syntax: syntax}, opts) do
     for {_field_num, %{name_atom: name, oneof: nil} = prop} <- field_props,
         %{^name => value} = struct,
-        emit?(syntax, prop, value) || opts[:emit_unpopulated] do
+        emit?(syntax, prop, value, opts[:emit_unpopulated]) do
       encode_field(prop, value, opts)
     end
   end
@@ -301,18 +301,17 @@ defmodule Protobuf.JSON.Encode do
   defp maybe_repeat(%{repeated?: false}, val, fun), do: fun.(val)
   defp maybe_repeat(%{repeated?: true}, val, fun), do: Enum.map(val, fun)
 
-  defp emit?(:proto2, %{default: value}, value), do: false
-  defp emit?(:proto2, %{optional?: true}, val), do: not is_nil(val)
-  defp emit?(:proto3, %{proto3_optional?: true}, val), do: not is_nil(val)
-  defp emit?(_syntax, _prop, +0.0), do: false
-  defp emit?(_syntax, _prop, nil), do: false
-  defp emit?(_syntax, _prop, 0), do: false
-  defp emit?(_syntax, _prop, false), do: false
-  defp emit?(_syntax, _prop, []), do: false
-  defp emit?(_syntax, _prop, ""), do: false
-  defp emit?(_syntax, _prop, %{} = map) when map_size(map) == 0, do: false
-  defp emit?(_syntax, %{type: {:enum, enum}}, key) when is_atom(key), do: enum.value(key) != 0
-  defp emit?(_syntax, _prop, _value), do: true
+  defp emit?(_syntax, _field_prop, _value, true = _emit_unpopulated?) do
+    true
+  end
+
+  defp emit?(syntax, field_prop, value, _emit_unpopulated?) do
+    case Protobuf.Presence.get_field_presence(syntax, value, field_prop) do
+      :present -> true
+      :maybe -> false
+      :not_present -> false
+    end
+  end
 
   defp transform_module(message, module) do
     if transform_module = module.transform_module() do

lib/protobuf/presence.ex

@@ -0,0 +1,141 @@
+defmodule Protobuf.Presence do
+  @moduledoc false
+
+  alias Protobuf.FieldProps
+
+  @spec field_presence(message :: struct(), field :: atom()) :: :present | :not_present | :maybe
+  def field_presence(%mod{} = message, field) do
+    message_props = mod.__message_props__()
+    transformed_message = transform_module(message, mod)
+    fnum = Map.fetch!(message_props.field_tags, field)
+    field_prop = Map.fetch!(message_props.field_props, fnum)
+    value = get_oneof_value(transformed_message, message_props, field, field_prop)
+
+    transformed_value =
+      case field_prop do
+        %{embedded: true, type: mod} -> transform_module(value, mod)
+        _ -> value
+      end
+
+    get_field_presence(message_props.syntax, transformed_value, field_prop)
+  end
+
+  defp get_oneof_value(message, message_props, field, field_prop) do
+    case field_prop.oneof do
+      nil ->
+        Map.fetch!(message, field)
+
+      oneof_num ->
+        {oneof_field, _} = Enum.find(message_props.oneof, fn {_name, tag} -> tag == oneof_num end)
+
+        case Map.fetch!(message, oneof_field) do
+          {^field, value} -> value
+          _ -> nil
+        end
+    end
+  end
+
+  defp transform_module(message, module) do
+    if transform_module = module.transform_module() do
+      transform_module.encode(message, module)
+    else
+      message
+    end
+  end
+
+  # We probably want to make this public eventually, but it makes sense to hold
+  # it until we add editions support, since we definitely don't want to add
+  # `syntax` in a public API
+  @doc false
+  @spec get_field_presence(:proto2 | :proto3, term(), FieldProps.t()) ::
+          :present | :not_present | :maybe
+  def get_field_presence(syntax, value, field_prop)
+
+  # Repeated and maps are always implicit.
+  def get_field_presence(_syntax, [], _prop) do
+    :maybe
+  end
+
+  def get_field_presence(_syntax, val, %FieldProps{map?: true}) when is_map(val) do
+    if map_size(val) == 0 do
+      :maybe
+    else
+      :present
+    end
+  end
+
+  # For proto2 singular cardinality fields:
+  #
+  # - Non-one_of fields with default values have implicit presence
+  # - Others have explicit presence
+  def get_field_presence(:proto2, nil, _prop) do
+    :not_present
+  end
+
+  def get_field_presence(:proto2, value, %FieldProps{default: value, oneof: nil}) do
+    :maybe
+  end
+
+  def get_field_presence(:proto2, _value, _props) do
+    :present
+  end
+
+  # For proto3 singular cardinality fields:
+  #
+  # - Optional and Oneof fields have explicit presence tracking
+  # - Other fields have implicit presence tracking
+  def get_field_presence(:proto3, nil, %FieldProps{proto3_optional?: true}) do
+    :not_present
+  end
+
+  def get_field_presence(:proto3, _, %FieldProps{proto3_optional?: true}) do
+    :present
+  end
+
+  def get_field_presence(_syntax, value, %FieldProps{oneof: oneof}) when not is_nil(oneof) do
+    if is_nil(value) do
+      :not_present
+    else
+      :present
+    end
+  end
+
+  # Messages have explicit presence tracking in proto3
+  def get_field_presence(:proto3, nil, _prop) do
+    :not_present
+  end
+
+  # Defaults for different field types: implicit presence means they are maybe set
+  def get_field_presence(:proto3, 0, _prop) do
+    :maybe
+  end
+
+  def get_field_presence(:proto3, +0.0, _prop) do
+    :maybe
+  end
+
+  def get_field_presence(:proto3, "", _prop) do
+    :maybe
+  end
+
+  def get_field_presence(:proto3, false, _prop) do
+    :maybe
+  end
+
+  def get_field_presence(_syntax, value, %FieldProps{type: {:enum, enum_mod}}) do
+    if enum_default?(enum_mod, value) do
+      :maybe
+    else
+      :present
+    end
+  end
+
+  # Finally, everything else.
+  def get_field_presence(_syntax, _val, _prop) do
+    :present
+  end
+
+  defp enum_default?(enum_mod, val) when is_atom(val), do: enum_mod.value(val) == 0
+  defp enum_default?(_enum_mod, val) when is_integer(val), do: val == 0
+  defp enum_default?(_enum_mod, list) when is_list(list), do: false
+end