From 0e6bbc5fd33fc799e90a1dfd0a29ed1e7ec9daca Mon Sep 17 00:00:00 2001 From: felipe stival <14948182+v0idpwn@users.noreply.github.com> Date: Sun, 12 Jan 2025 23:05:04 -0300 Subject: [PATCH] Add native Elixir `JSON` support (#394) * Add `JSON` support This implementation defaults to `JSON` if it's available. If it's not, it tries to use `Jason` as it did before. Some exceptions are changed (only for `JSON`, `Jason` is unaffected). For example, instead of `Jason.DecodeError`, when using `JSON` we raise `Protobuf.JSON.DecodeError`, and encode errors have a more varied shape (since they can be mostly anything coming from Elixir side). * Add Elixir 1.18.1 to workflow * Add note * fix typo * Fix dialyzer * Update lib/protobuf/json.ex --- .github/workflows/main.yml | 4 +++- lib/protobuf/json.ex | 21 ++++++++------------ lib/protobuf/json/decode_error.ex | 15 +++++++++++++++ lib/protobuf/json/json_library.ex | 32 +++++++++++++++++++++++++++++++ test/protobuf/builder_test.exs | 16 +++++++++++----- test/protobuf/encoder_test.exs | 4 +++- test/protobuf/json_test.exs | 25 ++++++++++++++++++------ 7 files changed, 91 insertions(+), 26 deletions(-) create mode 100644 lib/protobuf/json/json_library.ex diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4cbc2b8c..f4049097 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,7 +16,9 @@ jobs: matrix: include: - otp: 27.0 - elixir: 1.17.0 + elixir: 1.18.1 + - otp: 27.0 + elixir: 1.17.3 - otp: 24.3 elixir: 1.12.3 diff --git a/lib/protobuf/json.ex b/lib/protobuf/json.ex index 63c7193a..ca0826a0 100644 --- a/lib/protobuf/json.ex +++ b/lib/protobuf/json.ex @@ -99,7 +99,7 @@ defmodule Protobuf.JSON do exist, decoding will raise an error. """ - alias Protobuf.JSON.{Encode, EncodeError, Decode, DecodeError} + alias Protobuf.JSON.{Encode, EncodeError, Decode, DecodeError, JSONLibrary} @type encode_opt() :: {:use_proto_names, boolean()} @@ -202,11 +202,7 @@ defmodule Protobuf.JSON do @spec encode_to_iodata(struct, [encode_opt]) :: {:ok, iodata()} | {:error, EncodeError.t() | Exception.t()} def encode_to_iodata(%_{} = struct, opts \\ []) when is_list(opts) do - if jason = load_jason() do - with {:ok, map} <- to_encodable(struct, opts), do: jason.encode_to_iodata(map) - else - {:error, EncodeError.new(:no_json_lib)} - end + with {:ok, map} <- to_encodable(struct, opts), do: JSONLibrary.encode_to_iodata(map) end @doc """ @@ -311,11 +307,12 @@ defmodule Protobuf.JSON do """ @spec decode(iodata, module) :: {:ok, struct} | {:error, DecodeError.t() | Exception.t()} def decode(iodata, module) when is_atom(module) do - if jason = load_jason() do - with {:ok, json_data} <- jason.decode(iodata), - do: from_decoded(json_data, module) - else - {:error, DecodeError.new(:no_json_lib)} + case JSONLibrary.decode(iodata) do + {:ok, json_data} -> + from_decoded(json_data, module) + + {:error, exception} -> + {:error, exception} end end @@ -344,6 +341,4 @@ defmodule Protobuf.JSON do catch error -> {:error, DecodeError.new(error)} end - - defp load_jason, do: Code.ensure_loaded?(Jason) and Jason end diff --git a/lib/protobuf/json/decode_error.ex b/lib/protobuf/json/decode_error.ex index daadd111..30a37002 100644 --- a/lib/protobuf/json/decode_error.ex +++ b/lib/protobuf/json/decode_error.ex @@ -69,4 +69,19 @@ defmodule Protobuf.JSON.DecodeError do def new({:bad_repeated, field, value}) do %__MODULE__{message: "Repeated field '#{field}' expected a list, got #{inspect(value)}"} end + + def new({:unexpected_end, position}) do + %__MODULE__{message: "Unexpected end at position #{inspect(position)}"} + end + + def new({:invalid_byte, position, byte}) do + %__MODULE__{message: "Invalid byte at position #{inspect(position)}, byte: #{inspect(byte)}"} + end + + def new({:unexpected_sequence, position, sequence}) do + %__MODULE__{ + message: + "Unexpected sequence at position #{inspect(position)}, sequence: #{inspect(sequence)}" + } + end end diff --git a/lib/protobuf/json/json_library.ex b/lib/protobuf/json/json_library.ex new file mode 100644 index 00000000..d9ba06b1 --- /dev/null +++ b/lib/protobuf/json/json_library.ex @@ -0,0 +1,32 @@ +defmodule Protobuf.JSON.JSONLibrary do + @moduledoc false + # Uses `JSON` for Elixir >= 1.18, Jason if Elixir < 1.18 and Jason available, + # or returns error otherwise + + cond do + Code.ensure_loaded?(JSON) -> + def encode_to_iodata(encodable) do + try do + {:ok, JSON.encode_to_iodata!(encodable)} + rescue + exception -> + {:error, exception} + end + end + + def decode(data) do + case JSON.decode(data) do + {:ok, decoded} -> {:ok, decoded} + {:error, error} -> {:error, Protobuf.JSON.DecodeError.new(error)} + end + end + + Code.ensure_loaded?(Jason) -> + def encode_to_iodata(encodable), do: Jason.encode_to_iodata(encodable) + def decode(data), do: Jason.decode(data) + + true -> + def encode_to_iodata(_), do: {:error, EncodeError.new(:no_json_lib)} + def decode(_), do: {:error, EncodeError.new(:no_json_lib)} + end +end diff --git a/test/protobuf/builder_test.exs b/test/protobuf/builder_test.exs index 9a72e6b8..7bd1d0b6 100644 --- a/test/protobuf/builder_test.exs +++ b/test/protobuf/builder_test.exs @@ -45,11 +45,17 @@ defmodule Protobuf.BuilderTest do end test "raises an error for non-list repeated embedded msgs" do - assert_raise Protocol.UndefinedError, - ~r/protocol Enumerable not implemented for %TestMsg.Foo.Bar/, - fn -> - Foo.new(h: Foo.Bar.new()) - end + # TODO: remove conditional once we support only Elixir 1.18+ + message = + if System.version() >= "1.18.0" do + ~r/protocol Enumerable not implemented for type TestMsg.Foo.Ba/ + else + ~r/protocol Enumerable not implemented for %TestMsg.Foo.Bar/ + end + + assert_raise Protocol.UndefinedError, message, fn -> + Foo.new(h: Foo.Bar.new()) + end end test "builds correct message for non matched struct" do diff --git a/test/protobuf/encoder_test.exs b/test/protobuf/encoder_test.exs index 527d0a9e..98873c00 100644 --- a/test/protobuf/encoder_test.exs +++ b/test/protobuf/encoder_test.exs @@ -283,7 +283,9 @@ defmodule Protobuf.EncoderTest do Encoder.encode(%TestMsg.Foo{c: 123}) end - message = ~r/protocol Enumerable not implemented for 123/ + # For Elixir 1.18+ it's `type Integer`, before, it was just `123` + # TODO: fix once we require Elixir 1.18+ + message = ~r/protocol Enumerable not implemented for (123|type Integer)/ assert_raise Protobuf.EncodeError, message, fn -> Encoder.encode(%TestMsg.Foo{e: 123}) diff --git a/test/protobuf/json_test.exs b/test/protobuf/json_test.exs index 9285f928..aff755ad 100644 --- a/test/protobuf/json_test.exs +++ b/test/protobuf/json_test.exs @@ -16,20 +16,33 @@ defmodule Protobuf.JSONTest do test "encoding string field with invalid UTF-8 data" do message = %Scalars{string: " \xff "} - assert {:error, %Jason.EncodeError{}} = Protobuf.JSON.encode(message) + assert {:error, exception} = Protobuf.JSON.encode(message) + assert is_exception(exception) end test "decoding string field with invalid UTF-8 data" do json = ~S|{"string":" \xff "}| - assert {:error, %Jason.DecodeError{}} = Protobuf.JSON.decode(json, Scalars) + assert {:error, exception} = Protobuf.JSON.decode(json, Scalars) + assert is_exception(exception) end describe "bang variants of encode and decode" do - test "decode!/2" do - json = ~S|{"string":" \xff "}| + # TODO: remove Jason when we require Elixir 1.18 + if Code.ensure_loaded?(JSON) do + test "decode!/2" do + json = ~S|{"string":" \xff "}| - assert_raise Jason.DecodeError, fn -> - Protobuf.JSON.decode!(json, Scalars) + assert_raise Protobuf.JSON.DecodeError, fn -> + Protobuf.JSON.decode!(json, Scalars) + end + end + else + test "decode!/2" do + json = ~S|{"string":" \xff "}| + + assert_raise Jason.DecodeError, fn -> + Protobuf.JSON.decode!(json, Scalars) + end end end end