From 394321f86f388f5b61f9c624dd92c170d47e4f5a Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Mon, 31 Oct 2022 14:31:39 -0400 Subject: [PATCH 01/54] branch to support FFI --- ruby/lib/google/{protobuf.rb => protobuf_ffi.rb} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename ruby/lib/google/{protobuf.rb => protobuf_ffi.rb} (100%) diff --git a/ruby/lib/google/protobuf.rb b/ruby/lib/google/protobuf_ffi.rb similarity index 100% rename from ruby/lib/google/protobuf.rb rename to ruby/lib/google/protobuf_ffi.rb From 11fcd231ecb01ea5aaa0dd9225ded137c62053e5 Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Mon, 31 Oct 2022 14:32:37 -0400 Subject: [PATCH 02/54] restore protobuf.rb --- ruby/lib/google/protobuf.rb | 79 +++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 ruby/lib/google/protobuf.rb diff --git a/ruby/lib/google/protobuf.rb b/ruby/lib/google/protobuf.rb new file mode 100644 index 000000000000..b7a671105158 --- /dev/null +++ b/ruby/lib/google/protobuf.rb @@ -0,0 +1,79 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# require mixins before we hook them into the java & c code +require 'google/protobuf/message_exts' + +# We define these before requiring the platform-specific modules. +# That way the module init can grab references to these. +module Google + module Protobuf + class Error < StandardError; end + class ParseError < Error; end + class TypeError < ::TypeError; end + end +end + +if RUBY_PLATFORM == "java" + require 'json' + require 'google/protobuf_java' +else + begin + require "google/#{RUBY_VERSION.sub(/\.\d+$/, '')}/protobuf_c" + rescue LoadError + require 'google/protobuf_c' + end + +end + +require 'google/protobuf/descriptor_dsl' +require 'google/protobuf/repeated_field' + +module Google + module Protobuf + + def self.encode(msg, options = {}) + msg.to_proto(options) + end + + def self.encode_json(msg, options = {}) + msg.to_json(options) + end + + def self.decode(klass, proto, options = {}) + klass.decode(proto, options) + end + + def self.decode_json(klass, json, options = {}) + klass.decode_json(json, options) + end + + end +end From 09a71143b4d6b86f7935052af8cbac343c86a609 Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Mon, 31 Oct 2022 14:40:08 -0400 Subject: [PATCH 03/54] native implementation --- ruby/lib/google/{protobuf.rb => protobuf_native.rb} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename ruby/lib/google/{protobuf.rb => protobuf_native.rb} (100%) diff --git a/ruby/lib/google/protobuf.rb b/ruby/lib/google/protobuf_native.rb similarity index 100% rename from ruby/lib/google/protobuf.rb rename to ruby/lib/google/protobuf_native.rb From 1e50c8b65cd4d72338dabd76f002fda136d44cca Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Mon, 31 Oct 2022 14:40:40 -0400 Subject: [PATCH 04/54] restore protobuf.rb --- ruby/lib/google/protobuf.rb | 79 +++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 ruby/lib/google/protobuf.rb diff --git a/ruby/lib/google/protobuf.rb b/ruby/lib/google/protobuf.rb new file mode 100644 index 000000000000..b7a671105158 --- /dev/null +++ b/ruby/lib/google/protobuf.rb @@ -0,0 +1,79 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# require mixins before we hook them into the java & c code +require 'google/protobuf/message_exts' + +# We define these before requiring the platform-specific modules. +# That way the module init can grab references to these. +module Google + module Protobuf + class Error < StandardError; end + class ParseError < Error; end + class TypeError < ::TypeError; end + end +end + +if RUBY_PLATFORM == "java" + require 'json' + require 'google/protobuf_java' +else + begin + require "google/#{RUBY_VERSION.sub(/\.\d+$/, '')}/protobuf_c" + rescue LoadError + require 'google/protobuf_c' + end + +end + +require 'google/protobuf/descriptor_dsl' +require 'google/protobuf/repeated_field' + +module Google + module Protobuf + + def self.encode(msg, options = {}) + msg.to_proto(options) + end + + def self.encode_json(msg, options = {}) + msg.to_json(options) + end + + def self.decode(klass, proto, options = {}) + klass.decode(proto, options) + end + + def self.decode_json(klass, json, options = {}) + klass.decode_json(json, options) + end + + end +end From 6cc806cc08cb7aa8794a0b3aef60e6416e7d1e44 Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Mon, 31 Oct 2022 15:10:55 -0400 Subject: [PATCH 05/54] Refactor `protobuf.rb` to separate native and ffi implementations. --- ruby/lib/google/protobuf.rb | 30 ++++-------- ruby/lib/google/protobuf_ffi.rb | 76 +++++++++++++----------------- ruby/lib/google/protobuf_native.rb | 38 +-------------- 3 files changed, 45 insertions(+), 99 deletions(-) diff --git a/ruby/lib/google/protobuf.rb b/ruby/lib/google/protobuf.rb index b7a671105158..5dfd91f5dd44 100644 --- a/ruby/lib/google/protobuf.rb +++ b/ruby/lib/google/protobuf.rb @@ -38,26 +38,6 @@ module Protobuf class Error < StandardError; end class ParseError < Error; end class TypeError < ::TypeError; end - end -end - -if RUBY_PLATFORM == "java" - require 'json' - require 'google/protobuf_java' -else - begin - require "google/#{RUBY_VERSION.sub(/\.\d+$/, '')}/protobuf_c" - rescue LoadError - require 'google/protobuf_c' - end - -end - -require 'google/protobuf/descriptor_dsl' -require 'google/protobuf/repeated_field' - -module Google - module Protobuf def self.encode(msg, options = {}) msg.to_proto(options) @@ -75,5 +55,15 @@ def self.decode_json(klass, json, options = {}) klass.decode_json(json, options) end + IMPLEMENTATION = case ENV['PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION'] + when nil, 'native', '' + require 'google/protobuf_native' + :NATIVE + when 'ffi' + require 'google/protobuf_native' + :FFI + else + raise RuntimeError("Unsupport value of PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION environment variable `#{ENV['PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION']}`. Must be one of `native` or `ffi`. Defaults to `native`.") + end end end diff --git a/ruby/lib/google/protobuf_ffi.rb b/ruby/lib/google/protobuf_ffi.rb index b7a671105158..a635b2edcb8c 100644 --- a/ruby/lib/google/protobuf_ffi.rb +++ b/ruby/lib/google/protobuf_ffi.rb @@ -1,5 +1,5 @@ # Protocol Buffers - Google's data interchange format -# Copyright 2008 Google Inc. All rights reserved. +# Copyright 2021 Google Inc. All rights reserved. # https://developers.google.com/protocol-buffers/ # # Redistribution and use in source and binary forms, with or without @@ -28,52 +28,44 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# require mixins before we hook them into the java & c code -require 'google/protobuf/message_exts' - -# We define these before requiring the platform-specific modules. -# That way the module init can grab references to these. -module Google - module Protobuf - class Error < StandardError; end - class ParseError < Error; end - class TypeError < ::TypeError; end - end -end - -if RUBY_PLATFORM == "java" - require 'json' - require 'google/protobuf_java' -else - begin - require "google/#{RUBY_VERSION.sub(/\.\d+$/, '')}/protobuf_c" - rescue LoadError - require 'google/protobuf_c' - end - -end - +require 'ffi' +require 'google/protobuf/ffi/internal/type_safety' +require 'google/protobuf/ffi/internal/arena' +require 'google/protobuf/ffi/internal/convert' +require 'google/protobuf/ffi/descriptor' +require 'google/protobuf/ffi/enum_descriptor' +require 'google/protobuf/ffi/field_descriptor' +require 'google/protobuf/ffi/oneof_descriptor' +require 'google/protobuf/ffi/ffi' +require 'google/protobuf/ffi/descriptor_pool' +require 'google/protobuf/ffi/file_descriptor' +require 'google/protobuf/ffi/map' +require 'google/protobuf/ffi/object_cache' +require 'google/protobuf/ffi/repeated_field' require 'google/protobuf/descriptor_dsl' -require 'google/protobuf/repeated_field' module Google module Protobuf - - def self.encode(msg, options = {}) - msg.to_proto(options) - end - - def self.encode_json(msg, options = {}) - msg.to_json(options) + def self.deep_copy(object) + case object + when RepeatedField + RepeatedField.send(:deep_copy, object) + when Google::Protobuf::Map + Google::Protobuf::Map.deep_copy(object) + when Google::Protobuf::MessageExts + object.class.send(:deep_copy, object.instance_variable_get(:@msg)) + else + raise NotImplementedError + end end - def self.decode(klass, proto, options = {}) - klass.decode(proto, options) + def self.discard_unknown(message) + raise FrozenError if message.frozen? + raise ArgumentError.new "Expected message, got #{message.class} instead." if message.instance_variable_get(:@msg).nil? + unless Google::Protobuf::FFI.message_discard_unknown(message.instance_variable_get(:@msg), message.class.descriptor, 128) + raise RuntimeError.new "Messages nested too deeply." + end + nil end - - def self.decode_json(klass, json, options = {}) - klass.decode_json(json, options) - end - end -end +end \ No newline at end of file diff --git a/ruby/lib/google/protobuf_native.rb b/ruby/lib/google/protobuf_native.rb index b7a671105158..f55d6c48ae6e 100644 --- a/ruby/lib/google/protobuf_native.rb +++ b/ruby/lib/google/protobuf_native.rb @@ -1,5 +1,5 @@ # Protocol Buffers - Google's data interchange format -# Copyright 2008 Google Inc. All rights reserved. +# Copyright 2021 Google Inc. All rights reserved. # https://developers.google.com/protocol-buffers/ # # Redistribution and use in source and binary forms, with or without @@ -28,19 +28,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# require mixins before we hook them into the java & c code -require 'google/protobuf/message_exts' - -# We define these before requiring the platform-specific modules. -# That way the module init can grab references to these. -module Google - module Protobuf - class Error < StandardError; end - class ParseError < Error; end - class TypeError < ::TypeError; end - end -end - if RUBY_PLATFORM == "java" require 'json' require 'google/protobuf_java' @@ -50,30 +37,7 @@ class TypeError < ::TypeError; end rescue LoadError require 'google/protobuf_c' end - end require 'google/protobuf/descriptor_dsl' require 'google/protobuf/repeated_field' - -module Google - module Protobuf - - def self.encode(msg, options = {}) - msg.to_proto(options) - end - - def self.encode_json(msg, options = {}) - msg.to_json(options) - end - - def self.decode(klass, proto, options = {}) - klass.decode(proto, options) - end - - def self.decode_json(klass, json, options = {}) - klass.decode_json(json, options) - end - - end -end From dfc7dbb1872d9fc3372e4e42efb15f1265ea9394 Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Fri, 6 Jan 2023 14:48:06 -0500 Subject: [PATCH 06/54] Fix spelling `Unsupport` -> `Unsupported` --- ruby/lib/google/protobuf.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ruby/lib/google/protobuf.rb b/ruby/lib/google/protobuf.rb index 5dfd91f5dd44..29b367352f19 100644 --- a/ruby/lib/google/protobuf.rb +++ b/ruby/lib/google/protobuf.rb @@ -60,10 +60,10 @@ def self.decode_json(klass, json, options = {}) require 'google/protobuf_native' :NATIVE when 'ffi' - require 'google/protobuf_native' + require 'google/protobuf_ffi' :FFI else - raise RuntimeError("Unsupport value of PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION environment variable `#{ENV['PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION']}`. Must be one of `native` or `ffi`. Defaults to `native`.") + raise RuntimeError("Unsupported value of PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION environment variable `#{ENV['PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION']}`. Must be one of `native` or `ffi`. Defaults to `native`.") end end end From fb395f843f52ba952c83a0b97a453049f0cda5d4 Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Fri, 6 Jan 2023 17:41:01 -0500 Subject: [PATCH 07/54] Fix exception raising by properly instantiating the exception. --- ruby/lib/google/protobuf.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ruby/lib/google/protobuf.rb b/ruby/lib/google/protobuf.rb index 29b367352f19..1dd898e3b44e 100644 --- a/ruby/lib/google/protobuf.rb +++ b/ruby/lib/google/protobuf.rb @@ -63,7 +63,7 @@ def self.decode_json(klass, json, options = {}) require 'google/protobuf_ffi' :FFI else - raise RuntimeError("Unsupported value of PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION environment variable `#{ENV['PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION']}`. Must be one of `native` or `ffi`. Defaults to `native`.") + raise RuntimeError.new("Unsupported value of PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION environment variable `#{ENV['PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION']}`. Must be one of `native` or `ffi`. Defaults to `native`.") end end end From d9767caf6b841ddd2ddbe53878cc0c4f0bb43dc7 Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Fri, 6 Jan 2023 17:42:09 -0500 Subject: [PATCH 08/54] Add check on `PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION` environment variable to handle FFI. --- ruby/google-protobuf.gemspec | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/ruby/google-protobuf.gemspec b/ruby/google-protobuf.gemspec index 14f334a0aecb..b7509b57e0ba 100644 --- a/ruby/google-protobuf.gemspec +++ b/ruby/google-protobuf.gemspec @@ -11,13 +11,24 @@ Gem::Specification.new do |s| s.metadata = { "source_code_uri" => "https://github.com/protocolbuffers/protobuf/tree/#{git_tag}/ruby" } s.require_paths = ["lib"] s.files = Dir.glob('lib/**/*.rb') - if RUBY_PLATFORM == "java" - s.platform = "java" - s.files += ["lib/google/protobuf_java.jar"] - else - s.files += Dir.glob('ext/**/*') - s.extensions= ["ext/google/protobuf_c/extconf.rb"] - s.add_development_dependency "rake-compiler-dock", "= 1.2.1" end + s.test_files = %w[tests/basic.rb tests/stress.rb tests/generated_code_test.rb] + case ENV['PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION'] + when nil, 'native', '' + if RUBY_PLATFORM == "java" + s.platform = "java" + s.files += ["lib/google/protobuf_java.jar"] + else + s.files += Dir.glob('ext/**/*') + s.extensions= ["ext/google/protobuf_c/extconf.rb"] + s.add_development_dependency "rake-compiler-dock", "= 1.2.1" + end + when 'ffi' + s.add_dependency "ffi", "~>1" + s.add_dependency "ffi-compiler", "~>1" + s.extensions = Dir.glob(%w[Rakefile ext/**/*.{c,h}]) + else + raise RuntimeError.new("Unsupported value of PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION environment variable `#{ENV['PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION']}`. If set, must be one of `native` or `ffi`. Defaults to `native`.") + end s.required_ruby_version = '>= 2.3' s.add_development_dependency "rake-compiler", "~> 1.1.0" s.add_development_dependency "test-unit", '~> 3.0', '>= 3.0.9' From 3792576927ad99b0ea497682bd116089cfc48d0d Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Fri, 6 Jan 2023 17:42:35 -0500 Subject: [PATCH 09/54] Add FFI implementation --- ruby/lib/google/protobuf/ffi/descriptor.rb | 765 ++++++++++++++++++ .../google/protobuf/ffi/descriptor_pool.rb | 83 ++ .../google/protobuf/ffi/enum_descriptor.rb | 192 +++++ ruby/lib/google/protobuf/ffi/ffi.rb | 371 +++++++++ .../google/protobuf/ffi/field_descriptor.rb | 326 ++++++++ .../google/protobuf/ffi/file_descriptor.rb | 65 ++ .../lib/google/protobuf/ffi/internal/arena.rb | 73 ++ .../google/protobuf/ffi/internal/convert.rb | 342 ++++++++ .../protobuf/ffi/internal/type_safety.rb | 48 ++ ruby/lib/google/protobuf/ffi/map.rb | 404 +++++++++ ruby/lib/google/protobuf/ffi/object_cache.rb | 94 +++ .../google/protobuf/ffi/oneof_descriptor.rb | 114 +++ .../lib/google/protobuf/ffi/repeated_field.rb | 516 ++++++++++++ 13 files changed, 3393 insertions(+) create mode 100644 ruby/lib/google/protobuf/ffi/descriptor.rb create mode 100644 ruby/lib/google/protobuf/ffi/descriptor_pool.rb create mode 100644 ruby/lib/google/protobuf/ffi/enum_descriptor.rb create mode 100644 ruby/lib/google/protobuf/ffi/ffi.rb create mode 100644 ruby/lib/google/protobuf/ffi/field_descriptor.rb create mode 100644 ruby/lib/google/protobuf/ffi/file_descriptor.rb create mode 100644 ruby/lib/google/protobuf/ffi/internal/arena.rb create mode 100644 ruby/lib/google/protobuf/ffi/internal/convert.rb create mode 100644 ruby/lib/google/protobuf/ffi/internal/type_safety.rb create mode 100644 ruby/lib/google/protobuf/ffi/map.rb create mode 100644 ruby/lib/google/protobuf/ffi/object_cache.rb create mode 100644 ruby/lib/google/protobuf/ffi/oneof_descriptor.rb create mode 100644 ruby/lib/google/protobuf/ffi/repeated_field.rb diff --git a/ruby/lib/google/protobuf/ffi/descriptor.rb b/ruby/lib/google/protobuf/ffi/descriptor.rb new file mode 100644 index 000000000000..fa30c1693146 --- /dev/null +++ b/ruby/lib/google/protobuf/ffi/descriptor.rb @@ -0,0 +1,765 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2022 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +module Google + module Protobuf + ## + # Message Descriptor - Descriptor for short. + class Descriptor + attr :descriptor_pool, :msg_class + include Enumerable + + # FFI Interface methods and setup + extend ::FFI::DataConverter + native_type ::FFI::Type::POINTER + + class << self + prepend Google::Protobuf::Internal::TypeSafety + + # @param value [Descriptor] Descriptor to convert to an FFI native type + # @param _ [Object] Unused + def to_native(value, _ = nil) + msg_def_ptr = value.nil? ? nil : value.instance_variable_get(:@msg_def) + return ::FFI::Pointer::NULL if msg_def_ptr.nil? + raise "Underlying msg_def was null!" if msg_def_ptr.null? + msg_def_ptr + end + + ## + # @param msg_def [::FFI::Pointer] MsgDef pointer to be wrapped + # @param _ [Object] Unused + def from_native(msg_def, _ = nil) + return nil if msg_def.nil? or msg_def.null? + # Calling get_message_file_def(msg_def) would create a cyclic + # dependency because FFI would then complain about passing an + # FFI::Pointer instance instead of a Descriptor. Instead, directly + # read the top of the MsgDef structure an extract the FileDef*. + # file_def = Google::Protobuf::FFI.get_message_file_def msg_def + message_def_struct = Google::Protobuf::FFI::Upb_MessageDef.new(msg_def) + file_def = message_def_struct[:file_def] + raise RuntimeError.new "FileDef is nil" if file_def.nil? + raise RuntimeError.new "FileDef is null" if file_def.null? + pool_def = Google::Protobuf::FFI.file_def_pool file_def + raise RuntimeError.new "PoolDef is nil" if pool_def.nil? + raise RuntimeError.new "PoolDef is null" if pool_def.null? + pool = Google::Protobuf::ObjectCache.get(pool_def) + raise "Cannot find pool in ObjectCache!" if pool.nil? + descriptor = pool.descriptor_class_by_def[msg_def.address] + if descriptor.nil? + pool.descriptor_class_by_def[msg_def.address] = private_constructor(msg_def, pool) + else + descriptor + end + end + end + + ## + # Great write up of this strategy: + # See https://blog.appsignal.com/2018/08/07/ruby-magic-changing-the-way-ruby-creates-objects.html + def self.new(*arguments, &block) + raise "Descriptor objects may not be created from Ruby." + end + + def to_s + inspect + end + + def inspect + "Descriptor - (not the message class) #{name}" + end + + def file_descriptor + @descriptor_pool.send(:get_file_descriptor, Google::Protobuf::FFI.get_message_file_def(self)) + end + + def name + @name ||= Google::Protobuf::FFI.get_message_fullname(self) + end + + def each_oneof &block + n = Google::Protobuf::FFI.oneof_count(self) + 0.upto(n-1) do |i| + yield(Google::Protobuf::FFI.get_oneof_by_index(self, i)) + end + nil + end + + def each &block + n = Google::Protobuf::FFI.field_count(self) + 0.upto(n-1) do |i| + yield(Google::Protobuf::FFI.get_field_by_index(self, i)) + end + nil + end + + def lookup(name) + Google::Protobuf::FFI.get_field_by_name(self, name, name.size) + end + + def lookup_oneof(name) + Google::Protobuf::FFI.get_oneof_by_name(self, name, name.size) + end + + def msgclass + @msg_class ||= build_message_class + end + + private + + extend Google::Protobuf::Internal::Convert + + def initialize(msg_def, descriptor_pool) + @msg_def = msg_def + @msg_class = nil + @descriptor_pool = descriptor_pool + end + + def self.private_constructor(msg_def, descriptor_pool) + instance = allocate + instance.send(:initialize, msg_def, descriptor_pool) + instance + end + + def wrapper? + if defined? @wrapper + @wrapper + else + @wrapper = case Google::Protobuf::FFI.get_well_known_type self + when :DoubleValue, :FloatValue, :Int64Value, :UInt64Value, :Int32Value, :UInt32Value, :StringValue, :BytesValue, :BoolValue + true + else + false + end + end + end + + def self.get_message(msg, descriptor, arena) + return nil if msg.nil? or msg.null? + message = ObjectCache.get(msg) + if message.nil? + message = descriptor.msgclass.send(:private_constructor, arena, msg: msg) + end + message + end + + def pool + @descriptor_pool + end + + def build_message_class + descriptor = self + Class.new(Google::Protobuf::const_get(:AbstractMessage)) do + @descriptor = descriptor + class << self + attr_accessor :descriptor + private + attr_accessor :oneof_field_names + include ::Google::Protobuf::Internal::Convert + end + + alias original_method_missing method_missing + def method_missing(method_name, *args) + method_missing_internal method_name, *args, mode: :method_missing + end + + def respond_to_missing?(method_name, include_private = false) + method_missing_internal(method_name, mode: :respond_to_missing?) || super + end + + ## + # Public constructor. Automatically allocates from a new Arena. + def self.new(initial_value = nil) + instance = allocate + instance.send(:initialize, initial_value) + instance + end + + def dup + duplicate = self.class.private_constructor(@arena) + mini_table = Google::Protobuf::FFI.get_mini_table(self.class.descriptor) + size = mini_table[:size] + duplicate.instance_variable_get(:@msg).write_string_length(@msg.read_string_length(size), size) + duplicate + end + alias clone dup + + def eql?(other) + return false unless self.class === other + encoding_options = Google::Protobuf::FFI::Upb_Encode_Deterministic | Google::Protobuf::FFI::Upb_Encode_SkipUnknown + temporary_arena = Google::Protobuf::FFI.create_arena + mini_table = Google::Protobuf::FFI.get_mini_table(self.class.descriptor) + size_one = ::FFI::MemoryPointer.new(:size_t, 1) + encoding_one = Google::Protobuf::FFI.encode_message(@msg, mini_table, encoding_options, temporary_arena, size_one) + size_two = ::FFI::MemoryPointer.new(:size_t, 1) + encoding_two = Google::Protobuf::FFI.encode_message(other.instance_variable_get(:@msg), mini_table, encoding_options, temporary_arena, size_two) + if encoding_one.null? or encoding_two.null? + raise ParseError.new "Error comparing messages" + end + size_one.read(:size_t) == size_two.read(:size_t) and Google::Protobuf::FFI.memcmp(encoding_one, encoding_two, size_one.read(:size_t)).zero? + end + alias == eql? + + def hash + encoding_options = Google::Protobuf::FFI::Upb_Encode_Deterministic | Google::Protobuf::FFI::Upb_Encode_SkipUnknown + temporary_arena = Google::Protobuf::FFI.create_arena + mini_table_ptr = Google::Protobuf::FFI.get_mini_table(self.class.descriptor) + size_ptr = ::FFI::MemoryPointer.new(:size_t, 1) + encoding = Google::Protobuf::FFI.encode_message(@msg, mini_table_ptr, encoding_options, temporary_arena, size_ptr) + if encoding.null? + raise ParseError.new "Error calculating hash" + end + Google::Protobuf::FFI.hash(encoding, size_ptr.read(:size_t), 0) + end + + def to_h + to_h_internal @msg, self.class.descriptor + end + + ## + # call-seq: + # Message.inspect => string + # + # Returns a human-readable string representing this message. It will be + # formatted as "". Each + # field's value is represented according to its own #inspect method. + def inspect + self.class.inspect_internal @msg + end + + def to_s + self.inspect + end + + ## + # call-seq: + # Message.[](index) => value + # Accesses a field's value by field name. The provided field name + # should be a string. + def [](name) + raise TypeError.new "Expected String for name but got #{name.class}" unless name.is_a? String + index_internal name + end + + ## + # call-seq: + # Message.[]=(index, value) + # Sets a field's value by field name. The provided field name should + # be a string. + # @param name [String] Name of the field to be set + # @param value [Object] Value to set the field to + def []=(name, value) + raise TypeError.new "Expected String for name but got #{name.class}" unless name.is_a? String + index_assign_internal(value, name: name) + end + + ## + # call-seq: + # MessageClass.decode(data, options) => message + # + # Decodes the given data (as a string containing bytes in protocol buffers wire + # format) under the interpretation given by this message class's definition + # and returns a message object with the corresponding field values. + # @param data [String] Binary string in Protobuf wire format to decode + # @param options [Hash] options for the decoder + # @option options [Integer] :recursion_limit Set to maximum decoding depth for message (default is 64) + def self.decode(data, options = {}) + raise ArgumentError.new "Expected hash arguments." unless options.is_a? Hash + raise ArgumentError.new "Expected string for binary protobuf data." unless data.is_a? String + decoding_options = 0 + depth = options[:recursion_limit] + + if depth.is_a? Numeric + decoding_options |= Google::Protobuf::FFI.decode_max_depth(depth.to_i) + end + + message = new + mini_table_ptr = Google::Protobuf::FFI.get_mini_table(message.class.descriptor) + status = Google::Protobuf::FFI.decode_message(data, data.bytesize, message.instance_variable_get(:@msg), mini_table_ptr, nil, decoding_options, message.instance_variable_get(:@arena)) + raise ParseError.new "Error occurred during parsing" unless status == :Ok + message + end + + ## + # call-seq: + # MessageClass.encode(msg, options) => bytes + # + # Encodes the given message object to its serialized form in protocol buffers + # wire format. + # @param options [Hash] options for the encoder + # @option options [Integer] :recursion_limit Set to maximum encoding depth for message (default is 64) + def self.encode(message, options = {}) + raise ArgumentError.new "Message of wrong type." unless message.is_a? self + raise ArgumentError.new "Expected hash arguments." unless options.is_a? Hash + + encoding_options = 0 + depth = options[:recursion_limit] + + if depth.is_a? Numeric + encoding_options |= Google::Protobuf::FFI.decode_max_depth(depth.to_i) + end + + encode_internal(message.instance_variable_get(:@msg), encoding_options) do |encoding, size, _| + if encoding.nil? or encoding.null? + raise RuntimeError.new "Exceeded maximum depth (possibly cycle)" + else + encoding.read_string_length(size).force_encoding("ASCII-8BIT").freeze + end + end + end + + ## + # all-seq: + # MessageClass.decode_json(data, options = {}) => message + # + # Decodes the given data (as a string containing bytes in protocol buffers wire + # format) under the interpretation given by this message class's definition + # and returns a message object with the corresponding field values. + # + # @param options [Hash] options for the decoder + # @option options [Boolean] :ignore_unknown_fields Set true to ignore unknown fields (default is to raise an error) + # @return [Message] + def self.decode_json(data, options = {}) + decoding_options = 0 + unless options.is_a? Hash + if options.respond_to? :to_h + options options.to_h + else + #TODO(jatl) can this error message be improve to include what was received? + raise ArgumentError.new "Expected hash arguments" + end + end + raise ArgumentError.new "Expected string for JSON data." unless data.is_a? String + raise RuntimeError.new "Cannot parse a wrapper directly" if descriptor.send(:wrapper?) + + if options[:ignore_unknown_fields] + decoding_options |= Google::Protobuf::FFI::Upb_JsonDecode_IgnoreUnknown + end + + message = new + pool_def = pool_def_from_message_definition(message.class.descriptor) + status = Google::Protobuf::FFI::Status.new + unless Google::Protobuf::FFI.json_decode_message(data, data.bytesize, message.instance_variable_get(:@msg), message.class.descriptor, pool_def, decoding_options, message.instance_variable_get(:@arena), status) + raise ParseError.new "Error occurred during parsing: #{Google::Protobuf::FFI.error_message(status)}" + end + message + end + + def self.encode_json(message, options = {}) + encoding_options = 0 + unless options.is_a? Hash + if options.respond_to? :to_h + options = options.to_h + else + #TODO(jatl) can this error message be improve to include what was received? + raise ArgumentError.new "Expected hash arguments" + end + end + + if options[:preserve_proto_fieldnames] + encoding_options |= Google::Protobuf::FFI::Upb_JsonEncode_UseProtoNames + end + if options[:emit_defaults] + encoding_options |= Google::Protobuf::FFI::Upb_JsonEncode_EmitDefaults + end + + buffer_size = 1024 + buffer = ::FFI::MemoryPointer.new(:char, buffer_size) + status = Google::Protobuf::FFI::Status.new + msg = message.instance_variable_get(:@msg) + pool_def = pool_def_from_message_definition(message.class.descriptor) + size = Google::Protobuf::FFI::json_encode_message(msg, message.class.descriptor, pool_def, encoding_options, buffer, buffer_size, status) + unless status[:ok] + raise ParseError.new "Error occurred during encoding: #{Google::Protobuf::FFI.error_message(status)}" + end + + if size >= buffer_size + buffer_size = size + 1 + buffer = ::FFI::MemoryPointer.new(:char, buffer_size) + status.clear + size = Google::Protobuf::FFI::json_encode_message(msg, message.class.descriptor, pool_def, encoding_options, buffer, buffer_size, status) + unless status[:ok] + raise ParseError.new "Error occurred during encoding: #{Google::Protobuf::FFI.error_message(status)}" + end + if size >= buffer_size + raise ParseError.new "Inconsistent JSON encoding sizes - was #{buffer_size - 1}, now #{size}" + end + end + + buffer.read_string_length(size).force_encoding("UTF-8").freeze + end + + @descriptor.each do |field_descriptor| + field_name = field_descriptor.name + unless instance_methods(true).include?(field_name.to_sym) + #TODO(jatl) - at a high level, dispatching to either + # index_internal or get_field would be logically correct, but slightly slower. + if field_descriptor.map? + define_method(field_name) do + mutable_message_value = Google::Protobuf::FFI.get_mutable_message @msg, field_descriptor, @arena + get_map_field(mutable_message_value[:map], field_descriptor) + end + elsif field_descriptor.repeated? + define_method(field_name) do + mutable_message_value = Google::Protobuf::FFI.get_mutable_message @msg, field_descriptor, @arena + get_repeated_field(mutable_message_value[:array], field_descriptor) + end + elsif field_descriptor.sub_message? + define_method(field_name) do + return nil unless Google::Protobuf::FFI.get_message_has @msg, field_descriptor + mutable_message = Google::Protobuf::FFI.get_mutable_message @msg, field_descriptor, @arena + sub_message = mutable_message[:msg] + sub_message_def = Google::Protobuf::FFI.get_subtype_as_message(field_descriptor) + Descriptor.send(:get_message, sub_message, sub_message_def, @arena) + end + else + c_type = field_descriptor.send(:c_type) + if c_type == :enum + define_method(field_name) do + message_value = Google::Protobuf::FFI.get_message_value @msg, field_descriptor + convert_upb_to_ruby message_value, c_type, Google::Protobuf::FFI.get_subtype_as_enum(field_descriptor) + end + else + define_method(field_name) do + message_value = Google::Protobuf::FFI.get_message_value @msg, field_descriptor + convert_upb_to_ruby message_value, c_type + end + end + end + define_method("#{field_name}=") do |value| + index_assign_internal(value, field_descriptor: field_descriptor) + end + define_method("clear_#{field_name}") do + clear_internal(field_descriptor) + end + if field_descriptor.type == :enum + define_method("#{field_name}_const") do + if field_descriptor.repeated? + return_value = [] + get_field(field_descriptor).send(:each_msg_val) do |msg_val| + return_value << msg_val[:int32_val] + end + return_value + else + message_value = Google::Protobuf::FFI.get_message_value @msg, field_descriptor + message_value[:int32_val] + end + end + end + if !field_descriptor.repeated? and field_descriptor.wrapper? + define_method("#{field_name}_as_value") do + get_field(field_descriptor, unwrap: true) + end + define_method("#{field_name}_as_value=") do |value| + if value.nil? + clear_internal(field_descriptor) + else + index_assign_internal(value, field_descriptor: field_descriptor, wrap: true) + end + end + end + if field_descriptor.has_presence? + if field_descriptor.sub_message? or field_descriptor.send(:real_containing_oneof).nil? or + Google::Protobuf::FFI.message_def_syntax(Google::Protobuf::FFI.get_containing_message_def(field_descriptor)) == :Proto2 + define_method("has_#{field_name}?") do + Google::Protobuf::FFI.get_message_has(@msg, field_descriptor) + end + end + end + end + end + + @oneof_field_names = [] + + @descriptor.each_oneof do |oneof_descriptor| + field_name = oneof_descriptor.name.to_sym + @oneof_field_names << field_name + unless instance_methods(true).include?(field_name) + define_method(field_name) do + field_descriptor = Google::Protobuf::FFI.get_message_which_oneof(@msg, oneof_descriptor) + if field_descriptor.nil? + return + else + return field_descriptor.name.to_sym + end + end + define_method("clear_#{field_name}") do + field_descriptor = Google::Protobuf::FFI.get_message_which_oneof(@msg, oneof_descriptor) + unless field_descriptor.nil? + clear_internal(field_descriptor) + end + end + define_method("has_#{field_name}?") do + !Google::Protobuf::FFI.get_message_which_oneof(@msg, oneof_descriptor).nil? + end + end + end + + private + # Implementation details below are subject to breaking changes without + # warning and are intended for use only within the gem. + + def method_missing_internal(method_name, *args, mode: nil) + raise ArgumentError.new "method_missing_internal called with invalid mode #{mode.inspect}" unless [:respond_to_missing?, :method_missing].include? mode + + #TODO(jatl) not being allowed is not the same thing as not responding, but this is needed to pass tests + if method_name.to_s.end_with? '=' + if self.class.send(:oneof_field_names).include? method_name.to_s[0..-2].to_sym + return false if mode == :respond_to_missing? + raise RuntimeError.new "Oneof accessors are read-only." + end + end + + original_method_missing(method_name, *args) if mode == :method_missing + end + + def self.private_constructor(arena, msg: nil, initial_value: nil) + instance = allocate + instance.send(:initialize, initial_value, arena, msg) + instance + end + + def clear_internal(field_def) + raise FrozenError.new "can't modify frozen #{self.class}" if frozen? + Google::Protobuf::FFI.clear_message_field(@msg, field_def) + end + + def index_internal(name) + field_descriptor = self.class.descriptor.lookup(name) + get_field field_descriptor unless field_descriptor.nil? + end + + #TODO(jatl) - well known types keeps us on our toes by overloading methods. + # How much of the public API needs to be defended? + def index_assign_internal(value, name: nil, field_descriptor: nil, wrap: false) + raise FrozenError.new "can't modify frozen #{self.class}" if frozen? + if field_descriptor.nil? + field_descriptor = self.class.descriptor.lookup(name) + if field_descriptor.nil? + raise ArgumentError.new "Unknown field: #{name}" + end + end + unless field_descriptor.send :set_value_on_message, value, @msg, @arena, wrap: wrap + raise RuntimeError.new "allocation failed" + end + end + + ## + # @param initial_value [Object] initial value of this Message + # @param arena [Arena] Optional; Arena where this message will be allocated + # @param msg [::FFI::Pointer] Optional; value of this message + def initialize(initial_value = nil, arena = nil, msg = nil) + @arena = arena || Google::Protobuf::FFI.create_arena + @msg = msg || Google::Protobuf::FFI.new_message_from_def(self.class.descriptor, @arena) + + unless initial_value.nil? + raise ArgumentError.new "Expected hash arguments or message, not #{initial_value.class}" unless initial_value.respond_to? :each + + field_def_ptr = ::FFI::MemoryPointer.new :pointer + oneof_def_ptr = ::FFI::MemoryPointer.new :pointer + + initial_value.each do |key, value| + raise ArgumentError.new "Expected string or symbols as hash keys when initializing proto from hash." unless [String, Symbol].include? key.class + + unless Google::Protobuf::FFI.find_msg_def_by_name self.class.descriptor, key.to_s, key.to_s.bytesize, field_def_ptr, oneof_def_ptr + raise ArgumentError.new "Unknown field name '#{key}' in initialization map entry." + end + raise NotImplementedError.new "Haven't added oneofsupport yet" unless oneof_def_ptr.get_pointer(0).null? + raise NotImplementedError.new "Expected a field def" if field_def_ptr.get_pointer(0).null? + + field_descriptor = FieldDescriptor.from_native field_def_ptr.get_pointer(0) + + next if value.nil? + if field_descriptor.map? + index_assign_internal(Google::Protobuf::Map.send(:construct_for_field, field_descriptor, @arena, value: value), name: key.to_s) + elsif field_descriptor.repeated? + index_assign_internal(RepeatedField.send(:construct_for_field, field_descriptor, @arena, values: value), name: key.to_s) + # TODO - Is it OK not trap this and just let []= convert it for me?? + # elsif field_descriptor.sub_message? + # raise NotImplementedError + else + index_assign_internal(value, name: key.to_s) + end + end + end + + # Should always be the last expression of the initializer to avoid + # leaking references to this object before construction is complete. + Google::Protobuf::ObjectCache.add @msg, self + end + + include Google::Protobuf::Internal::Convert + + + def self.inspect_field(field_descriptor, c_type, message_value) + if field_descriptor.sub_message? + sub_msg_descriptor = Google::Protobuf::FFI.get_subtype_as_message(field_descriptor) + sub_msg_descriptor.msgclass.send(:inspect_internal, message_value[:msg_val]) + else + convert_upb_to_ruby(message_value, c_type, field_descriptor.subtype).inspect + end + end + + # @param field_def [::FFI::Pointer] Pointer to the Message + def self.inspect_internal(msg) + field_output = [] + descriptor.each do |field_descriptor| + next if field_descriptor.has_presence? && !Google::Protobuf::FFI.get_message_has(msg, field_descriptor) + if field_descriptor.map? + # TODO(jatl) Adapted - from map#each_msg_val and map#inspect- can this be refactored to reduce echo without introducing a arena allocation? + message_descriptor = field_descriptor.subtype + key_field_def = Google::Protobuf::FFI.get_field_by_number(message_descriptor, 1) + key_field_type = Google::Protobuf::FFI.get_type(key_field_def) + + value_field_def = Google::Protobuf::FFI.get_field_by_number(message_descriptor, 2) + value_field_type = Google::Protobuf::FFI.get_type(value_field_def) + + message_value = Google::Protobuf::FFI.get_message_value(msg, field_descriptor) + iter = ::FFI::MemoryPointer.new(:size_t, 1) + iter.write(:size_t, Google::Protobuf::FFI::Upb_Map_Begin) + key_value_pairs = [] + while Google::Protobuf::FFI.map_next(message_value[:map_val], iter) do + iter_size_t = iter.read(:size_t) + key_message_value = Google::Protobuf::FFI.map_key(message_value[:map_val], iter_size_t) + value_message_value = Google::Protobuf::FFI.map_value(message_value[:map_val], iter_size_t) + key_string = convert_upb_to_ruby(key_message_value, key_field_type).inspect + value_string = inspect_field(value_field_def, value_field_type, value_message_value) + key_value_pairs << "#{key_string}=>#{value_string}" + end + field_output << "#{field_descriptor.name}: {#{key_value_pairs.join(", ")}}" + elsif field_descriptor.repeated? + # TODO(jatl) Adapted - from repeated_field#each - can this be refactored to reduce echo? + repeated_field_output = [] + message_value = Google::Protobuf::FFI.get_message_value(msg, field_descriptor) + array = message_value[:array_val] + n = array.null? ? 0 : Google::Protobuf::FFI.array_size(array) + 0.upto(n - 1) do |i| + element = Google::Protobuf::FFI.get_msgval_at(array, i) + repeated_field_output << inspect_field(field_descriptor, field_descriptor.send(:c_type), element) + end + field_output << "#{field_descriptor.name}: [#{repeated_field_output.join(", ")}]" + else + message_value = Google::Protobuf::FFI.get_message_value msg, field_descriptor + rendered_value = inspect_field(field_descriptor, field_descriptor.send(:c_type), message_value) + field_output << "#{field_descriptor.name}: #{rendered_value}" + end + end + "<#{name}: #{field_output.join(', ')}>" + end + + ## + # Gets a field of this message identified by the argument definition. + # + # @param field [FieldDescriptor] Descriptor of the field to get + def get_field(field, unwrap: false) + if field.map? + mutable_message_value = Google::Protobuf::FFI.get_mutable_message @msg, field, @arena + get_map_field(mutable_message_value[:map], field) + elsif field.repeated? + mutable_message_value = Google::Protobuf::FFI.get_mutable_message @msg, field, @arena + get_repeated_field(mutable_message_value[:array], field) + elsif field.sub_message? + return nil unless Google::Protobuf::FFI.get_message_has @msg, field + sub_message_def = Google::Protobuf::FFI.get_subtype_as_message(field) + if unwrap + if field.has?(self) + wrapper_message_value = Google::Protobuf::FFI.get_message_value @msg, field + fields = Google::Protobuf::FFI.field_count(sub_message_def) + raise "Sub message has #{fields} fields! Expected exactly 1." unless fields == 1 + value_field_def = Google::Protobuf::FFI.get_field_by_number sub_message_def, 1 + message_value = Google::Protobuf::FFI.get_message_value wrapper_message_value[:msg_val], value_field_def + convert_upb_to_ruby message_value, Google::Protobuf::FFI.get_c_type(value_field_def) + else + nil + end + else + mutable_message = Google::Protobuf::FFI.get_mutable_message @msg, field, @arena + sub_message = mutable_message[:msg] + Descriptor.send(:get_message, sub_message, sub_message_def, @arena) + end + else + c_type = field.send(:c_type) + message_value = Google::Protobuf::FFI.get_message_value @msg, field + if c_type == :enum + convert_upb_to_ruby message_value, c_type, Google::Protobuf::FFI.get_subtype_as_enum(field) + else + convert_upb_to_ruby message_value, c_type + end + end + end + + ## + # @param array [::FFI::Pointer] Pointer to the Array + # @param field [Google::Protobuf::FieldDescriptor] Type of the repeated field + def get_repeated_field(array, field) + return nil if array.nil? or array.null? + repeated_field = ObjectCache.get(array) + if repeated_field.nil? + repeated_field = RepeatedField.send(:construct_for_field, field, @arena, array: array) + end + repeated_field + end + + + ## + # @param map [::FFI::Pointer] Pointer to the Map + # @param field [Google::Protobuf::FieldDescriptor] Type of the map field + def get_map_field(map, field) + return nil if map.nil? or map.null? + map_field = ObjectCache.get(map) + if map_field.nil? + map_field = Google::Protobuf::Map.send(:construct_for_field, field, @arena, map: map) + end + map_field + end + + def self.deep_copy(msg, arena = nil) + arena ||= Google::Protobuf::FFI.create_arena + encode_internal(msg) do |encoding, size, mini_table_ptr| + message = private_constructor(arena) + if encoding.nil? or encoding.null? or Google::Protobuf::FFI.decode_message(encoding, size, message.instance_variable_get(:@msg), mini_table_ptr, nil, 0, arena) != :Ok + raise ParseError.new "Error occurred copying proto" + end + message + end + end + + def self.encode_internal(msg, encoding_options = 0) + temporary_arena = Google::Protobuf::FFI.create_arena + + mini_table_ptr = Google::Protobuf::FFI.get_mini_table(descriptor) + size_ptr = ::FFI::MemoryPointer.new(:size_t, 1) + encoding = Google::Protobuf::FFI.encode_message(msg, mini_table_ptr, encoding_options, temporary_arena, size_ptr) + yield encoding, size_ptr.read(:size_t), mini_table_ptr + end + end + end + end + end +end diff --git a/ruby/lib/google/protobuf/ffi/descriptor_pool.rb b/ruby/lib/google/protobuf/ffi/descriptor_pool.rb new file mode 100644 index 000000000000..c12e7293f374 --- /dev/null +++ b/ruby/lib/google/protobuf/ffi/descriptor_pool.rb @@ -0,0 +1,83 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2022 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +module Google + module Protobuf + class DescriptorPool + attr :descriptor_pool + attr_accessor :descriptor_class_by_def + + def initialize + @descriptor_pool = ::FFI::AutoPointer.new(Google::Protobuf::FFI.create_DescriptorPool, Google::Protobuf::FFI.method(:free_DescriptorPool)) + @descriptor_class_by_def = {} + + # Should always be the last expression of the initializer to avoid + # leaking references to this object before construction is complete. + Google::Protobuf::ObjectCache.add @descriptor_pool, self + end + + def add_serialized_file(file_contents) + # Allocate memory sized to file_contents + memBuf = ::FFI::MemoryPointer.new(:char, file_contents.bytesize) + # Insert the data + memBuf.put_bytes(0, file_contents) + file_descriptor_proto = Google::Protobuf::FFI.parse memBuf, file_contents.bytesize + raise ArgumentError.new("Unable to parse FileDescriptorProto") if file_descriptor_proto.null? + + status = Google::Protobuf::FFI::Status.new + file_descriptor = Google::Protobuf::FFI.add_serialized_file @descriptor_pool, file_descriptor_proto, status + if file_descriptor.null? + raise TypeError.new("Unable to build file to DescriptorPool: #{Google::Protobuf::FFI.error_message(status)}") + else + @descriptor_class_by_def[file_descriptor.address] = FileDescriptor.new file_descriptor, self + end + end + + def lookup name + Google::Protobuf::FFI.lookup_msg(@descriptor_pool, name) || + Google::Protobuf::FFI.lookup_enum(@descriptor_pool, name) + end + + def self.generated_pool + @@generated_pool ||= DescriptorPool.new + end + + private + + # Implementation details below are subject to breaking changes without + # warning and are intended for use only within the gem. + + def get_file_descriptor file_def + return nil if file_def.null? + @descriptor_class_by_def[file_def.address] ||= FileDescriptor.new(file_def, self) + end + end + end +end diff --git a/ruby/lib/google/protobuf/ffi/enum_descriptor.rb b/ruby/lib/google/protobuf/ffi/enum_descriptor.rb new file mode 100644 index 000000000000..a36f2fe07ef6 --- /dev/null +++ b/ruby/lib/google/protobuf/ffi/enum_descriptor.rb @@ -0,0 +1,192 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2022 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +module Google + module Protobuf + class EnumDescriptor + attr :descriptor_pool, :enum_def + include Enumerable + + # FFI Interface methods and setup + extend ::FFI::DataConverter + native_type ::FFI::Type::POINTER + + class << self + prepend Google::Protobuf::Internal::TypeSafety + + # @param value [Arena] Arena to convert to an FFI native type + # @param _ [Object] Unused + def to_native(value, _) + value.instance_variable_get(:@enum_def) || ::FFI::Pointer::NULL + end + + ## + # @param enum_def [::FFI::Pointer] EnumDef pointer to be wrapped + # @param _ [Object] Unused + def from_native(enum_def, _) + return nil if enum_def.nil? or enum_def.null? + # Calling get_message_file_def(enum_def) would create a cyclic + # dependency because FFI would then complain about passing an + # FFI::Pointer instance instead of a Descriptor. Instead, directly + # read the top of the MsgDef structure an extract the FileDef*. + # file_def = Google::Protobuf::FFI.get_message_file_def enum_def + enum_def_struct = Google::Protobuf::FFI::Upb_EnumDef.new(enum_def) + file_def = enum_def_struct[:file_def] + raise RuntimeError.new "FileDef is nil" if file_def.nil? + raise RuntimeError.new "FileDef is null" if file_def.null? + pool_def = Google::Protobuf::FFI.file_def_pool file_def + raise RuntimeError.new "PoolDef is nil" if pool_def.nil? + raise RuntimeError.new "PoolDef is null" if pool_def.null? + pool = Google::Protobuf::ObjectCache.get(pool_def) + raise "Cannot find pool in ObjectCache!" if pool.nil? + descriptor = pool.descriptor_class_by_def[enum_def.address] + if descriptor.nil? + pool.descriptor_class_by_def[enum_def.address] = private_constructor(enum_def, pool) + else + descriptor + end + end + end + + def self.new(*arguments, &block) + raise "Descriptor objects may not be created from Ruby." + end + + def file_descriptor + @descriptor_pool.send(:get_file_descriptor, Google::Protobuf::FFI.get_enum_file_descriptor(self)) + end + + def name + Google::Protobuf::FFI.get_enum_fullname(self) + end + + def to_s + inspect + end + + def inspect + "#{self.class.name}: #{name}" + end + + def lookup_name(name) + self.class.send(:lookup_name, self, name) + end + + def lookup_value(number) + self.class.send(:lookup_value, self, number) + end + + def each &block + n = Google::Protobuf::FFI.enum_value_count(self) + 0.upto(n-1) do |i| + enum_value = Google::Protobuf::FFI.enum_value_by_index(self, i) + yield(Google::Protobuf::FFI.enum_name(enum_value).to_sym, Google::Protobuf::FFI.enum_number(enum_value)) + end + nil + end + + def enummodule + if @module.nil? + @module = build_enum_module + end + @module + end + + private + + def initialize(enum_def, descriptor_pool) + @descriptor_pool = descriptor_pool + @enum_def = enum_def + @module = nil + end + + def self.private_constructor(enum_def, descriptor_pool) + instance = allocate + instance.send(:initialize, enum_def, descriptor_pool) + instance + end + + def self.lookup_value(enum_def, number) + enum_value = Google::Protobuf::FFI.enum_value_by_number(enum_def, number) + if enum_value.null? + nil + else + Google::Protobuf::FFI.enum_name(enum_value).to_sym + end + end + + def self.lookup_name(enum_def, name) + enum_value = Google::Protobuf::FFI.enum_value_by_name(enum_def, name.to_s, name.size) + if enum_value.null? + nil + else + Google::Protobuf::FFI.enum_number(enum_value) + end + end + + def definition + @enum_def + end + + def build_enum_module + descriptor = self + dynamic_module = Module.new do + @descriptor = descriptor + class << self + attr_accessor :descriptor + end + + def self.lookup(number) + descriptor.lookup_value number + end + def self.resolve(name) + descriptor.lookup_name name + end + + private + def definition + self.class.send(:descriptor) + end + + end + self.each do |name, value| + if name[0] < 'A' || name[0] > 'Z' + warn( + "Enum value '#{name}' does not start with an uppercase letter " + + "as is required for Ruby constants.") + else + dynamic_module.const_set(name.to_sym, value) + end + end + dynamic_module + end + end + end +end diff --git a/ruby/lib/google/protobuf/ffi/ffi.rb b/ruby/lib/google/protobuf/ffi/ffi.rb new file mode 100644 index 000000000000..9a9ac0639420 --- /dev/null +++ b/ruby/lib/google/protobuf/ffi/ffi.rb @@ -0,0 +1,371 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2022 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +require 'ffi' +require 'ffi-compiler/loader' + +module Google + module Protobuf + class FFI + extend ::FFI::Library + # Workaround for Bazel's use of symlinks + JRuby's __FILE__ and `caller` + # that resolves them. + if ENV['BAZEL'] == 'true' + ffi_lib ::FFI::Compiler::Loader.find 'protobuf_c', ENV['PWD'] + else + ffi_lib ::FFI::Compiler::Loader.find 'protobuf_c' + end + + + Arena = Google::Protobuf::Internal::Arena + + # Partial definitions of the top of structs used for bootstrapping FFI. + class Upb_MessageDef < ::FFI::Struct + layout :opts, :pointer, + :mini_table, :pointer, + :file_def, :pointer + end + class Upb_EnumDef < ::FFI::Struct + layout :opts, :pointer, + :mini_table, :pointer, + :file_def, :pointer + end + class Upb_FieldDef < ::FFI::Struct + layout :opts, :pointer, + :file_def, :pointer + end + class Upb_OneofDef < ::FFI::Struct + layout :opts, :pointer, + :parent, :pointer + end + + MessageDef = Google::Protobuf::Descriptor + EnumDef = Google::Protobuf::EnumDescriptor + FieldDef = Google::Protobuf::FieldDescriptor + OneofDef = Google::Protobuf::OneofDescriptor + + typedef :pointer, :Array + typedef :pointer, :DefPool + typedef :pointer, :EnumValueDef + typedef :pointer, :ExtensionRegistry + typedef :pointer, :FieldDefPointer + typedef :pointer, :FileDef + typedef :pointer, :FileDescriptorProto + typedef :pointer, :Map + typedef :pointer, :Message # Instances of a message + typedef :pointer, :OneofDefPointer + typedef :pointer, :binary_string + if ::FFI::Platform::ARCH == "aarch64" + typedef :u_int8_t, :uint8_t + typedef :u_int16_t, :uint16_t + typedef :u_int32_t, :uint32_t + typedef :u_int64_t, :uint64_t + end + + FieldType = enum( + :double, 1, + :float, + :int64, + :uint64, + :int32, + :fixed64, + :fixed32, + :bool, + :string, + :group, + :message, + :bytes, + :uint32, + :enum, + :sfixed32, + :sfixed64, + :sint32, + :sint64 + ) + + CType = enum( + :bool, 1, + :float, + :int32, + :uint32, + :enum, + :message, + :double, + :int64, + :uint64, + :string, + :bytes + ) + + Label = enum( + :optional, 1, + :required, + :repeated + ) + + class StringView < ::FFI::Struct + layout :data, :pointer, + :size, :size_t + end + + class MessageValue < ::FFI::Union + layout :bool_val, :bool, + :float_val, :float, + :double_val, :double, + :int32_val, :int32_t, + :int64_val, :int64_t, + :uint32_val, :uint32_t, + :uint64_val,:uint64_t, + :map_val, :pointer, + :msg_val, :pointer, + :array_val,:pointer, + :str_val, StringView + end + + class MutableMessageValue < ::FFI::Union + layout :map, :Map, + :msg, :Message, + :array, :Array + end + + Syntax = enum( + :Proto2, 2, + :Proto3 + ) + + # All the different kind of well known type messages. For simplicity of check, + # number wrappers and string wrappers are grouped together. Make sure the + # order and merber of these groups are not changed. + + WellKnown = enum( + :Unspecified, + :Any, + :FieldMask, + :Duration, + :Timestamp, + # number wrappers + :DoubleValue, + :FloatValue, + :Int64Value, + :UInt64Value, + :Int32Value, + :UInt32Value, + # string wrappers + :StringValue, + :BytesValue, + :BoolValue, + :Value, + :ListValue, + :Struct + ) + + DecodeStatus = enum( + :Ok, + :Malformed, # Wire format was corrupt + :OutOfMemory, # Arena alloc failed + :BadUtf8, # String field had bad UTF-8 + :MaxDepthExceeded, # Exceeded UPB_DECODE_MAXDEPTH + + # CheckRequired failed, but the parse otherwise succeeded. + :MissingRequired, + ) + + class MiniTable < ::FFI::Struct + layout :subs, :pointer, + :fields, :pointer, + :size, :uint16_t, + :field_count, :uint16_t, + :ext, :uint8_t, # upb_ExtMode, declared as uint8_t so sizeof(ext) == 1 + :dense_below, :uint8_t, + :table_mask, :uint8_t, + :required_count, :uint8_t # Required fields have the lowest hasbits. + # To statically initialize the tables of variable length, we need a flexible + # array member, and we need to compile in gnu99 mode (constant initialization + # of flexible array members is a GNU extension, not in C99 unfortunately. */ + # _upb_FastTable_Entry fasttable[]; + end + + ## Map + Upb_Map_Begin = -1 + + ## Encoding Status + Upb_Status_MaxMessage = 127 + Upb_Encode_Deterministic = 1 + Upb_Encode_SkipUnknown = 2 + + ## JSON Encoding options + # When set, emits 0/default values. TODO(haberman): proto3 only? + Upb_JsonEncode_EmitDefaults = 1 + # When set, use normal (snake_case) field names instead of JSON (camelCase) names. + Upb_JsonEncode_UseProtoNames = 2 + + ## JSON Decoding options + Upb_JsonDecode_IgnoreUnknown = 1 + + class Status < ::FFI::Struct + layout :ok, :bool, + :msg, [:char, Upb_Status_MaxMessage] + + def initialize + super + FFI.clear self + end + end + + # Arena + attach_function :create_arena, :Arena_create, [], Arena + attach_function :fuse_arena, :upb_Arena_Fuse, [Arena, Arena], :bool + # Argument takes a :pointer rather than a typed Arena here due to + # implementation details of FFI::AutoPointer. + attach_function :free_arena, :upb_Arena_Free, [:pointer], :void + attach_function :arena_malloc, :upb_Arena_FastMalloc, [Arena, :size_t], :pointer + + # Array + attach_function :append_array, :upb_Array_Append, [:Array, MessageValue.by_value, Arena], :bool + attach_function :get_msgval_at,:upb_Array_Get, [:Array, :size_t], MessageValue.by_value + attach_function :create_array, :upb_Array_New, [Arena, CType], :Array + attach_function :array_resize, :upb_Array_Resize, [:Array, :size_t, Arena], :bool + attach_function :array_set, :upb_Array_Set, [:Array, :size_t, MessageValue.by_value], :void + attach_function :array_size, :upb_Array_Size, [:Array], :size_t + + # DefPool + attach_function :add_serialized_file, :upb_DefPool_AddFile, [:DefPool, :FileDescriptorProto, Status.by_ref], :FileDef + attach_function :free_DescriptorPool, :upb_DefPool_Free, [:DefPool], :void + attach_function :create_DescriptorPool, :upb_DefPool_New, [], :DefPool + attach_function :lookup_enum, :upb_DefPool_FindEnumByName, [:DefPool, :string], EnumDef + attach_function :lookup_msg, :upb_DefPool_FindMessageByName, [:DefPool, :string], MessageDef + + # EnumDescriptor + attach_function :get_enum_file_descriptor, :upb_EnumDef_File, [EnumDef], :FileDef + attach_function :enum_value_by_name, :upb_EnumDef_FindValueByNameWithSize,[EnumDef, :string, :size_t], :EnumValueDef + attach_function :enum_value_by_number, :upb_EnumDef_FindValueByNumber, [EnumDef, :int], :EnumValueDef + attach_function :get_enum_fullname, :upb_EnumDef_FullName, [EnumDef], :string + attach_function :enum_value_by_index, :upb_EnumDef_Value, [EnumDef, :int], :EnumValueDef + attach_function :enum_value_count, :upb_EnumDef_ValueCount, [EnumDef], :int + attach_function :enum_name, :upb_EnumValueDef_Name, [:EnumValueDef], :string + attach_function :enum_number, :upb_EnumValueDef_Number, [:EnumValueDef], :int + + # FileDescriptor + attach_function :file_def_name, :upb_FileDef_Name, [:FileDef], :string + attach_function :file_def_syntax, :upb_FileDef_Syntax, [:FileDef], Syntax + attach_function :file_def_pool, :upb_FileDef_Pool, [:FileDef], :DefPool + + # FileDescriptorProto + attach_function :parse, :FileDescriptorProto_parse, [:binary_string, :size_t], :FileDescriptorProto + + # FieldDescriptor + attach_function :get_containing_message_def, :upb_FieldDef_ContainingType, [FieldDescriptor], MessageDef + attach_function :get_c_type, :upb_FieldDef_CType, [FieldDescriptor], CType + attach_function :get_default, :upb_FieldDef_Default, [FieldDescriptor], MessageValue.by_value + attach_function :get_subtype_as_enum, :upb_FieldDef_EnumSubDef, [FieldDescriptor], EnumDef + attach_function :get_has_presence, :upb_FieldDef_HasPresence, [FieldDescriptor], :bool + attach_function :is_map, :upb_FieldDef_IsMap, [FieldDescriptor], :bool + attach_function :is_repeated, :upb_FieldDef_IsRepeated, [FieldDescriptor], :bool + attach_function :is_sub_message, :upb_FieldDef_IsSubMessage, [FieldDescriptor], :bool + attach_function :get_json_name, :upb_FieldDef_JsonName, [FieldDescriptor], :string + attach_function :get_label, :upb_FieldDef_Label, [FieldDescriptor], Label + attach_function :get_subtype_as_message, :upb_FieldDef_MessageSubDef, [FieldDescriptor], MessageDef + attach_function :get_full_name, :upb_FieldDef_Name, [FieldDescriptor], :string + attach_function :get_number, :upb_FieldDef_Number, [FieldDescriptor], :uint32_t + attach_function :real_containing_oneof, :upb_FieldDef_RealContainingOneof,[FieldDescriptor], OneofDef + attach_function :get_type, :upb_FieldDef_Type, [FieldDescriptor], FieldType + + # Map + attach_function :map_clear, :upb_Map_Clear, [:Map], :void + attach_function :map_delete, :upb_Map_Delete, [:Map, MessageValue.by_value], :bool + attach_function :map_get, :upb_Map_Get, [:Map, MessageValue.by_value, MessageValue.by_ref], :bool + attach_function :create_map, :upb_Map_New, [Arena, CType, CType], :Map + attach_function :map_size, :upb_Map_Size, [:Map], :size_t + attach_function :map_set, :upb_Map_Set, [:Map, MessageValue.by_value, MessageValue.by_value, Arena], :bool + + # MapIterator + attach_function :map_next, :upb_MapIterator_Next, [:Map, :pointer], :bool + attach_function :map_done, :upb_MapIterator_Done, [:Map, :size_t], :bool + attach_function :map_key, :upb_MapIterator_Key, [:Map, :size_t], MessageValue.by_value + attach_function :map_value, :upb_MapIterator_Value, [:Map, :size_t], MessageValue.by_value + + # MessageDef + attach_function :new_message_from_def, :upb_Message_New, [MessageDef, Arena], :Message + attach_function :get_field_by_index, :upb_MessageDef_Field, [MessageDef, :int], FieldDescriptor + attach_function :field_count, :upb_MessageDef_FieldCount, [MessageDef], :int + attach_function :get_message_file_def, :upb_MessageDef_File, [MessageDef], :FileDef + attach_function :get_field_by_name, :upb_MessageDef_FindFieldByNameWithSize, [MessageDef, :string, :size_t], FieldDescriptor + attach_function :get_field_by_number, :upb_MessageDef_FindFieldByNumber, [MessageDef, :uint32_t], FieldDescriptor + attach_function :get_oneof_by_name, :upb_MessageDef_FindOneofByNameWithSize, [MessageDef, :string, :size_t], OneofDef + attach_function :get_message_fullname, :upb_MessageDef_FullName, [MessageDef], :string + attach_function :get_mini_table, :upb_MessageDef_MiniTable, [MessageDef], MiniTable.ptr + attach_function :get_oneof_by_index, :upb_MessageDef_Oneof, [MessageDef, :int], OneofDef + attach_function :oneof_count, :upb_MessageDef_OneofCount, [MessageDef], :int + attach_function :get_well_known_type, :upb_MessageDef_WellKnownType, [MessageDef], WellKnown + attach_function :message_def_syntax, :upb_MessageDef_Syntax, [MessageDef], Syntax + attach_function :find_msg_def_by_name, :upb_MessageDef_FindByNameWithSize, [MessageDef, :string, :size_t, :FieldDefPointer, :OneofDefPointer], :bool + + # Message + attach_function :clear_message_field, :upb_Message_ClearField, [:Message, FieldDescriptor], :void + attach_function :get_message_value, :upb_Message_Get, [:Message, FieldDescriptor], MessageValue.by_value + attach_function :get_message_has, :upb_Message_Has, [:Message, FieldDescriptor], :bool + attach_function :set_message_field, :upb_Message_Set, [:Message, FieldDescriptor, MessageValue.by_value, Arena], :bool + attach_function :encode_message, :upb_Encode, [:Message, MiniTable.by_ref, :size_t, Arena, :pointer], :binary_string + attach_function :json_decode_message, :upb_JsonDecode, [:binary_string, :size_t, :Message, MessageDef, :DefPool, :int, Arena, Status.by_ref], :bool + attach_function :json_encode_message, :upb_JsonEncode, [:Message, MessageDef, :DefPool, :int, :binary_string, :size_t, Status.by_ref], :size_t + attach_function :decode_message, :upb_Decode, [:binary_string, :size_t, :Message, MiniTable.by_ref, :ExtensionRegistry, :int, Arena], DecodeStatus + attach_function :get_mutable_message, :upb_Message_Mutable, [:Message, FieldDescriptor, Arena], MutableMessageValue.by_value + attach_function :get_message_which_oneof, :upb_Message_WhichOneof, [:Message, OneofDef], FieldDescriptor + attach_function :message_discard_unknown, :upb_Message_DiscardUnknown, [:Message, MessageDef, :int], :bool + + # MessageValue + attach_function :message_value_equal, :Msgval_IsEqual, [MessageValue.by_value, MessageValue.by_value, CType, MessageDef], :bool + attach_function :message_value_hash, :Msgval_GetHash, [MessageValue.by_value, CType, MessageDef, :uint64_t], :uint64_t + + # OneofDescriptor + attach_function :get_oneof_name, :upb_OneofDef_Name, [OneofDef], :string + attach_function :get_oneof_field_count, :upb_OneofDef_FieldCount, [OneofDef], :int + attach_function :get_oneof_field_by_index, :upb_OneofDef_Field, [OneofDef, :int], FieldDescriptor + + # RepeatableField + + # Status + attach_function :clear, :upb_Status_Clear, [Status.by_ref], :void + attach_function :error_message, :upb_Status_ErrorMessage, [Status.by_ref], :string + + # Generic + attach_function :memcmp, [:pointer, :pointer, :size_t], :int + attach_function :memcpy, [:pointer, :pointer, :size_t], :int + + # Misc + attach_function :hash, :_upb_Hash, [:pointer, :size_t, :uint64_t], :uint32_t + + # Alternatives to pre-processor macros + def self.decode_max_depth(i) + i << 16 + end + end + end +end diff --git a/ruby/lib/google/protobuf/ffi/field_descriptor.rb b/ruby/lib/google/protobuf/ffi/field_descriptor.rb new file mode 100644 index 000000000000..b163d1bfdc3c --- /dev/null +++ b/ruby/lib/google/protobuf/ffi/field_descriptor.rb @@ -0,0 +1,326 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2022 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +module Google + module Protobuf + class FieldDescriptor + attr :field_def, :descriptor_pool + + include Google::Protobuf::Internal::Convert + + # FFI Interface methods and setup + extend ::FFI::DataConverter + native_type ::FFI::Type::POINTER + + class << self + prepend Google::Protobuf::Internal::TypeSafety + + # @param value [FieldDescriptor] FieldDescriptor to convert to an FFI native type + # @param _ [Object] Unused + def to_native(value, _) + field_def_ptr = value.instance_variable_get(:@field_def) + warn "Underlying field_def was nil!" if field_def_ptr.nil? + raise "Underlying field_def was null!" if !field_def_ptr.nil? and field_def_ptr.null? + # || ::FFI::Pointer::NULL + field_def_ptr + end + + ## + # @param field_def [::FFI::Pointer] FieldDef pointer to be wrapped + # @param _ [Object] Unused + def from_native(field_def, _ = nil) + return nil if field_def.nil? or field_def.null? + # Calling upb_FieldDef_File(field_def) would create a cyclic + # dependency because either 1) we'd have to define the method to accept + # an untyped pointer or 2) FFI would complain about passing a + # FFI::Pointer instance instead of a FieldDescriptor. Instead, directly + # read the top of the FieldDef structure and extract the FileDef*. + field_def_struct = Google::Protobuf::FFI::Upb_FieldDef.new(field_def) + file_def = field_def_struct[:file_def] + raise RuntimeError.new "FileDef is nil" if file_def.nil? + raise RuntimeError.new "FileDef is null" if file_def.null? + pool_def = Google::Protobuf::FFI.file_def_pool file_def + raise RuntimeError.new "PoolDef is nil" if pool_def.nil? + raise RuntimeError.new "PoolDef is null" if pool_def.null? + pool = Google::Protobuf::ObjectCache.get(pool_def) + raise "Cannot find pool in ObjectCache!" if pool.nil? + descriptor = pool.descriptor_class_by_def[field_def.address] + if descriptor.nil? + pool.descriptor_class_by_def[field_def.address] = private_constructor(field_def, pool) + else + descriptor + end + end + end + + def self.new(*arguments, &block) + raise "Descriptor objects may not be created from Ruby." + end + + def to_s + inspect + end + + def inspect + "#{self.class.name}: #{name}" + end + + def name + @name ||= Google::Protobuf::FFI.get_full_name(self) + end + + def json_name + @json_name ||= Google::Protobuf::FFI.get_json_name(self) + end + + def number + @number ||= Google::Protobuf::FFI.get_number(self) + end + + def type + @type ||= Google::Protobuf::FFI.get_type(self) + end + + def label + @label ||= Google::Protobuf::FFI::Label[Google::Protobuf::FFI.get_label(self)] + end + + def default + return nil if Google::Protobuf::FFI.is_sub_message(self) + if Google::Protobuf::FFI.is_repeated(self) + message_value = Google::Protobuf::FFI::MessageValue.new + else + message_value = Google::Protobuf::FFI.get_default(self) + end + enum_def = Google::Protobuf::FFI.get_subtype_as_enum(self) + if enum_def.null? + convert_upb_to_ruby message_value, c_type + else + convert_upb_to_ruby message_value, c_type, enum_def + end + end + + def submsg_name + if defined? @submsg_name + @submsg_name + else + @submsg_name = case c_type + when :enum + Google::Protobuf::FFI.get_enum_fullname Google::Protobuf::FFI.get_subtype_as_enum self + when :message + Google::Protobuf::FFI.get_message_fullname Google::Protobuf::FFI.get_subtype_as_message self + else + nil + end + end + end + + ## + # Tests if this field has been set on the argument message. + # + # @param msg [Google::Protobuf::Message] + # @return [Object] Value of the field on this message. + # @raise [TypeError] If the field is not defined on this message. + def get(msg) + if msg.class.descriptor == Google::Protobuf::FFI.get_containing_message_def(self) + msg.send :get_field, self + else + raise TypeError.new "get method called on wrong message type" + end + end + + def subtype + if defined? @subtype + @subtype + else + @subtype = case c_type + when :enum + Google::Protobuf::FFI.get_subtype_as_enum(self) + when :message + Google::Protobuf::FFI.get_subtype_as_message(self) + else + nil + end + end + end + + ## + # Tests if this field has been set on the argument message. + # + # @param msg [Google::Protobuf::Message] + # @return [Boolean] True iff message has this field set + # @raise [TypeError] If this field does not exist on the message + # @raise [ArgumentError] If this field does not track presence + def has?(msg) + if msg.class.descriptor != Google::Protobuf::FFI.get_containing_message_def(self) + raise TypeError.new "has method called on wrong message type" + end + unless has_presence? + raise ArgumentError.new "does not track presence" + end + + Google::Protobuf::FFI.get_message_has msg.instance_variable_get(:@msg), self + end + + ## + # Tests if this field tracks presence. + # + # @return [Boolean] True iff this field tracks presence + def has_presence? + @has_presence ||= Google::Protobuf::FFI.get_has_presence(self) + end + + # @param msg [Google::Protobuf::Message] + def clear(msg) + if msg.class.descriptor != Google::Protobuf::FFI.get_containing_message_def(self) + raise TypeError.new "clear method called on wrong message type" + end + Google::Protobuf::FFI.clear_message_field msg.instance_variable_get(:@msg), self + nil + end + + ## + # call-seq: + # FieldDescriptor.set(message, value) + # + # Sets the value corresponding to this field to the given value on the given + # message. Raises an exception if message is of the wrong type. Performs the + # ordinary type-checks for field setting. + # + # @param msg [Google::Protobuf::Message] + # @param value [Object] + def set(msg, value) + if msg.class.descriptor != Google::Protobuf::FFI.get_containing_message_def(self) + raise TypeError.new "set method called on wrong message type" + end + unless set_value_on_message value, msg.instance_variable_get(:@msg), msg.instance_variable_get(:@arena) + raise RuntimeError.new "allocation failed" + end + nil + end + + def map? + @map ||= Google::Protobuf::FFI.is_map self + end + + def repeated? + @repeated ||= Google::Protobuf::FFI.is_repeated self + end + + def sub_message? + @sub_message ||= Google::Protobuf::FFI.is_sub_message self + end + + def wrapper? + if defined? @wrapper + @wrapper + else + message_descriptor = Google::Protobuf::FFI.get_subtype_as_message(self) + @wrapper = message_descriptor.nil? ? false : message_descriptor.send(:wrapper?) + end + end + + private + + def initialize(field_def, descriptor_pool) + @field_def = field_def + @descriptor_pool = descriptor_pool + end + + def self.private_constructor(field_def, descriptor_pool) + instance = allocate + instance.send(:initialize, field_def, descriptor_pool) + instance + end + + # TODO(jatl) Can this be added to the public API? + def real_containing_oneof + @real_containing_oneof ||= Google::Protobuf::FFI.real_containing_oneof self + end + + # Implementation details below are subject to breaking changes without + # warning and are intended for use only within the gem. + + ## + # Sets the field this FieldDescriptor represents to the given value on the given message. + # @param value [Object] Value to be set + # @param msg [::FFI::Pointer] Pointer the the upb_Message + # @param arena [Arena] Arena of the message that owns msg + def set_value_on_message(value, msg, arena, wrap: false) + message_to_alter = msg + field_def_to_set = self + if map? + raise TypeError.new "Expected map" unless value.is_a? Google::Protobuf::Map + message_descriptor = subtype + + key_field_def = Google::Protobuf::FFI.get_field_by_number(message_descriptor, 1) + key_field_type = Google::Protobuf::FFI.get_type(key_field_def) + raise TypeError.new "Map key type does not match field's key type" unless key_field_type == value.send(:key_type) + + value_field_def = Google::Protobuf::FFI.get_field_by_number(message_descriptor, 2) + value_field_type = Google::Protobuf::FFI.get_type(value_field_def) + raise TypeError.new "Map value type does not match field's value type" unless value_field_type == value.send(:value_type) + + raise TypeError.new "Map value type has wrong message/enum class" unless value_field_def.subtype == value.send(:descriptor) + + arena.fuse(value.send(:arena)) + message_value = Google::Protobuf::FFI::MessageValue.new + message_value[:map_val] = value.send(:map_ptr) + elsif repeated? + raise TypeError.new "Expected repeated field array" unless value.is_a? RepeatedField + raise TypeError.new "Repeated field array has wrong message/enum class" unless value.send(:type) == type + arena.fuse(value.send(:arena)) + message_value = Google::Protobuf::FFI::MessageValue.new + message_value[:array_val] = value.send(:array) + else + if value.nil? and (sub_message? or !real_containing_oneof.nil?) + Google::Protobuf::FFI.clear_message_field message_to_alter, field_def_to_set + return true + end + if wrap + value_field_def = Google::Protobuf::FFI.get_field_by_number subtype, 1 + type_for_conversion = Google::Protobuf::FFI.get_c_type(value_field_def) + raise RuntimeError.new "Not expecting to get a msg or enum when unwrapping" if [:enum, :message].include? type_for_conversion + message_value = convert_ruby_to_upb(value, arena, type_for_conversion, nil) + message_to_alter = Google::Protobuf::FFI.get_mutable_message(msg, self, arena)[:msg] + field_def_to_set = value_field_def + else + message_value = convert_ruby_to_upb(value, arena, c_type, subtype) + end + end + Google::Protobuf::FFI.set_message_field message_to_alter, field_def_to_set, message_value, arena + end + + def c_type + @c_type ||= Google::Protobuf::FFI.get_c_type(self) + end + end + end +end diff --git a/ruby/lib/google/protobuf/ffi/file_descriptor.rb b/ruby/lib/google/protobuf/ffi/file_descriptor.rb new file mode 100644 index 000000000000..d19838f5e470 --- /dev/null +++ b/ruby/lib/google/protobuf/ffi/file_descriptor.rb @@ -0,0 +1,65 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2022 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +module Google + module Protobuf + class FileDescriptor + attr :descriptor_pool, :file_def + + def initialize(file_def, descriptor_pool) + @descriptor_pool = descriptor_pool + @file_def = file_def + end + + def to_s + inspect + end + + def inspect + "#{self.class.name}: #{name}" + end + + def syntax + case Google::Protobuf::FFI.file_def_syntax(@file_def) + when :Proto3 + :proto3 + when :Proto2 + :proto2 + else + nil + end + end + + def name + Google::Protobuf::FFI.file_def_name(@file_def) + end + end + end +end diff --git a/ruby/lib/google/protobuf/ffi/internal/arena.rb b/ruby/lib/google/protobuf/ffi/internal/arena.rb new file mode 100644 index 000000000000..946176e2e367 --- /dev/null +++ b/ruby/lib/google/protobuf/ffi/internal/arena.rb @@ -0,0 +1,73 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2022 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +## +# Implementation details below are subject to breaking changes without +# warning and are intended for use only within the gem. +module Google + module Protobuf + module Internal + class Arena + + # FFI Interface methods and setup + extend ::FFI::DataConverter + native_type ::FFI::Type::POINTER + + class << self + prepend Google::Protobuf::Internal::TypeSafety + + # @param value [Arena] Arena to convert to an FFI native type + # @param _ [Object] Unused + def to_native(value, _) + value.instance_variable_get(:@arena) || ::FFI::Pointer::NULL + end + + ## + # @param value [::FFI::Pointer] Arena pointer to be wrapped + # @param _ [Object] Unused + def from_native(value, _) + new(value) + end + end + + def initialize(pointer) + @arena = ::FFI::AutoPointer.new(pointer, Google::Protobuf::FFI.method(:free_arena)) + end + + def fuse(other_arena) + return if other_arena == self + unless Google::Protobuf::FFI.fuse_arena(self, other_arena) + raise RuntimeError.new "Unable to fuse arenas. This should never happen since Ruby does not use initial blocks" + end + end + end + end + end +end diff --git a/ruby/lib/google/protobuf/ffi/internal/convert.rb b/ruby/lib/google/protobuf/ffi/internal/convert.rb new file mode 100644 index 000000000000..aa1dc4a8958e --- /dev/null +++ b/ruby/lib/google/protobuf/ffi/internal/convert.rb @@ -0,0 +1,342 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2022 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +## +# Implementation details below are subject to breaking changes without +# warning and are intended for use only within the gem. +module Google + module Protobuf + module Internal + module Convert + + # Arena should be the + # @param value [Object] Value to convert + # @param arena [Arena] Arena that owns the Message where the MessageValue + # will be set + # @return [Google::Protobuf::FFI::MessageValue] + def convert_ruby_to_upb(value, arena, c_type, msg_or_enum_def) + raise ArgumentError.new "Expected Descriptor or EnumDescriptor, instead got #{msg_or_enum_def.class}" unless [NilClass, Descriptor, EnumDescriptor].include? msg_or_enum_def.class + return_value = Google::Protobuf::FFI::MessageValue.new + case c_type + when :float + raise TypeError.new "Expected number type for float field '#{name}' (given #{value.class})." unless value.respond_to? :to_f + return_value[:float_val] = value.to_f + when :double + raise TypeError.new "Expected number type for double field '#{name}' (given #{value.class})." unless value.respond_to? :to_f + return_value[:double_val] = value.to_f + when :bool + raise TypeError.new "Invalid argument for boolean field '#{name}' (given #{value.class})." unless [TrueClass, FalseClass].include? value.class + return_value[:bool_val] = value + when :string + raise TypeError.new "Invalid argument for string field '#{name}' (given #{value.class})." unless [Symbol, String].include? value.class + begin + string_value = value.to_s.encode("UTF-8") + rescue Encoding::UndefinedConversionError + # TODO(jatl) - why not include the field name here? + raise Encoding::UndefinedConversionError.new "String is invalid UTF-8" + end + return_value[:str_val][:size] = string_value.bytesize + return_value[:str_val][:data] = Google::Protobuf::FFI.arena_malloc(arena, string_value.bytesize) + # TODO(jatl) - how important is it to still use arena malloc, versus the following? + # buffer = ::FFI::MemoryPointer.new(:char, string_value.bytesize) + # buffer.put_bytes(0, string_value) + # return_value[:str_val][:data] = buffer + raise NoMemoryError.new "Cannot allocate #{string_value.bytesize} bytes for string on Arena" if return_value[:str_val][:data].nil? || return_value[:str_val][:data].null? + return_value[:str_val][:data].write_string(string_value) + when :bytes + raise TypeError.new "Invalid argument for bytes field '#{name}' (given #{value.class})." unless value.is_a? String + string_value = value.encode("ASCII-8BIT") + return_value[:str_val][:size] = string_value.bytesize + return_value[:str_val][:data] = Google::Protobuf::FFI.arena_malloc(arena, string_value.bytesize) + raise NoMemoryError.new "Cannot allocate #{string_value.bytesize} bytes for bytes on Arena" if return_value[:str_val][:data].nil? || return_value[:str_val][:data].null? + return_value[:str_val][:data].write_string_length(string_value, string_value.bytesize) + when :message + raise TypeError.new "nil message not allowed here." if value.nil? + if value.is_a? Hash + raise RuntimeError.new "Attempted to initialize message from Hash for field #{name} but have no definition" if msg_or_enum_def.nil? + new_message = msg_or_enum_def.msgclass. + send(:private_constructor, arena, initial_value: value) + return_value[:msg_val] = new_message.instance_variable_get(:@msg) + return return_value + end + + descriptor = value.class.respond_to?(:descriptor) ? value.class.descriptor : nil + if descriptor != msg_or_enum_def + wkt = Google::Protobuf::FFI.get_well_known_type(msg_or_enum_def) + case wkt + when :Timestamp + raise TypeError.new "Invalid type #{value.class} to assign to submessage field '#{name}'." unless value.kind_of? Time + new_message = Google::Protobuf::FFI.new_message_from_def msg_or_enum_def, arena + sec = Google::Protobuf::FFI::MessageValue.new + sec[:int64_val] = value.tv_sec + sec_field_def = Google::Protobuf::FFI.get_field_by_number msg_or_enum_def, 1 + raise "Should be impossible" unless Google::Protobuf::FFI.set_message_field new_message, sec_field_def, sec, arena + nsec_field_def = Google::Protobuf::FFI.get_field_by_number msg_or_enum_def, 2 + nsec = Google::Protobuf::FFI::MessageValue.new + nsec[:int32_val] = value.tv_nsec + raise "Should be impossible" unless Google::Protobuf::FFI.set_message_field new_message, nsec_field_def, nsec, arena + return_value[:msg_val] = new_message + when :Duration + raise TypeError.new "Invalid type #{value.class} to assign to submessage field '#{name}'." unless value.kind_of? Numeric + new_message = Google::Protobuf::FFI.new_message_from_def msg_or_enum_def, arena + sec = Google::Protobuf::FFI::MessageValue.new + sec[:int64_val] = value + sec_field_def = Google::Protobuf::FFI.get_field_by_number msg_or_enum_def, 1 + raise "Should be impossible" unless Google::Protobuf::FFI.set_message_field new_message, sec_field_def, sec, arena + nsec_field_def = Google::Protobuf::FFI.get_field_by_number msg_or_enum_def, 2 + nsec = Google::Protobuf::FFI::MessageValue.new + nsec[:int32_val] = ((value.to_f - value.to_i) * 1000000000).round + raise "Should be impossible" unless Google::Protobuf::FFI.set_message_field new_message, nsec_field_def, nsec, arena + return_value[:msg_val] = new_message + else + raise TypeError.new "Invalid type #{value.class} to assign to submessage field '#{name}'." + end + else + arena.fuse(value.instance_variable_get(:@arena)) + return_value[:msg_val] = value.instance_variable_get :@msg + end + when :enum + return_value[:int32_val] = case value + when Numeric + value.to_i + when String, Symbol + enum_number = EnumDescriptor.send(:lookup_name, msg_or_enum_def, value.to_s) + #TODO(jatl) add the bad value to the error message after tests pass + raise RangeError.new "Unknown symbol value for enum field '#{name}'." if enum_number.nil? + enum_number + else + raise TypeError.new "Expected number or symbol type for enum field '#{name}'." + end + #TODO(jatl) After all tests pass, improve error message across integer type by including actual offending value + when :int32 + raise TypeError.new "Expected number type for integral field '#{name}' (given #{value.class})." unless value.is_a? Numeric + raise RangeError.new "Non-integral floating point value assigned to integer field '#{name}' (given #{value.class})." if value.floor != value + raise RangeError.new "Value assigned to int32 field '#{name}' (given #{value.class}) with more than 32-bits." unless value.to_i.bit_length < 32 + return_value[:int32_val] = value.to_i + when :uint32 + raise TypeError.new "Expected number type for integral field '#{name}' (given #{value.class})." unless value.is_a? Numeric + raise RangeError.new "Non-integral floating point value assigned to integer field '#{name}' (given #{value.class})." if value.floor != value + raise RangeError.new "Assigning negative value to unsigned integer field '#{name}' (given #{value.class})." if value < 0 + raise RangeError.new "Value assigned to uint32 field '#{name}' (given #{value.class}) with more than 32-bits." unless value.to_i.bit_length < 33 + return_value[:uint32_val] = value.to_i + when :int64 + raise TypeError.new "Expected number type for integral field '#{name}' (given #{value.class})." unless value.is_a? Numeric + raise RangeError.new "Non-integral floating point value assigned to integer field '#{name}' (given #{value.class})." if value.floor != value + raise RangeError.new "Value assigned to int64 field '#{name}' (given #{value.class}) with more than 64-bits." unless value.to_i.bit_length < 64 + return_value[:int64_val] = value.to_i + when :uint64 + raise TypeError.new "Expected number type for integral field '#{name}' (given #{value.class})." unless value.is_a? Numeric + raise RangeError.new "Non-integral floating point value assigned to integer field '#{name}' (given #{value.class})." if value.floor != value + raise RangeError.new "Assigning negative value to unsigned integer field '#{name}' (given #{value.class})." if value < 0 + raise RangeError.new "Value assigned to uint64 field '#{name}' (given #{value.class}) with more than 64-bits." unless value.to_i.bit_length < 65 + return_value[:uint64_val] = value.to_i + else + raise RuntimeError.new "Unsupported type #{c_type}" + end + return_value + end + + ## + # Safe to call without an arena if the caller has checked that c_type + # is not :message. + # @param message_value [Google::Protobuf::FFI::MessageValue] Value to be converted. + # @param c_type [Google::Protobuf::FFI::CType] Enum representing the type of message_value + # @param msg_or_enum_def [::FFI::Pointer] Pointer to the MsgDef or EnumDef definition + # @param arena [Google::Protobuf::Internal::Arena] Arena to create Message instances, if needed + def convert_upb_to_ruby(message_value, c_type, msg_or_enum_def = nil, arena = nil) + throw TypeError.new "Expected MessageValue but got #{message_value.class}" unless message_value.is_a? Google::Protobuf::FFI::MessageValue + + case c_type + when :bool + message_value[:bool_val] + when :int32 + message_value[:int32_val] + when :uint32 + message_value[:uint32_val] + when :double + message_value[:double_val] + when :int64 + message_value[:int64_val] + when :uint64 + message_value[:uint64_val] + when :string + if message_value[:str_val][:size].zero? + "" + else + message_value[:str_val][:data].read_string_length(message_value[:str_val][:size]).force_encoding("UTF-8").freeze + end + when :bytes + if message_value[:str_val][:size].zero? + "" + else + message_value[:str_val][:data].read_string_length(message_value[:str_val][:size]).force_encoding("ASCII-8BIT").freeze + end + when :float + message_value[:float_val] + when :enum + EnumDescriptor.send(:lookup_value, msg_or_enum_def, message_value[:int32_val]) || message_value[:int32_val] + when :message + raise "Null Arena for message" if arena.nil? + Descriptor.send(:get_message, message_value[:msg_val], msg_or_enum_def, arena) + else + raise RuntimeError.new "Unexpected type #{c_type}" + end + end + + # @param message_descriptor [Descriptor] Message Descriptor + # @return [::FFI::Pointer] PoolDef pointer + def pool_def_from_message_definition(message_descriptor) + raise RuntimeError.new "Descriptor is nil" if message_descriptor.nil? + file_def = Google::Protobuf::FFI.get_message_file_def message_descriptor + raise RuntimeError.new "FileDef is nil" if file_def.nil? + raise RuntimeError.new "FileDef is null" if file_def.null? + pool_def = Google::Protobuf::FFI.file_def_pool file_def + raise RuntimeError.new "PoolDef is nil" if pool_def.nil? + raise RuntimeError.new "PoolDef is null" if pool_def.null? + pool_def + end + + def to_h_internal(msg, message_descriptor) + return nil if msg.nil? or msg.null? + hash = {} + is_proto2 = Google::Protobuf::FFI.message_def_syntax(message_descriptor) == :Proto2 + message_descriptor.each do |field_descriptor| + # TODO: Legacy behavior, remove when we fix the is_proto2 differences. + if !is_proto2 and + field_descriptor.sub_message? and + !field_descriptor.repeated? and + !Google::Protobuf::FFI.get_message_has(msg, field_descriptor) + hash[field_descriptor.name.to_sym] = nil + next + end + + # Do not include fields that are not present (oneof or optional fields). + if is_proto2 and field_descriptor.has_presence? and !Google::Protobuf::FFI.get_message_has(msg, field_descriptor) + next + end + + message_value = Google::Protobuf::FFI.get_message_value msg, field_descriptor + + # Proto2 omits empty map/repeated fields also. + if field_descriptor.map? + hash_entry = map_create_hash(message_value[:map_val], field_descriptor) + elsif field_descriptor.repeated? + array = message_value[:array_val] + if is_proto2 and (array.null? || Google::Protobuf::FFI.array_size(array).zero?) + next + end + hash_entry = repeated_field_create_array(array, field_descriptor, field_descriptor.type) + else + hash_entry = scalar_create_hash(message_value, field_descriptor.type, field_descriptor: field_descriptor) + end + + hash[field_descriptor.name.to_sym] = hash_entry + + end + + hash + end + + def map_create_hash(map_ptr, field_descriptor) + return {} if map_ptr.nil? or map_ptr.null? + return_value = {} + + message_descriptor = field_descriptor.send(:subtype) + key_field_def = Google::Protobuf::FFI.get_field_by_number(message_descriptor, 1) + key_field_type = Google::Protobuf::FFI.get_type(key_field_def) + + value_field_def = Google::Protobuf::FFI.get_field_by_number(message_descriptor, 2) + value_field_type = Google::Protobuf::FFI.get_type(value_field_def) + + iter = ::FFI::MemoryPointer.new(:size_t, 1) + iter.write(:size_t, Google::Protobuf::FFI::Upb_Map_Begin) + while Google::Protobuf::FFI.map_next(map_ptr, iter) do + iter_size_t = iter.read(:size_t) + key_message_value = Google::Protobuf::FFI.map_key(map_ptr, iter_size_t) + value_message_value = Google::Protobuf::FFI.map_value(map_ptr, iter_size_t) + hash_key = convert_upb_to_ruby(key_message_value, key_field_type) + hash_value = scalar_create_hash(value_message_value, value_field_type, msg_or_enum_descriptor: value_field_def.subtype) + return_value[hash_key] = hash_value + end + return_value + end + + def repeated_field_create_array(array, field_descriptor, type) + return_value = [] + # puts "JATL - about to call FFI method array_size(#{array}); stacktrace:\n#{caller.join("\n")}" + n = (array.nil? || array.null?) ? 0 : Google::Protobuf::FFI.array_size(array) + 0.upto(n - 1) do |i| + message_value = Google::Protobuf::FFI.get_msgval_at(array, i) + return_value << scalar_create_hash(message_value, type, field_descriptor: field_descriptor) + end + return_value + end + + # @param field_descriptor [FieldDescriptor] Descriptor of the field to convert to a hash. + def scalar_create_hash(message_value, type, field_descriptor: nil, msg_or_enum_descriptor: nil) + if [:message, :enum].include? type + if field_descriptor.nil? + if msg_or_enum_descriptor.nil? + raise "scalar_create_hash requires either a FieldDescriptor, MessageDescriptor, or EnumDescriptor as an argument, but received only nil" + end + else + msg_or_enum_descriptor = field_descriptor.subtype + end + if type == :message + to_h_internal(message_value[:msg_val], msg_or_enum_descriptor) + elsif type == :enum + convert_upb_to_ruby message_value, type, msg_or_enum_descriptor + end + else + convert_upb_to_ruby message_value, type + end + end + + def message_value_deep_copy(message_value, type, descriptor, arena) + raise unless message_value.is_a? Google::Protobuf::FFI::MessageValue + new_message_value = Google::Protobuf::FFI::MessageValue.new + case type + when :string, :bytes + # TODO(jatl) - how important is it to still use arena malloc, versus using FFI MemoryPointers? + new_message_value[:str_val][:size] = message_value[:str_val][:size] + new_message_value[:str_val][:data] = Google::Protobuf::FFI.arena_malloc(arena, message_value[:str_val][:size]) + raise NoMemoryError.new "Allocation failed" if new_message_value[:str_val][:data].nil? or new_message_value[:str_val][:data].null? + Google::Protobuf::FFI.memcpy(new_message_value[:str_val][:data], message_value[:str_val][:data], message_value[:str_val][:size]) + when :message + new_message_value[:msg_val] = descriptor.msgclass.send(:deep_copy, message_value[:msg_val], arena).instance_variable_get(:@msg) + else + Google::Protobuf::FFI.memcpy(new_message_value.to_ptr, message_value.to_ptr, Google::Protobuf::FFI::MessageValue.size) + end + new_message_value + end + end + end + end +end diff --git a/ruby/lib/google/protobuf/ffi/internal/type_safety.rb b/ruby/lib/google/protobuf/ffi/internal/type_safety.rb new file mode 100644 index 000000000000..82903a7bc5a3 --- /dev/null +++ b/ruby/lib/google/protobuf/ffi/internal/type_safety.rb @@ -0,0 +1,48 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2022 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# A to_native DataConverter method that raises an error if the value is not of the same type. +# Adapted from to https://www.varvet.com/blog/advanced-topics-in-ruby-ffi/ +module Google + module Protobuf + module Internal + module TypeSafety + def to_native(value, ctx) + if value.kind_of?(self) or value.nil? + super + else + raise TypeError.new "Expected a kind of #{name}, was #{value.class}" + end + end + end + end + end +end + diff --git a/ruby/lib/google/protobuf/ffi/map.rb b/ruby/lib/google/protobuf/ffi/map.rb new file mode 100644 index 000000000000..2a581075f59b --- /dev/null +++ b/ruby/lib/google/protobuf/ffi/map.rb @@ -0,0 +1,404 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2022 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +module Google + module Protobuf + class Map + include Enumerable + ## + # call-seq: + # Map.new(key_type, value_type, value_typeclass = nil, init_hashmap = {}) + # => new map + # + # Allocates a new Map container. This constructor may be called with 2, 3, or 4 + # arguments. The first two arguments are always present and are symbols (taking + # on the same values as field-type symbols in message descriptors) that + # indicate the type of the map key and value fields. + # + # The supported key types are: :int32, :int64, :uint32, :uint64, :bool, + # :string, :bytes. + # + # The supported value types are: :int32, :int64, :uint32, :uint64, :bool, + # :string, :bytes, :enum, :message. + # + # The third argument, value_typeclass, must be present if value_type is :enum + # or :message. As in RepeatedField#new, this argument must be a message class + # (for :message) or enum module (for :enum). + # + # The last argument, if present, provides initial content for map. Note that + # this may be an ordinary Ruby hashmap or another Map instance with identical + # key and value types. Also note that this argument may be present whether or + # not value_typeclass is present (and it is unambiguously separate from + # value_typeclass because value_typeclass's presence is strictly determined by + # value_type). The contents of this initial hashmap or Map instance are + # shallow-copied into the new Map: the original map is unmodified, but + # references to underlying objects will be shared if the value type is a + # message type. + def self.new(key_type, value_type, value_typeclass = nil, init_hashmap = {}) + instance = allocate + # TODO(jatl) This argument mangling doesn't agree with the type signature, + # but does align with the text of the comments and is required to make unit tests pass. + if init_hashmap.empty? and ![:enum, :message].include?(value_type) + init_hashmap = value_typeclass + value_typeclass = nil + end + instance.send(:initialize, key_type, value_type, value_type_class: value_typeclass, initial_values: init_hashmap) + instance + end + + ## + # call-seq: + # Map.keys => [list_of_keys] + # + # Returns the list of keys contained in the map, in unspecified order. + def keys + return_value = [] + internal_iterator do |iterator| + key_message_value = Google::Protobuf::FFI.map_key(@map_ptr, iterator) + return_value << convert_upb_to_ruby(key_message_value, key_type) + end + return_value + end + + ## + # call-seq: + # Map.values => [list_of_values] + # + # Returns the list of values contained in the map, in unspecified order. + def values + return_value = [] + internal_iterator do |iterator| + value_message_value = Google::Protobuf::FFI.map_value(@map_ptr, iterator) + return_value << convert_upb_to_ruby(value_message_value, value_type, descriptor, arena) + end + return_value + end + + ## + # call-seq: + # Map.[](key) => value + # + # Accesses the element at the given key. Throws an exception if the key type is + # incorrect. Returns nil when the key is not present in the map. + def [](key) + value = Google::Protobuf::FFI::MessageValue.new + key_message_value = convert_ruby_to_upb(key, arena, key_type, nil) + if Google::Protobuf::FFI.map_get(@map_ptr, key_message_value, value) + convert_upb_to_ruby(value, value_type, descriptor, arena) + end + end + + ## + # call-seq: + # Map.[]=(key, value) => value + # + # Inserts or overwrites the value at the given key with the given new value. + # Throws an exception if the key type is incorrect. Returns the new value that + # was just inserted. + def []=(key, value) + raise FrozenError.new "can't modify frozen #{self.class}" if frozen? + key_message_value = convert_ruby_to_upb(key, arena, key_type, nil) + value_message_value = convert_ruby_to_upb(value, arena, value_type, descriptor) + Google::Protobuf::FFI.map_set(@map_ptr, key_message_value, value_message_value, arena) + value + end + + def has_key?(key) + key_message_value = convert_ruby_to_upb(key, arena, key_type, nil) + Google::Protobuf::FFI.map_get(@map_ptr, key_message_value, nil) + end + + ## + # call-seq: + # Map.delete(key) => old_value + # + # Deletes the value at the given key, if any, returning either the old value or + # nil if none was present. Throws an exception if the key is of the wrong type. + def delete(key) + raise FrozenError.new "can't modify frozen #{self.class}" if frozen? + value = Google::Protobuf::FFI::MessageValue.new + key_message_value = convert_ruby_to_upb(key, arena, key_type, nil) + return_value = if Google::Protobuf::FFI.map_get(@map_ptr, key_message_value, value) + convert_upb_to_ruby(value, value_type, descriptor, arena) + end + Google::Protobuf::FFI.map_delete(@map_ptr, key_message_value) + return_value + end + + def clear + raise FrozenError.new "can't modify frozen #{self.class}" if frozen? + Google::Protobuf::FFI.map_clear(@map_ptr) + nil + end + + def length + Google::Protobuf::FFI.map_size(@map_ptr) + end + alias size length + + ## + # call-seq: + # Map.dup => new_map + # + # Duplicates this map with a shallow copy. References to all non-primitive + # element objects (e.g., submessages) are shared. + def dup + internal_dup + end + alias clone dup + + ## + # call-seq: + # Map.==(other) => boolean + # + # Compares this map to another. Maps are equal if they have identical key sets, + # and for each key, the values in both maps compare equal. Elements are + # compared as per normal Ruby semantics, by calling their :== methods (or + # performing a more efficient comparison for primitive types). + # + # Maps with dissimilar key types or value types/typeclasses are never equal, + # even if value comparison (for example, between integers and floats) would + # have otherwise indicated that every element has equal value. + def ==(other) + if other.is_a? Hash + other = self.class.send(:private_constructor, key_type, value_type, descriptor, initial_values: other) + elsif !other.is_a? Google::Protobuf::Map + return false + end + + return true if object_id == other.object_id + return false if key_type != other.send(:key_type) or value_type != other.send(:value_type) or descriptor != other.send(:descriptor) or length != other.length + other_map_ptr = other.send(:map_ptr) + each_msg_val do |key_message_value, value_message_value| + other_value = Google::Protobuf::FFI::MessageValue.new + return false unless Google::Protobuf::FFI.map_get(other_map_ptr, key_message_value, other_value) + return false unless Google::Protobuf::FFI.message_value_equal(value_message_value, other_value, value_type, descriptor) + end + true + end + + def hash + return_value = 0 + each_msg_val do |key_message_value, value_message_value| + return_value = Google::Protobuf::FFI.message_value_hash(key_message_value, key_type, nil, return_value) + return_value = Google::Protobuf::FFI.message_value_hash(value_message_value, value_type, descriptor, return_value) + end + return_value + end + + ## + # call-seq: + # Map.to_h => {} + # + # Returns a Ruby Hash object containing all the values within the map + def to_h + return {} if map_ptr.nil? or map_ptr.null? + return_value = {} + each_msg_val do |key_message_value, value_message_value| + hash_key = convert_upb_to_ruby(key_message_value, key_type) + hash_value = scalar_create_hash(value_message_value, value_type, msg_or_enum_descriptor: descriptor) + return_value[hash_key] = hash_value + end + return_value + end + + def inspect + key_value_pairs = [] + each_msg_val do |key_message_value, value_message_value| + key_string = convert_upb_to_ruby(key_message_value, key_type).inspect + if value_type == :message + sub_msg_descriptor = Google::Protobuf::FFI.get_subtype_as_message(descriptor) + value_string = sub_msg_descriptor.msgclass.send(:inspect_internal, value_message_value[:msg_val]) + else + value_string = convert_upb_to_ruby(value_message_value, value_type, descriptor).inspect + end + key_value_pairs << "#{key_string}=>#{value_string}" + end + "{#{key_value_pairs.join(", ")}}" + end + + ## + # call-seq: + # Map.merge(other_map) => map + # + # Copies key/value pairs from other_map into a copy of this map. If a key is + # set in other_map and this map, the value from other_map overwrites the value + # in the new copy of this map. Returns the new copy of this map with merged + # contents. + def merge(other) + internal_merge(other) + end + + ## + # call-seq: + # Map.each(&block) + # + # Invokes &block on each |key, value| pair in the map, in unspecified order. + # Note that Map also includes Enumerable; map thus acts like a normal Ruby + # sequence. + def each &block + each_msg_val do |key_message_value, value_message_value| + key_value = convert_upb_to_ruby(key_message_value, key_type) + value_value = convert_upb_to_ruby(value_message_value, value_type, descriptor, arena) + yield key_value, value_value + end + nil + end + + private + attr :arena, :map_ptr, :key_type, :value_type, :descriptor, :name + + include Google::Protobuf::Internal::Convert + + def internal_iterator + iter = ::FFI::MemoryPointer.new(:size_t, 1) + iter.write(:size_t, Google::Protobuf::FFI::Upb_Map_Begin) + while Google::Protobuf::FFI.map_next(@map_ptr, iter) do + iter_size_t = iter.read(:size_t) + yield iter_size_t + end + end + + def each_msg_val &block + internal_iterator do |iterator| + key_message_value = Google::Protobuf::FFI.map_key(@map_ptr, iterator) + value_message_value = Google::Protobuf::FFI.map_value(@map_ptr, iterator) + yield key_message_value, value_message_value + end + end + + def internal_dup + instance = self.class.send(:private_constructor, key_type, value_type, descriptor, arena: arena) + new_map_ptr = instance.send(:map_ptr) + each_msg_val do |key_message_value, value_message_value| + Google::Protobuf::FFI.map_set(new_map_ptr, key_message_value, value_message_value, arena) + end + instance + end + + def internal_merge_into_self(other) + case other + when Hash + other.each do |key, value| + key_message_value = convert_ruby_to_upb(key, arena, key_type, nil) + value_message_value = convert_ruby_to_upb(value, arena, value_type, descriptor) + Google::Protobuf::FFI.map_set(@map_ptr, key_message_value, value_message_value, arena) + end + when Google::Protobuf::Map + unless key_type == other.send(:key_type) and value_type == other.send(:value_type) and descriptor == other.descriptor + raise ArgumentError.new "Attempt to merge Map with mismatching types" #TODO(jatl) Improve error message by adding type information + end + arena.fuse(other.send(:arena)) + iter = ::FFI::MemoryPointer.new(:size_t, 1) + iter.write(:size_t, Google::Protobuf::FFI::Upb_Map_Begin) + other.send(:each_msg_val) do |key_message_value, value_message_value| + Google::Protobuf::FFI.map_set(@map_ptr, key_message_value, value_message_value, arena) + end + else + raise ArgumentError.new "Unknown type merging into Map" #TODO(jatl) improve this error message by including type information + end + self + end + + def internal_merge(other) + internal_dup.internal_merge_into_self(other) + end + + def initialize(key_type, value_type, value_type_class: nil, initial_values: nil, arena: nil, map: nil, descriptor: nil, name: nil) + @name = name || 'Map' + + unless [:int32, :int64, :uint32, :uint64, :bool, :string, :bytes].include? key_type + raise ArgumentError.new "Invalid key type for map." #TODO(jatl) improve error message to include what type was passed + end + @key_type = key_type + + unless [:int32, :int64, :uint32, :uint64, :bool, :string, :bytes, :enum, :message].include? value_type + raise ArgumentError.new "Invalid value type for map." #TODO(jatl) improve error message to include what type was passed + end + @value_type = value_type + + if !descriptor.nil? + raise ArgumentError "Expected descriptor to be a Descriptor or EnumDescriptor" unless [EnumDescriptor, Descriptor].include? descriptor.class + @descriptor = descriptor + elsif [:message, :enum].include? value_type + raise ArgumentError.new "Expected at least 3 arguments for message/enum." if value_type_class.nil? + descriptor = value_type_class.respond_to?(:descriptor) ? value_type_class.descriptor : nil + raise ArgumentError.new "Type class #{value_type_class} has no descriptor. Please pass a class or enum as returned by the DescriptorPool." if descriptor.nil? + @descriptor = descriptor + else + @descriptor = nil + end + + @arena = arena || Google::Protobuf::FFI.create_arena + @map_ptr = map || Google::Protobuf::FFI.create_map(@arena, @key_type, @value_type) + + internal_merge_into_self(initial_values) unless initial_values.nil? + + # Should always be the last expression of the initializer to avoid + # leaking references to this object before construction is complete. + ObjectCache.add(@map_ptr, self) + end + + # @param field [FieldDescriptor] Descriptor of the field where the RepeatedField will be assigned + # @param values [Hash|Map] Initial value; may be nil or empty + # @param arena [Arena] Owning message's arena + def self.construct_for_field(field, arena, value: nil, map: nil) + raise ArgumentError.new "Expected Hash object as initializer value for map field '#{field.name}' (given #{value.class})." unless value.nil? or value.is_a? Hash + instance = allocate + raise ArgumentError.new "Expected field with type :message, instead got #{field.class}" unless field.type == :message + message_descriptor = field.send(:subtype) + key_field_def = Google::Protobuf::FFI.get_field_by_number(message_descriptor, 1) + key_field_type = Google::Protobuf::FFI.get_type(key_field_def) + + value_field_def = Google::Protobuf::FFI.get_field_by_number(message_descriptor, 2) + value_field_type = Google::Protobuf::FFI.get_type(value_field_def) + instance.send(:initialize, key_field_type, value_field_type, initial_values: value, name: field.name, arena: arena, map: map, descriptor: value_field_def.subtype) + instance + end + + def self.private_constructor(key_type, value_type, descriptor, initial_values: nil, arena: nil) + instance = allocate + instance.send(:initialize, key_type, value_type, descriptor: descriptor, initial_values: initial_values, arena: arena) + instance + end + + extend Google::Protobuf::Internal::Convert + + def self.deep_copy(map) + instance = allocate + instance.send(:initialize, map.send(:key_type), map.send(:value_type), descriptor: map.send(:descriptor)) + map.send(:each_msg_val) do |key_message_value, value_message_value| + Google::Protobuf::FFI.map_set(instance.send(:map_ptr), key_message_value, message_value_deep_copy(value_message_value, map.send(:value_type), map.send(:descriptor), instance.send(:arena)), instance.send(:arena)) + end + instance + end + end + end +end diff --git a/ruby/lib/google/protobuf/ffi/object_cache.rb b/ruby/lib/google/protobuf/ffi/object_cache.rb new file mode 100644 index 000000000000..8ad0de109bee --- /dev/null +++ b/ruby/lib/google/protobuf/ffi/object_cache.rb @@ -0,0 +1,94 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2022 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +require 'weakref' + +module Google + module Protobuf + module ObjectCache + @@lock = Mutex.new + @@cache = {} + def self.drop(key) + @@lock.synchronize do + @@cache.delete(key.address) + end + end + + def self.get(key) + @@lock.synchronize do + value = @@cache[key.address] + begin + if value.nil? + return nil + else + if value.weakref_alive? + return value.__getobj__ + else + @@cache.delete(key.address) + return nil + end + end + rescue WeakRef::RefError + @@cache.delete(key.address) + return nil + end + end + nil + end + + def self.add(key, value) + raise ArgumentError.new "WeakRef values are not allowed" if value.is_a? WeakRef + @@lock.synchronize do + if @@cache.include? key.address + existing_value = @@cache[key.address] + begin + if existing_value.nil? + raise ArgumentError.new "Key already exists in ObjectCache but has nil value" + else + if existing_value.weakref_alive? + original = existing_value.__getobj__ + raise ArgumentError.new "Key already exists in ObjectCache for different value" unless original.object_id == value.object_id + else + @@cache.delete(key.address) + nil + end + end + rescue WeakRef::RefError + @@cache.delete(key.address) + nil + end + end + @@cache[key.address] = WeakRef.new value + nil + end + end + end + end +end \ No newline at end of file diff --git a/ruby/lib/google/protobuf/ffi/oneof_descriptor.rb b/ruby/lib/google/protobuf/ffi/oneof_descriptor.rb new file mode 100644 index 000000000000..366c5e8030a1 --- /dev/null +++ b/ruby/lib/google/protobuf/ffi/oneof_descriptor.rb @@ -0,0 +1,114 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2022 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +module Google + module Protobuf + class OneofDescriptor + attr :descriptor_pool, :oneof_def + include Enumerable + + # FFI Interface methods and setup + extend ::FFI::DataConverter + native_type ::FFI::Type::POINTER + + class << self + prepend Google::Protobuf::Internal::TypeSafety + + # @param value [OneofDescriptor] FieldDescriptor to convert to an FFI native type + # @param _ [Object] Unused + def to_native(value, _ = nil) + oneof_def_ptr = value.instance_variable_get(:@oneof_def) + warn "Underlying oneof_def was nil!" if oneof_def_ptr.nil? + raise "Underlying oneof_def was null!" if !oneof_def_ptr.nil? and oneof_def_ptr.null? + # || ::FFI::Pointer::NULL + oneof_def_ptr + end + + ## + # @param oneof_def [::FFI::Pointer] OneofDef pointer to be wrapped + # @param _ [Object] Unused + def from_native(oneof_def, _ = nil) + return nil if oneof_def.nil? or oneof_def.null? + # Calling upb_OneofDef_ContainingType(oneof_def) would create a cyclic + # dependency because either 1) we'd have to define the method to accept + # an untyped pointer or 2) FFI would complain about passing a + # FFI::Pointer instance instead of a OneofDescriptor. Instead, directly + # read the top of the OneDef structure and extract the MsgDef*. + oneof_def_struct = Google::Protobuf::FFI::Upb_OneofDef.new(oneof_def) + message_descriptor = Descriptor.from_native(oneof_def_struct[:parent]) + raise RuntimeError.new "Message Descriptor is nil" if message_descriptor.nil? + file_def = Google::Protobuf::FFI.get_message_file_def message_descriptor + raise RuntimeError.new "FileDef is nil" if file_def.nil? + raise RuntimeError.new "FileDef is null" if file_def.null? + pool_def = Google::Protobuf::FFI.file_def_pool file_def + raise RuntimeError.new "PoolDef is nil" if pool_def.nil? + raise RuntimeError.new "PoolDef is null" if pool_def.null? + pool = Google::Protobuf::ObjectCache.get(pool_def) + raise "Cannot find pool in ObjectCache!" if pool.nil? + descriptor = pool.descriptor_class_by_def[oneof_def.address] + if descriptor.nil? + pool.descriptor_class_by_def[oneof_def.address] = private_constructor(oneof_def, pool) + else + descriptor + end + end + end + + def self.new(*arguments, &block) + raise "OneofDescriptor objects may not be created from Ruby." + end + + def name + Google::Protobuf::FFI.get_oneof_name(self) + end + + def each &block + n = Google::Protobuf::FFI.get_oneof_field_count(self) + 0.upto(n-1) do |i| + yield(Google::Protobuf::FFI.get_oneof_field_by_index(self, i)) + end + nil + end + + private + + def initialize(oneof_def, descriptor_pool) + @descriptor_pool = descriptor_pool + @oneof_def = oneof_def + end + + def self.private_constructor(oneof_def, descriptor_pool) + instance = allocate + instance.send(:initialize, oneof_def, descriptor_pool) + instance + end + end + end +end diff --git a/ruby/lib/google/protobuf/ffi/repeated_field.rb b/ruby/lib/google/protobuf/ffi/repeated_field.rb new file mode 100644 index 000000000000..542e57434400 --- /dev/null +++ b/ruby/lib/google/protobuf/ffi/repeated_field.rb @@ -0,0 +1,516 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +require 'forwardable' + +# +# This class makes RepeatedField act (almost-) like a Ruby Array. +# It has convenience methods that extend the core C or Java based +# methods. +# +# This is a best-effort to mirror Array behavior. Two comments: +# 1) patches always welcome :) +# 2) if performance is an issue, feel free to rewrite the method +# in jruby and C. The source code has plenty of examples +# +# KNOWN ISSUES +# - #[]= doesn't allow less used approaches such as `arr[1, 2] = 'fizz'` +# - #concat should return the orig array +# - #push should accept multiple arguments and push them all at the same time +# +module Google + module Protobuf + class RepeatedField + extend Forwardable + # NOTE: using delegators rather than method_missing to make the + # relationship explicit instead of implicit + def_delegators :to_ary, + :&, :*, :-, :'<=>', + :assoc, :bsearch, :bsearch_index, :combination, :compact, :count, + :cycle, :dig, :drop, :drop_while, :eql?, :fetch, :find_index, :flatten, + :include?, :index, :inspect, :join, + :pack, :permutation, :product, :pretty_print, :pretty_print_cycle, + :rassoc, :repeated_combination, :repeated_permutation, :reverse, + :rindex, :rotate, :sample, :shuffle, :shelljoin, + :to_s, :transpose, :uniq, :| + + include Enumerable + + ## + # call-seq: + # RepeatedField.new(type, type_class = nil, initial_values = []) + # + # Creates a new repeated field. The provided type must be a Ruby symbol, and + # an take on the same values as those accepted by FieldDescriptor#type=. If + # the type is :message or :enum, type_class must be non-nil, and must be the + # Ruby class or module returned by Descriptor#msgclass or + # EnumDescriptor#enummodule, respectively. An initial list of elements may also + # be provided. + def self.new(type, type_class = nil, initial_values = []) + instance = allocate + # TODO(jatl) This argument mangling doesn't agree with the type signature in the comments + # but is required to make unit tests pass; + if type_class.is_a?(Enumerable) and initial_values.empty? and ![:enum, :message].include?(type) + initial_values = type_class + type_class = nil + end + instance.send(:initialize, type, type_class: type_class, initial_values: initial_values) + instance + end + + ## + # call-seq: + # RepeatedField.each(&block) + # + # Invokes the block once for each element of the repeated field. RepeatedField + # also includes Enumerable; combined with this method, the repeated field thus + # acts like an ordinary Ruby sequence. + def each &block + each_msg_val do |element| + yield(convert_upb_to_ruby(element, type, descriptor, arena)) + end + self + end + + def [](*args) + count = length + if args.size < 1 + raise ArgumentError.new "Index or range is a required argument." + end + if args[0].is_a? Range + if args.size > 1 + raise ArgumentError.new "Expected 1 when passing Range argument, but got #{args.size}" + end + range = args[0] + # Handle begin-less and/or endless ranges, when supported. + index_of_first = range.respond_to?(:begin) ? range.begin : range.last + index_of_first = 0 if index_of_first.nil? + end_of_range = range.respond_to?(:end) ? range.end : range.last + index_of_last = end_of_range.nil? ? -1 : end_of_range + + if index_of_last < 0 + index_of_last += count + end + unless range.exclude_end? and !end_of_range.nil? + index_of_last += 1 + end + index_of_first += count if index_of_first < 0 + length = index_of_last - index_of_first + return [] if length.zero? + elsif args[0].is_a? Integer + index_of_first = args[0] + index_of_first += count if index_of_first < 0 + if args.size > 2 + raise ArgumentError.new "Expected 1 or 2 arguments, but got #{args.size}" + end + if args.size == 1 # No length specified, return one element + if array.null? or index_of_first < 0 or index_of_first >= count + return nil + else + return convert_upb_to_ruby(Google::Protobuf::FFI.get_msgval_at(array, index_of_first), type, descriptor, arena) + end + else + length = [args[1],count].min + end + else + raise NotImplementedError + end + + if array.null? or index_of_first < 0 or index_of_first >= count + nil + else + if index_of_first + length > count + length = count - index_of_first + end + if length < 0 + nil + else + subarray(index_of_first, length) + end + end + end + alias at [] + + + def []=(index, value) + raise FrozenError if frozen? + count = length + index += count if index < 0 + return nil if index < 0 + if index >= count + resize(index+1) + empty_message_value = Google::Protobuf::FFI::MessageValue.new # Implicitly clear + count.upto(index-1) do |i| + Google::Protobuf::FFI.array_set(array, i, empty_message_value) + end + end + Google::Protobuf::FFI.array_set(array, index, convert_ruby_to_upb(value, arena, type, descriptor)) + nil + end + + def push(*elements) + raise FrozenError if frozen? + internal_push(*elements) + end + + def <<(element) + raise FrozenError if frozen? + push element + end + + def replace(replacements) + raise FrozenError if frozen? + clear + push(*replacements) + end + + def clear + raise FrozenError if frozen? + resize 0 + self + end + + def length + array.null? ? 0 : Google::Protobuf::FFI.array_size(array) + end + alias size :length + + def dup + instance = self.class.allocate + instance.send(:initialize, type, descriptor: descriptor, arena: arena) + each_msg_val do |element| + instance.send(:append_msg_val, element) + end + instance + end + alias clone dup + + def ==(other) + return true if other.object_id == object_id + if other.is_a? RepeatedField + return false unless other.length == length + each_msg_val_with_index do |msg_val, i| + other_msg_val = Google::Protobuf::FFI.get_msgval_at(other.send(:array), i) + unless Google::Protobuf::FFI.message_value_equal(msg_val, other_msg_val, type, descriptor) + return false + end + end + return true + elsif other.is_a? Enumerable + return to_ary == other.to_a + end + false + end + + ## + # call-seq: + # RepeatedField.to_ary => array + # + # Used when converted implicitly into array, e.g. compared to an Array. + # Also called as a fallback of Object#to_a + def to_ary + return_value = [] + each do |element| + return_value << element + end + return_value + end + + def hash + return_value = 0 + each_msg_val do |msg_val| + return_value = Google::Protobuf::FFI.message_value_hash(msg_val, type, descriptor, return_value) + end + return_value + end + + def +(other) + if other.is_a? RepeatedField + if type != other.instance_variable_get(:@type) or descriptor != other.instance_variable_get(:@descriptor) + raise ArgumentError.new "Attempt to append RepeatedField with different element type." + end + fuse_arena(other.send(:arena)) + super_set = dup + other.send(:each_msg_val) do |msg_val| + super_set.send(:append_msg_val, msg_val) + end + super_set + elsif other.is_a? Enumerable + super_set = dup + super_set.push(*other.to_a) + else + raise ArgumentError.new "Unknown type appending to RepeatedField" + end + end + + def concat(other) + raise ArgumentError.new "Expected Enumerable, but got #{other.class}" unless other.is_a? Enumerable + push(*other.to_a) + end + + def first(n=nil) + if n.nil? + return self[0] + elsif n < 0 + raise ArgumentError, "negative array size" + else + return self[0...n] + end + end + + + def last(n=nil) + if n.nil? + return self[-1] + elsif n < 0 + raise ArgumentError, "negative array size" + else + start = [self.size-n, 0].max + return self[start...self.size] + end + end + + + def pop(n=nil) + if n + results = [] + n.times{ results << pop_one } + return results + else + return pop_one + end + end + + + def empty? + self.size == 0 + end + + # array aliases into enumerable + alias_method :each_index, :each_with_index + alias_method :slice, :[] + alias_method :values_at, :select + alias_method :map, :collect + + + class << self + def define_array_wrapper_method(method_name) + define_method(method_name) do |*args, &block| + arr = self.to_a + result = arr.send(method_name, *args) + self.replace(arr) + return result if result + return block ? block.call : result + end + end + private :define_array_wrapper_method + + + def define_array_wrapper_with_result_method(method_name) + define_method(method_name) do |*args, &block| + # result can be an Enumerator, Array, or nil + # Enumerator can sometimes be returned if a block is an optional argument and it is not passed in + # nil usually specifies that no change was made + result = self.to_a.send(method_name, *args, &block) + if result + new_arr = result.to_a + self.replace(new_arr) + if result.is_a?(Enumerator) + # generate a fresh enum; rewinding the exiting one, in Ruby 2.2, will + # reset the enum with the same length, but all the #next calls will + # return nil + result = new_arr.to_enum + # generate a wrapper enum so any changes which occur by a chained + # enum can be captured + ie = ProxyingEnumerator.new(self, result) + result = ie.to_enum + end + end + result + end + end + private :define_array_wrapper_with_result_method + end + + + %w(delete delete_at shift slice! unshift).each do |method_name| + define_array_wrapper_method(method_name) + end + + + %w(collect! compact! delete_if fill flatten! insert reverse! + rotate! select! shuffle! sort! sort_by! uniq!).each do |method_name| + define_array_wrapper_with_result_method(method_name) + end + alias_method :keep_if, :select! + alias_method :map!, :collect! + alias_method :reject!, :delete_if + + + # propagates changes made by user of enumerator back to the original repeated field. + # This only applies in cases where the calling function which created the enumerator, + # such as #sort!, modifies itself rather than a new array, such as #sort + class ProxyingEnumerator < Struct.new(:repeated_field, :external_enumerator) + def each(*args, &block) + results = [] + external_enumerator.each_with_index do |val, i| + result = yield(val) + results << result + #nil means no change occurred from yield; usually occurs when #to_a is called + if result + repeated_field[i] = result if result != val + end + end + results + end + end + + private + include Google::Protobuf::Internal::Convert + + attr :name, :arena, :array, :type, :descriptor + + def internal_push(*elements) + elements.each do |element| + append_msg_val convert_ruby_to_upb(element, arena, type, descriptor) + end + self + end + + def pop_one + raise FrozenError if frozen? + count = length + return nil if length.zero? + last_element = Google::Protobuf::FFI.get_msgval_at(array, count-1) + return_value = convert_upb_to_ruby(last_element, type, descriptor, arena) + resize(count-1) + return_value + end + + def subarray(start, length) + return_result = [] + (start..(start + length - 1)).each do |i| + element = Google::Protobuf::FFI.get_msgval_at(array, i) + return_result << convert_upb_to_ruby(element, type, descriptor, arena) + end + return_result + end + + def each_msg_val_with_index &block + n = array.null? ? 0 : Google::Protobuf::FFI.array_size(array) + 0.upto(n-1) do |i| + yield Google::Protobuf::FFI.get_msgval_at(array, i), i + end + end + + def each_msg_val &block + each_msg_val_with_index do |msg_val, _| + yield msg_val + end + end + + # @param msg_val [Google::Protobuf::FFI::MessageValue] Value to append + def append_msg_val(msg_val) + unless Google::Protobuf::FFI.append_array(array, msg_val, arena) + raise NoMemoryError.new "Could not allocate room for #{msg_val} in Arena" + end + end + + # @param new_size [Integer] New size of the array + def resize(new_size) + unless Google::Protobuf::FFI.array_resize(array, new_size, arena) + raise NoMemoryError.new "Array resize to #{new_size} failed!" + end + end + + def initialize(type, type_class: nil, initial_values: nil, name: nil, arena: nil, array: nil, descriptor: nil) + @name = name || 'RepeatedField' + raise ArgumentError.new "Expected argument type to be a Symbol" unless type.is_a? Symbol + field_number = Google::Protobuf::FFI::FieldType[type] + raise ArgumentError.new "Unsupported type '#{type}'" if field_number.nil? + if !descriptor.nil? + @descriptor = descriptor + elsif [:message, :enum].include? type + raise ArgumentError.new "Expected at least 2 arguments for message/enum." if type_class.nil? + descriptor = type_class.respond_to?(:descriptor) ? type_class.descriptor : nil + raise ArgumentError.new "Type class #{type_class} has no descriptor. Please pass a class or enum as returned by the DescriptorPool." if descriptor.nil? + @descriptor = descriptor + else + @descriptor = nil + end + @type = type + + @arena = arena || Google::Protobuf::FFI.create_arena + @array = array || Google::Protobuf::FFI.create_array(@arena, @type) + unless initial_values.nil? + unless initial_values.is_a? Enumerable + raise ArgumentError.new "Expected array as initializer value for repeated field '#{name}' (given #{initial_values.class})." + end + internal_push(*initial_values) + end + + # Should always be the last expression of the initializer to avoid + # leaking references to this object before construction is complete. + ObjectCache.add(@array, self) + end + + # @param field [FieldDescriptor] Descriptor of the field where the RepeatedField will be assigned + # @param values [Enumerable] Initial values; may be nil or empty + # @param arena [Arena] Owning message's arena + def self.construct_for_field(field, arena, values: nil, array: nil) + instance = allocate + options = {initial_values: values, name: field.name, arena: arena, array: array} + if [:enum, :message].include? field.type + options[:descriptor] = field.subtype + end + instance.send(:initialize, field.type, **options) + instance + end + + def fuse_arena(arena) + arena.fuse(arena) + end + + extend Google::Protobuf::Internal::Convert + + def self.deep_copy(repeated_field) + instance = allocate + instance.send(:initialize, repeated_field.send(:type), descriptor: repeated_field.send(:descriptor)) + instance.send(:resize, repeated_field.length) + new_array = instance.send(:array) + repeated_field.send(:each_msg_val_with_index) do |element, i| + Google::Protobuf::FFI.array_set(new_array, i, message_value_deep_copy(element, repeated_field.send(:type), repeated_field.send(:descriptor), instance.send(:arena))) + end + instance + end + + end + end +end From 589629d5c92b97f325515e288e0e22628c6964f3 Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Tue, 10 Jan 2023 13:03:48 -0800 Subject: [PATCH 10/54] Add ffi flags --- ruby/BUILD.bazel | 25 +++++++++++++++++++++++++ ruby/google-protobuf.gemspec | 2 ++ ruby/lib/google/BUILD.bazel | 8 +++++++- 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/ruby/BUILD.bazel b/ruby/BUILD.bazel index c581355c5404..fe215943ece7 100755 --- a/ruby/BUILD.bazel +++ b/ruby/BUILD.bazel @@ -2,6 +2,8 @@ # # See also code generation logic under /src/google/protobuf/compiler/ruby. +load("@bazel_skylib//lib:selects.bzl", "selects") +load("@bazel_skylib//rules:common_settings.bzl", "string_flag") load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix") load("@rules_ruby//ruby:defs.bzl", "ruby_library") load("//build_defs:internal_shell.bzl", "inline_sh_binary") @@ -13,6 +15,29 @@ load("//:protobuf_version.bzl", "PROTOBUF_RUBY_VERSION") # Ruby Runtime ################################################################################ +string_flag( + name = "ffi", + build_setting_default = "disabled", + values = [ + "enabled", + "disabled", + ], +) + +config_setting( + name = "ffi_enabled", + flag_values = { + "ffi": "enabled" + }, +) + +config_setting( + name = "ffi_disabled", + flag_values = { + "ffi": "disabled" + }, +) + ruby_library( name = "protobuf", deps = ["//ruby/lib/google:protobuf_lib"], diff --git a/ruby/google-protobuf.gemspec b/ruby/google-protobuf.gemspec index b7509b57e0ba..f57b98a84198 100644 --- a/ruby/google-protobuf.gemspec +++ b/ruby/google-protobuf.gemspec @@ -30,6 +30,8 @@ Gem::Specification.new do |s| raise RuntimeError.new("Unsupported value of PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION environment variable `#{ENV['PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION']}`. If set, must be one of `native` or `ffi`. Defaults to `native`.") end s.required_ruby_version = '>= 2.3' + s.add_development_dependency "ffi", "~>1" + s.add_development_dependency "ffi-compiler", "~>1" s.add_development_dependency "rake-compiler", "~> 1.1.0" s.add_development_dependency "test-unit", '~> 3.0', '>= 3.0.9' end diff --git a/ruby/lib/google/BUILD.bazel b/ruby/lib/google/BUILD.bazel index 4cfa5cbcfe41..16a34eefa649 100644 --- a/ruby/lib/google/BUILD.bazel +++ b/ruby/lib/google/BUILD.bazel @@ -46,7 +46,13 @@ ruby_library( srcs = glob([ "**/*.rb", ]), - deps = ["//:well_known_ruby_protos"], + deps = ["//:well_known_ruby_protos"] + select({ + "//ruby:ffi_enabled": [ + "@protobuf_bundle//:ffi", + "@protobuf_bundle//:ffi-compiler", + ], + "//conditions:default": [], + }), includes = ["ruby/lib"], data = select({ "@rules_ruby//ruby/runtime:config_jruby": ["protobuf_java.jar"], From 2ec3aa355a1c94b1ea7a76e8a183856ab68f0a83 Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Wed, 11 Jan 2023 09:54:35 -0500 Subject: [PATCH 11/54] Refactor convert.c to separate CRuby-specific glue that has dependencies on Ruby headers from functionality needed by FFI. --- ruby/Rakefile | 83 +++++++++++++++++++----- ruby/ext/google/protobuf_c/convert.c | 55 +++++----------- ruby/ext/google/shared/shared_convert.c | 85 +++++++++++++++++++++++++ ruby/ext/google/shared/shared_convert.h | 41 ++++++++++++ 4 files changed, 208 insertions(+), 56 deletions(-) create mode 100644 ruby/ext/google/shared/shared_convert.c create mode 100644 ruby/ext/google/shared/shared_convert.h diff --git a/ruby/Rakefile b/ruby/Rakefile index c05e1159ea93..0e02f7c4dff9 100644 --- a/ruby/Rakefile +++ b/ruby/Rakefile @@ -3,6 +3,10 @@ require "rubygems/package_task" require "rake/extensiontask" unless RUBY_PLATFORM == "java" require "rake/testtask" +USE_FFI = ENV['PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION'] == 'ffi' + +require "ffi-compiler/compile_task" if USE_FFI + spec = Gem::Specification.load("google-protobuf.gemspec") well_known_protos = %w[ @@ -68,33 +72,75 @@ unless ENV['IN_DOCKER'] == 'true' or ENV['BAZEL'] == 'true' end end -if RUBY_PLATFORM == "java" - task :clean => :require_mvn do - system("mvn --batch-mode clean") - end - - task :compile => :require_mvn do - system("mvn --batch-mode package") - end - - task :require_mvn do - raise ArgumentError, "maven needs to be installed" if `which mvn` == '' - end - -else - unless ENV['IN_DOCKER'] == 'true' +task :copy_third_party do + unless File.exist? 'ext/google/protobuf_c/third_party/utf8_range' + puts "copy_third_party was called and had actual work to do" # We need utf8_range in-tree. if ENV['BAZEL'] == 'true' utf8_root = '../external/utf8_range' else utf8_root = '../third_party/utf8_range' end - FileUtils.mkdir_p("ext/google/protobuf_c") FileUtils.cp(utf8_root+"/utf8_range.h", "ext/google/protobuf_c") FileUtils.cp(utf8_root+"/naive.c", "ext/google/protobuf_c") FileUtils.cp(utf8_root+"/range2-neon.c", "ext/google/protobuf_c") FileUtils.cp(utf8_root+"/range2-sse.c", "ext/google/protobuf_c") end +end + +task :copy_shared do + FileUtils.cp(Dir.glob("ext/google/shared/*.{c,h}"), "ext/google/protobuf_c") +end + +if USE_FFI + # FFI::CompilerTask's constructor walks the filesystem at initialization time + # in order to create subtasks for each source file, so the files from third_party + # must be copied into place before the task is defined. + ffi_compiler_config_block = Proc.new do |c| + c.cflags << "-std=gnu99 -O3 -DNDEBUG" + if RbConfig::CONFIG['target_os'] =~ /darwin|linux/ + # c.cflags << "-fvisibility=hidden -Wall -Wsign-compare -Wno-declaration-after-statement" + c.cflags << "-Wall -Wsign-compare -Wno-declaration-after-statement" + end + if RbConfig::CONFIG['target_os'] =~ /linux/ + # Instruct the linker to point memcpy calls at our __wrap_memcpy wrapper. + c.ldflags << "-Wl,-wrap,memcpy" + end + end + + if File.exist? 'ext/google/protobuf_c/third_party/utf8_range' + desc "compiler tasks" + namespace "ffi-compiler" do + FFI::Compiler::CompileTask.new('ext/google/protobuf_c', &ffi_compiler_config_block) + end + else + task "ffi-compiler:default" do + Rake::Task[:copy_third_party].invoke + desc "compiler tasks" + namespace "ffi-compiler" do + FFI::Compiler::CompileTask.new('ext/google/protobuf_c', &ffi_compiler_config_block) + Rake::Task[Rake::Task[:default].prereqs.first].invoke + end + end + end +else + if RUBY_PLATFORM == "java" + task :clean => :require_mvn do + system("mvn --batch-mode clean") + end + + task :compile => :require_mvn do + system("mvn --batch-mode package") + end + + task :require_mvn do + raise ArgumentError, "maven needs to be installed" if `which mvn` == '' + end + + else + unless ENV['IN_DOCKER'] == 'true' + task :build => [:copy_third_party, :copy_shared] + end Rake::ExtensionTask.new("protobuf_c", spec) do |ext| unless RUBY_PLATFORM =~ /darwin/ @@ -147,11 +193,16 @@ else task 'gem:native' => [:genproto, 'gem:windows', 'gem:java'] end end +end + +task :compile => ["ffi-compiler:default"] if USE_FFI task :genproto => genproto_output task :clean do sh "rm -f #{genproto_output.join(' ')}" + sh "rm -rf `find ext/google/protobuf_c/* -maxdepth 0 -type d`" + sh "rm -rf pkg" end Gem::PackageTask.new(spec) do |pkg| diff --git a/ruby/ext/google/protobuf_c/convert.c b/ruby/ext/google/protobuf_c/convert.c index bdc71599fbe9..07823fbc6ded 100644 --- a/ruby/ext/google/protobuf_c/convert.c +++ b/ruby/ext/google/protobuf_c/convert.c @@ -41,6 +41,7 @@ #include "message.h" #include "protobuf.h" +#include "shared_convert.h" static upb_StringView Convert_StringData(VALUE str, upb_Arena* arena) { upb_StringView ret; @@ -312,50 +313,24 @@ upb_MessageValue Msgval_DeepCopy(upb_MessageValue msgval, TypeInfo type_info, bool Msgval_IsEqual(upb_MessageValue val1, upb_MessageValue val2, TypeInfo type_info) { - switch (type_info.type) { - case kUpb_CType_Bool: - return memcmp(&val1, &val2, 1) == 0; - case kUpb_CType_Float: - case kUpb_CType_Int32: - case kUpb_CType_UInt32: - case kUpb_CType_Enum: - return memcmp(&val1, &val2, 4) == 0; - case kUpb_CType_Double: - case kUpb_CType_Int64: - case kUpb_CType_UInt64: - return memcmp(&val1, &val2, 8) == 0; - case kUpb_CType_String: - case kUpb_CType_Bytes: - return val1.str_val.size == val2.str_val.size && - memcmp(val1.str_val.data, val2.str_val.data, val1.str_val.size) == - 0; - case kUpb_CType_Message: - return Message_Equal(val1.msg_val, val2.msg_val, type_info.def.msgdef); - default: - rb_raise(rb_eRuntimeError, "Internal error, unexpected type"); + upb_Status status; + upb_Status_Clear(&status); + bool return_value = shared_Msgval_IsEqual(val1, val2, type_info.type, type_info.def.msgdef, &status); + if (upb_Status_IsOk(&status)) { + return return_value; + } else { + rb_raise(rb_eRuntimeError, upb_Status_ErrorMessage(&status)); } } uint64_t Msgval_GetHash(upb_MessageValue val, TypeInfo type_info, uint64_t seed) { - switch (type_info.type) { - case kUpb_CType_Bool: - return _upb_Hash(&val, 1, seed); - case kUpb_CType_Float: - case kUpb_CType_Int32: - case kUpb_CType_UInt32: - case kUpb_CType_Enum: - return _upb_Hash(&val, 4, seed); - case kUpb_CType_Double: - case kUpb_CType_Int64: - case kUpb_CType_UInt64: - return _upb_Hash(&val, 8, seed); - case kUpb_CType_String: - case kUpb_CType_Bytes: - return _upb_Hash(val.str_val.data, val.str_val.size, seed); - case kUpb_CType_Message: - return Message_Hash(val.msg_val, type_info.def.msgdef, seed); - default: - rb_raise(rb_eRuntimeError, "Internal error, unexpected type"); + upb_Status status; + upb_Status_Clear(&status); + bool return_value = shared_Msgval_GetHash(val, type_info.type, type_info.def.msgdef, seed, &status); + if (upb_Status_IsOk(&status)) { + return return_value; + } else { + rb_raise(rb_eRuntimeError, upb_Status_ErrorMessage(&status)); } } diff --git a/ruby/ext/google/shared/shared_convert.c b/ruby/ext/google/shared/shared_convert.c new file mode 100644 index 000000000000..a19c1dfe5629 --- /dev/null +++ b/ruby/ext/google/shared/shared_convert.c @@ -0,0 +1,85 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// ----------------------------------------------------------------------------- +// Ruby <-> upb data conversion functions. Strictly free of dependencies on +// Ruby interpreter internals. + +#include "shared_convert.h" + +bool shared_Msgval_IsEqual(upb_MessageValue val1, upb_MessageValue val2, + upb_CType type, upb_MessageDef* msgdef, upb_Status* status) { + switch (type) { + case kUpb_CType_Bool: + return memcmp(&val1, &val2, 1) == 0; + case kUpb_CType_Float: + case kUpb_CType_Int32: + case kUpb_CType_UInt32: + case kUpb_CType_Enum: + return memcmp(&val1, &val2, 4) == 0; + case kUpb_CType_Double: + case kUpb_CType_Int64: + case kUpb_CType_UInt64: + return memcmp(&val1, &val2, 8) == 0; + case kUpb_CType_String: + case kUpb_CType_Bytes: + return val1.str_val.size == val2.str_val.size && + memcmp(val1.str_val.data, val2.str_val.data, val1.str_val.size) == + 0; + case kUpb_CType_Message: + return Message_Equal(val1.msg_val, val2.msg_val, msgdef); + default: + upb_Status_SetErrorMessage(status, "Internal error, unexpected type"); + } +} + +uint64_t shared_Msgval_GetHash(upb_MessageValue val, upb_CType type, upb_MessageDef* msgdef, + uint64_t seed, upb_Status* status) { + switch (type) { + case kUpb_CType_Bool: + return _upb_Hash(&val, 1, seed); + case kUpb_CType_Float: + case kUpb_CType_Int32: + case kUpb_CType_UInt32: + case kUpb_CType_Enum: + return _upb_Hash(&val, 4, seed); + case kUpb_CType_Double: + case kUpb_CType_Int64: + case kUpb_CType_UInt64: + return _upb_Hash(&val, 8, seed); + case kUpb_CType_String: + case kUpb_CType_Bytes: + return _upb_Hash(val.str_val.data, val.str_val.size, seed); + case kUpb_CType_Message: + return Message_Hash(val.msg_val, msgdef, seed); + default: + upb_Status_SetErrorMessage(status, "Internal error, unexpected type"); + } +} \ No newline at end of file diff --git a/ruby/ext/google/shared/shared_convert.h b/ruby/ext/google/shared/shared_convert.h new file mode 100644 index 000000000000..53f5ccddf013 --- /dev/null +++ b/ruby/ext/google/shared/shared_convert.h @@ -0,0 +1,41 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// ----------------------------------------------------------------------------- +// Ruby <-> upb data conversion functions. Strictly free of dependencies on +// Ruby interpreter internals. + +#include "ruby-upb.h" + +bool shared_Msgval_IsEqual(upb_MessageValue val1, upb_MessageValue val2, + upb_CType type, upb_MessageDef* msgdef, upb_Status* status); + +uint64_t shared_Msgval_GetHash(upb_MessageValue val, upb_CType type, upb_MessageDef* msgdef, + uint64_t seed, upb_Status* status); \ No newline at end of file From a5ff141cde354cc43d0e5cb6718c538557fd9d38 Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Wed, 11 Jan 2023 12:56:51 -0500 Subject: [PATCH 12/54] Fix copy/paste casting bug. --- ruby/.gitignore | 10 +++++++++- ruby/ext/google/protobuf_c/convert.c | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/ruby/.gitignore b/ruby/.gitignore index 653309818f32..8a688943bed4 100644 --- a/ruby/.gitignore +++ b/ruby/.gitignore @@ -6,4 +6,12 @@ protobuf-jruby.iml target/ pkg/ tmp/ -tests/google/ \ No newline at end of file +tests/google/ +/ext/google/protobuf_c/shared_convert.c +/ext/google/protobuf_c/shared_convert.h +/ext/google/protobuf_c/utf8_range.h +/ext/google/protobuf_c/range2-sse.c +/ext/google/protobuf_c/range2-neon.c +/ext/google/protobuf_c/naive.c +/ext/google/protobuf_c/shared_message.c +/ext/google/protobuf_c/shared_message.h diff --git a/ruby/ext/google/protobuf_c/convert.c b/ruby/ext/google/protobuf_c/convert.c index 07823fbc6ded..73dbe0084b34 100644 --- a/ruby/ext/google/protobuf_c/convert.c +++ b/ruby/ext/google/protobuf_c/convert.c @@ -327,7 +327,7 @@ uint64_t Msgval_GetHash(upb_MessageValue val, TypeInfo type_info, uint64_t seed) { upb_Status status; upb_Status_Clear(&status); - bool return_value = shared_Msgval_GetHash(val, type_info.type, type_info.def.msgdef, seed, &status); + uint64_t return_value = shared_Msgval_GetHash(val, type_info.type, type_info.def.msgdef, seed, &status); if (upb_Status_IsOk(&status)) { return return_value; } else { From cec78501445ea7b14bda9595accba3bef0a99e39 Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Wed, 11 Jan 2023 12:59:15 -0500 Subject: [PATCH 13/54] Refactor message.c to separate CRuby-specific glue that has dependencies on Ruby headers from functionality needed by FFI. --- ruby/ext/google/protobuf_c/extconf.rb | 3 +- ruby/ext/google/protobuf_c/message.c | 51 ++++----------- ruby/ext/google/shared/shared_message.c | 87 +++++++++++++++++++++++++ ruby/ext/google/shared/shared_message.h | 48 ++++++++++++++ 4 files changed, 150 insertions(+), 39 deletions(-) create mode 100644 ruby/ext/google/shared/shared_message.c create mode 100644 ruby/ext/google/shared/shared_message.h diff --git a/ruby/ext/google/protobuf_c/extconf.rb b/ruby/ext/google/protobuf_c/extconf.rb index b7c439b0b2a6..4bb49bb21570 100755 --- a/ruby/ext/google/protobuf_c/extconf.rb +++ b/ruby/ext/google/protobuf_c/extconf.rb @@ -22,6 +22,7 @@ $srcs = ["protobuf.c", "convert.c", "defs.c", "message.c", "repeated_field.c", "map.c", "ruby-upb.c", "wrap_memcpy.c", - "naive.c", "range2-neon.c", "range2-sse.c"] + "naive.c", "range2-neon.c", "range2-sse.c", "shared_convert.c", + "shared_message.c"] create_makefile(ext_name) diff --git a/ruby/ext/google/protobuf_c/message.c b/ruby/ext/google/protobuf_c/message.c index 8208cb650cb5..033574a12545 100644 --- a/ruby/ext/google/protobuf_c/message.c +++ b/ruby/ext/google/protobuf_c/message.c @@ -35,6 +35,7 @@ #include "map.h" #include "protobuf.h" #include "repeated_field.h" +#include "shared_message.h" static VALUE cParseError = Qnil; static VALUE cAbstractMessage = Qnil; @@ -692,29 +693,13 @@ static VALUE Message_dup(VALUE _self) { // Support function for Message_eq, and also used by other #eq functions. bool Message_Equal(const upb_Message* m1, const upb_Message* m2, const upb_MessageDef* m) { - if (m1 == m2) return true; - - size_t size1, size2; - int encode_opts = - kUpb_EncodeOption_SkipUnknown | kUpb_EncodeOption_Deterministic; - upb_Arena* arena_tmp = upb_Arena_New(); - const upb_MiniTable* layout = upb_MessageDef_MiniTable(m); - - // Compare deterministically serialized payloads with no unknown fields. - char* data1; - char* data2; - upb_EncodeStatus status1 = - upb_Encode(m1, layout, encode_opts, arena_tmp, &data1, &size1); - upb_EncodeStatus status2 = - upb_Encode(m2, layout, encode_opts, arena_tmp, &data2, &size2); - - if (status1 == kUpb_EncodeStatus_Ok && status2 == kUpb_EncodeStatus_Ok) { - bool ret = (size1 == size2) && (memcmp(data1, data2, size1) == 0); - upb_Arena_Free(arena_tmp); - return ret; + upb_Status status; + upb_Status_Clear(&status); + bool return_value = shared_Message_Equal(m1, m2, m, &status); + if (upb_Status_IsOk(&status)) { + return return_value; } else { - upb_Arena_Free(arena_tmp); - rb_raise(cParseError, "Error comparing messages"); + rb_raise(cParseError, upb_Status_ErrorMessage(&status)); } } @@ -739,23 +724,13 @@ static VALUE Message_eq(VALUE _self, VALUE _other) { uint64_t Message_Hash(const upb_Message* msg, const upb_MessageDef* m, uint64_t seed) { - upb_Arena* arena = upb_Arena_New(); - char* data; - size_t size; - - // Hash a deterministically serialized payloads with no unknown fields. - upb_EncodeStatus status = upb_Encode( - msg, upb_MessageDef_MiniTable(m), - kUpb_EncodeOption_SkipUnknown | kUpb_EncodeOption_Deterministic, arena, - &data, &size); - - if (status == kUpb_EncodeStatus_Ok) { - uint64_t ret = _upb_Hash(data, size, seed); - upb_Arena_Free(arena); - return ret; + upb_Status status; + upb_Status_Clear(&status); + uint64_t return_value = shared_Message_Hash(msg, m, seed, &status); + if (upb_Status_IsOk(&status)) { + return return_value; } else { - upb_Arena_Free(arena); - rb_raise(cParseError, "Error calculating hash"); + rb_raise(cParseError, upb_Status_ErrorMessage(&status)); } } diff --git a/ruby/ext/google/shared/shared_message.c b/ruby/ext/google/shared/shared_message.c new file mode 100644 index 000000000000..da9fd9636992 --- /dev/null +++ b/ruby/ext/google/shared/shared_message.c @@ -0,0 +1,87 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// ----------------------------------------------------------------------------- +// Ruby Message functions. Strictly free of dependencies on +// Ruby interpreter internals. + +#include "shared_message.h" + +// Support function for Message_Hash. Returns a hash value for the given message. +uint64_t shared_Message_Hash(const upb_Message* msg, const upb_MessageDef* m, + uint64_t seed, upb_Status* status) { + upb_Arena* arena = upb_Arena_New(); + char* data; + size_t size; + + // Hash a deterministically serialized payloads with no unknown fields. + upb_EncodeStatus encode_status = upb_Encode( + msg, upb_MessageDef_MiniTable(m), + kUpb_EncodeOption_SkipUnknown | kUpb_EncodeOption_Deterministic, arena, + &data, &size); + + if (encode_status == kUpb_EncodeStatus_Ok) { + uint64_t ret = _upb_Hash(data, size, seed); + upb_Arena_Free(arena); + return ret; + } else { + upb_Arena_Free(arena); + upb_Status_SetErrorMessage(status, "Error calculating hash"); + } +} + +// Support function for Message_Equal +bool shared_Message_Equal(const upb_Message* m1, const upb_Message* m2, + const upb_MessageDef* m, upb_Status* status) { + if (m1 == m2) return true; + + size_t size1, size2; + int encode_opts = + kUpb_EncodeOption_SkipUnknown | kUpb_EncodeOption_Deterministic; + upb_Arena* arena_tmp = upb_Arena_New(); + const upb_MiniTable* layout = upb_MessageDef_MiniTable(m); + + // Compare deterministically serialized payloads with no unknown fields. + char* data1; + char* data2; + upb_EncodeStatus status1 = + upb_Encode(m1, layout, encode_opts, arena_tmp, &data1, &size1); + upb_EncodeStatus status2 = + upb_Encode(m2, layout, encode_opts, arena_tmp, &data2, &size2); + + if (status1 == kUpb_EncodeStatus_Ok && status2 == kUpb_EncodeStatus_Ok) { + bool ret = (size1 == size2) && (memcmp(data1, data2, size1) == 0); + upb_Arena_Free(arena_tmp); + return ret; + } else { + upb_Arena_Free(arena_tmp); + upb_Status_SetErrorMessage(status, "Error comparing messages"); + } +} diff --git a/ruby/ext/google/shared/shared_message.h b/ruby/ext/google/shared/shared_message.h new file mode 100644 index 000000000000..0e46d2e6a816 --- /dev/null +++ b/ruby/ext/google/shared/shared_message.h @@ -0,0 +1,48 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// ----------------------------------------------------------------------------- +// Ruby Message functions. Strictly free of dependencies on +// Ruby interpreter internals. + +#ifndef RUBY_PROTOBUF_SHARED_MESSAGE_H_ +#define RUBY_PROTOBUF_SHARED_MESSAGE_H_ + +#include "ruby-upb.h" + +// Returns a hash value for the given message. +uint64_t shared_Message_Hash(const upb_Message* msg, const upb_MessageDef* m, + uint64_t seed, upb_Status* status); + +// Returns true if these two messages are equal. +bool shared_Message_Equal(const upb_Message* m1, const upb_Message* m2, + const upb_MessageDef* m, upb_Status* status); + +#endif // RUBY_PROTOBUF_SHARED_MESSAGE_H_ From 4734f344eefc34a9365371390a3e5b174d844af7 Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Wed, 11 Jan 2023 13:01:44 -0500 Subject: [PATCH 14/54] Have shared_convert methods invoke shared_message methods directly. Add `ifndef` armor around shared_convert.h. --- ruby/ext/google/shared/shared_convert.c | 4 ++-- ruby/ext/google/shared/shared_convert.h | 8 +++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/ruby/ext/google/shared/shared_convert.c b/ruby/ext/google/shared/shared_convert.c index a19c1dfe5629..57a8be7501ce 100644 --- a/ruby/ext/google/shared/shared_convert.c +++ b/ruby/ext/google/shared/shared_convert.c @@ -54,7 +54,7 @@ bool shared_Msgval_IsEqual(upb_MessageValue val1, upb_MessageValue val2, memcmp(val1.str_val.data, val2.str_val.data, val1.str_val.size) == 0; case kUpb_CType_Message: - return Message_Equal(val1.msg_val, val2.msg_val, msgdef); + return shared_Message_Equal(val1.msg_val, val2.msg_val, msgdef, status); default: upb_Status_SetErrorMessage(status, "Internal error, unexpected type"); } @@ -78,7 +78,7 @@ uint64_t shared_Msgval_GetHash(upb_MessageValue val, upb_CType type, upb_Message case kUpb_CType_Bytes: return _upb_Hash(val.str_val.data, val.str_val.size, seed); case kUpb_CType_Message: - return Message_Hash(val.msg_val, msgdef, seed); + return shared_Message_Hash(val.msg_val, msgdef, seed, status); default: upb_Status_SetErrorMessage(status, "Internal error, unexpected type"); } diff --git a/ruby/ext/google/shared/shared_convert.h b/ruby/ext/google/shared/shared_convert.h index 53f5ccddf013..8659cca772a4 100644 --- a/ruby/ext/google/shared/shared_convert.h +++ b/ruby/ext/google/shared/shared_convert.h @@ -32,10 +32,16 @@ // Ruby <-> upb data conversion functions. Strictly free of dependencies on // Ruby interpreter internals. +#ifndef RUBY_PROTOBUF_SHARED_CONVERT_H_ +#define RUBY_PROTOBUF_SHARED_CONVERT_H_ + #include "ruby-upb.h" +#include "shared_message.h" bool shared_Msgval_IsEqual(upb_MessageValue val1, upb_MessageValue val2, upb_CType type, upb_MessageDef* msgdef, upb_Status* status); uint64_t shared_Msgval_GetHash(upb_MessageValue val, upb_CType type, upb_MessageDef* msgdef, - uint64_t seed, upb_Status* status); \ No newline at end of file + uint64_t seed, upb_Status* status); + +#endif // RUBY_PROTOBUF_SHARED_CONVERT_H_ From 55cd7fef19f3dcf6a6e25f8a8a8758dd72658bd7 Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Mon, 31 Oct 2022 14:31:39 -0400 Subject: [PATCH 15/54] branch to support FFI --- ruby/lib/google/protobuf.rb | 69 --------------------------------- ruby/lib/google/protobuf_ffi.rb | 66 +++++++++++++++---------------- 2 files changed, 32 insertions(+), 103 deletions(-) delete mode 100644 ruby/lib/google/protobuf.rb diff --git a/ruby/lib/google/protobuf.rb b/ruby/lib/google/protobuf.rb deleted file mode 100644 index 1dd898e3b44e..000000000000 --- a/ruby/lib/google/protobuf.rb +++ /dev/null @@ -1,69 +0,0 @@ -# Protocol Buffers - Google's data interchange format -# Copyright 2008 Google Inc. All rights reserved. -# https://developers.google.com/protocol-buffers/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# require mixins before we hook them into the java & c code -require 'google/protobuf/message_exts' - -# We define these before requiring the platform-specific modules. -# That way the module init can grab references to these. -module Google - module Protobuf - class Error < StandardError; end - class ParseError < Error; end - class TypeError < ::TypeError; end - - def self.encode(msg, options = {}) - msg.to_proto(options) - end - - def self.encode_json(msg, options = {}) - msg.to_json(options) - end - - def self.decode(klass, proto, options = {}) - klass.decode(proto, options) - end - - def self.decode_json(klass, json, options = {}) - klass.decode_json(json, options) - end - - IMPLEMENTATION = case ENV['PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION'] - when nil, 'native', '' - require 'google/protobuf_native' - :NATIVE - when 'ffi' - require 'google/protobuf_ffi' - :FFI - else - raise RuntimeError.new("Unsupported value of PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION environment variable `#{ENV['PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION']}`. Must be one of `native` or `ffi`. Defaults to `native`.") - end - end -end diff --git a/ruby/lib/google/protobuf_ffi.rb b/ruby/lib/google/protobuf_ffi.rb index a635b2edcb8c..860513cbad92 100644 --- a/ruby/lib/google/protobuf_ffi.rb +++ b/ruby/lib/google/protobuf_ffi.rb @@ -28,44 +28,42 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -require 'ffi' -require 'google/protobuf/ffi/internal/type_safety' -require 'google/protobuf/ffi/internal/arena' -require 'google/protobuf/ffi/internal/convert' -require 'google/protobuf/ffi/descriptor' -require 'google/protobuf/ffi/enum_descriptor' -require 'google/protobuf/ffi/field_descriptor' -require 'google/protobuf/ffi/oneof_descriptor' -require 'google/protobuf/ffi/ffi' -require 'google/protobuf/ffi/descriptor_pool' -require 'google/protobuf/ffi/file_descriptor' -require 'google/protobuf/ffi/map' -require 'google/protobuf/ffi/object_cache' -require 'google/protobuf/ffi/repeated_field' -require 'google/protobuf/descriptor_dsl' +# require mixins before we hook them into the java & c code +require 'google/protobuf/message_exts' +# We define these before requiring the platform-specific modules. +# That way the module init can grab references to these. module Google module Protobuf - def self.deep_copy(object) - case object - when RepeatedField - RepeatedField.send(:deep_copy, object) - when Google::Protobuf::Map - Google::Protobuf::Map.deep_copy(object) - when Google::Protobuf::MessageExts - object.class.send(:deep_copy, object.instance_variable_get(:@msg)) - else - raise NotImplementedError - end + class Error < StandardError; end + class ParseError < Error; end + class TypeError < ::TypeError; end + + def self.encode(msg, options = {}) + msg.to_proto(options) end - def self.discard_unknown(message) - raise FrozenError if message.frozen? - raise ArgumentError.new "Expected message, got #{message.class} instead." if message.instance_variable_get(:@msg).nil? - unless Google::Protobuf::FFI.message_discard_unknown(message.instance_variable_get(:@msg), message.class.descriptor, 128) - raise RuntimeError.new "Messages nested too deeply." - end - nil + def self.encode_json(msg, options = {}) + msg.to_json(options) + end + + def self.decode(klass, proto, options = {}) + klass.decode(proto, options) + end + + def self.decode_json(klass, json, options = {}) + klass.decode_json(json, options) + end + + IMPLEMENTATION = case ENV['PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION'] + when nil, 'native', '' + require 'google/protobuf_native' + :NATIVE + when 'ffi' + require 'google/protobuf_ffi' + :FFI + else + raise RuntimeError.new("Unsupported value of PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION environment variable `#{ENV['PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION']}`. Must be one of `native` or `ffi`. Defaults to `native`.") end end -end \ No newline at end of file +end From 2f3838b9828ee7ff6dfb937528b555cee9112539 Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Mon, 31 Oct 2022 14:32:37 -0400 Subject: [PATCH 16/54] restore protobuf.rb --- ruby/lib/google/protobuf.rb | 79 +++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 ruby/lib/google/protobuf.rb diff --git a/ruby/lib/google/protobuf.rb b/ruby/lib/google/protobuf.rb new file mode 100644 index 000000000000..b7a671105158 --- /dev/null +++ b/ruby/lib/google/protobuf.rb @@ -0,0 +1,79 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# require mixins before we hook them into the java & c code +require 'google/protobuf/message_exts' + +# We define these before requiring the platform-specific modules. +# That way the module init can grab references to these. +module Google + module Protobuf + class Error < StandardError; end + class ParseError < Error; end + class TypeError < ::TypeError; end + end +end + +if RUBY_PLATFORM == "java" + require 'json' + require 'google/protobuf_java' +else + begin + require "google/#{RUBY_VERSION.sub(/\.\d+$/, '')}/protobuf_c" + rescue LoadError + require 'google/protobuf_c' + end + +end + +require 'google/protobuf/descriptor_dsl' +require 'google/protobuf/repeated_field' + +module Google + module Protobuf + + def self.encode(msg, options = {}) + msg.to_proto(options) + end + + def self.encode_json(msg, options = {}) + msg.to_json(options) + end + + def self.decode(klass, proto, options = {}) + klass.decode(proto, options) + end + + def self.decode_json(klass, json, options = {}) + klass.decode_json(json, options) + end + + end +end From c8656c4b40508fa35792bc5d9a789a05a8072e2f Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Mon, 31 Oct 2022 14:40:08 -0400 Subject: [PATCH 17/54] native implementation --- ruby/lib/google/protobuf.rb | 79 ------------------------------ ruby/lib/google/protobuf_native.rb | 38 +++++++++++++- 2 files changed, 37 insertions(+), 80 deletions(-) delete mode 100644 ruby/lib/google/protobuf.rb diff --git a/ruby/lib/google/protobuf.rb b/ruby/lib/google/protobuf.rb deleted file mode 100644 index b7a671105158..000000000000 --- a/ruby/lib/google/protobuf.rb +++ /dev/null @@ -1,79 +0,0 @@ -# Protocol Buffers - Google's data interchange format -# Copyright 2008 Google Inc. All rights reserved. -# https://developers.google.com/protocol-buffers/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# require mixins before we hook them into the java & c code -require 'google/protobuf/message_exts' - -# We define these before requiring the platform-specific modules. -# That way the module init can grab references to these. -module Google - module Protobuf - class Error < StandardError; end - class ParseError < Error; end - class TypeError < ::TypeError; end - end -end - -if RUBY_PLATFORM == "java" - require 'json' - require 'google/protobuf_java' -else - begin - require "google/#{RUBY_VERSION.sub(/\.\d+$/, '')}/protobuf_c" - rescue LoadError - require 'google/protobuf_c' - end - -end - -require 'google/protobuf/descriptor_dsl' -require 'google/protobuf/repeated_field' - -module Google - module Protobuf - - def self.encode(msg, options = {}) - msg.to_proto(options) - end - - def self.encode_json(msg, options = {}) - msg.to_json(options) - end - - def self.decode(klass, proto, options = {}) - klass.decode(proto, options) - end - - def self.decode_json(klass, json, options = {}) - klass.decode_json(json, options) - end - - end -end diff --git a/ruby/lib/google/protobuf_native.rb b/ruby/lib/google/protobuf_native.rb index f55d6c48ae6e..b7a671105158 100644 --- a/ruby/lib/google/protobuf_native.rb +++ b/ruby/lib/google/protobuf_native.rb @@ -1,5 +1,5 @@ # Protocol Buffers - Google's data interchange format -# Copyright 2021 Google Inc. All rights reserved. +# Copyright 2008 Google Inc. All rights reserved. # https://developers.google.com/protocol-buffers/ # # Redistribution and use in source and binary forms, with or without @@ -28,6 +28,19 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# require mixins before we hook them into the java & c code +require 'google/protobuf/message_exts' + +# We define these before requiring the platform-specific modules. +# That way the module init can grab references to these. +module Google + module Protobuf + class Error < StandardError; end + class ParseError < Error; end + class TypeError < ::TypeError; end + end +end + if RUBY_PLATFORM == "java" require 'json' require 'google/protobuf_java' @@ -37,7 +50,30 @@ rescue LoadError require 'google/protobuf_c' end + end require 'google/protobuf/descriptor_dsl' require 'google/protobuf/repeated_field' + +module Google + module Protobuf + + def self.encode(msg, options = {}) + msg.to_proto(options) + end + + def self.encode_json(msg, options = {}) + msg.to_json(options) + end + + def self.decode(klass, proto, options = {}) + klass.decode(proto, options) + end + + def self.decode_json(klass, json, options = {}) + klass.decode_json(json, options) + end + + end +end From 1c19455c957146d426bfce04df3831947704a27d Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Mon, 31 Oct 2022 14:40:40 -0400 Subject: [PATCH 18/54] restore protobuf.rb --- ruby/lib/google/protobuf.rb | 79 +++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 ruby/lib/google/protobuf.rb diff --git a/ruby/lib/google/protobuf.rb b/ruby/lib/google/protobuf.rb new file mode 100644 index 000000000000..b7a671105158 --- /dev/null +++ b/ruby/lib/google/protobuf.rb @@ -0,0 +1,79 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# require mixins before we hook them into the java & c code +require 'google/protobuf/message_exts' + +# We define these before requiring the platform-specific modules. +# That way the module init can grab references to these. +module Google + module Protobuf + class Error < StandardError; end + class ParseError < Error; end + class TypeError < ::TypeError; end + end +end + +if RUBY_PLATFORM == "java" + require 'json' + require 'google/protobuf_java' +else + begin + require "google/#{RUBY_VERSION.sub(/\.\d+$/, '')}/protobuf_c" + rescue LoadError + require 'google/protobuf_c' + end + +end + +require 'google/protobuf/descriptor_dsl' +require 'google/protobuf/repeated_field' + +module Google + module Protobuf + + def self.encode(msg, options = {}) + msg.to_proto(options) + end + + def self.encode_json(msg, options = {}) + msg.to_json(options) + end + + def self.decode(klass, proto, options = {}) + klass.decode(proto, options) + end + + def self.decode_json(klass, json, options = {}) + klass.decode_json(json, options) + end + + end +end From f461e72ad682ad54e9727868d9ff5baa7b78c3f4 Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Mon, 31 Oct 2022 15:10:55 -0400 Subject: [PATCH 19/54] Refactor `protobuf.rb` to separate native and ffi implementations. --- ruby/lib/google/protobuf.rb | 30 +++++-------- ruby/lib/google/protobuf_ffi.rb | 68 +++++++++++++++--------------- ruby/lib/google/protobuf_native.rb | 38 +---------------- 3 files changed, 46 insertions(+), 90 deletions(-) diff --git a/ruby/lib/google/protobuf.rb b/ruby/lib/google/protobuf.rb index b7a671105158..5dfd91f5dd44 100644 --- a/ruby/lib/google/protobuf.rb +++ b/ruby/lib/google/protobuf.rb @@ -38,26 +38,6 @@ module Protobuf class Error < StandardError; end class ParseError < Error; end class TypeError < ::TypeError; end - end -end - -if RUBY_PLATFORM == "java" - require 'json' - require 'google/protobuf_java' -else - begin - require "google/#{RUBY_VERSION.sub(/\.\d+$/, '')}/protobuf_c" - rescue LoadError - require 'google/protobuf_c' - end - -end - -require 'google/protobuf/descriptor_dsl' -require 'google/protobuf/repeated_field' - -module Google - module Protobuf def self.encode(msg, options = {}) msg.to_proto(options) @@ -75,5 +55,15 @@ def self.decode_json(klass, json, options = {}) klass.decode_json(json, options) end + IMPLEMENTATION = case ENV['PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION'] + when nil, 'native', '' + require 'google/protobuf_native' + :NATIVE + when 'ffi' + require 'google/protobuf_native' + :FFI + else + raise RuntimeError("Unsupport value of PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION environment variable `#{ENV['PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION']}`. Must be one of `native` or `ffi`. Defaults to `native`.") + end end end diff --git a/ruby/lib/google/protobuf_ffi.rb b/ruby/lib/google/protobuf_ffi.rb index 860513cbad92..8652410247ff 100644 --- a/ruby/lib/google/protobuf_ffi.rb +++ b/ruby/lib/google/protobuf_ffi.rb @@ -1,5 +1,5 @@ # Protocol Buffers - Google's data interchange format -# Copyright 2021 Google Inc. All rights reserved. +# Copyright 2008 Google Inc. All rights reserved. # https://developers.google.com/protocol-buffers/ # # Redistribution and use in source and binary forms, with or without @@ -28,42 +28,44 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# require mixins before we hook them into the java & c code -require 'google/protobuf/message_exts' +require 'ffi' +require 'google/protobuf/ffi/internal/type_safety' +require 'google/protobuf/ffi/internal/arena' +require 'google/protobuf/ffi/internal/convert' +require 'google/protobuf/ffi/descriptor' +require 'google/protobuf/ffi/enum_descriptor' +require 'google/protobuf/ffi/field_descriptor' +require 'google/protobuf/ffi/oneof_descriptor' +require 'google/protobuf/ffi/ffi' +require 'google/protobuf/ffi/descriptor_pool' +require 'google/protobuf/ffi/file_descriptor' +require 'google/protobuf/ffi/map' +require 'google/protobuf/ffi/object_cache' +require 'google/protobuf/ffi/repeated_field' +require 'google/protobuf/descriptor_dsl' -# We define these before requiring the platform-specific modules. -# That way the module init can grab references to these. module Google module Protobuf - class Error < StandardError; end - class ParseError < Error; end - class TypeError < ::TypeError; end - - def self.encode(msg, options = {}) - msg.to_proto(options) - end - - def self.encode_json(msg, options = {}) - msg.to_json(options) - end - - def self.decode(klass, proto, options = {}) - klass.decode(proto, options) - end - - def self.decode_json(klass, json, options = {}) - klass.decode_json(json, options) + def self.deep_copy(object) + case object + when RepeatedField + RepeatedField.send(:deep_copy, object) + when Google::Protobuf::Map + Google::Protobuf::Map.deep_copy(object) + when Google::Protobuf::MessageExts + object.class.send(:deep_copy, object.instance_variable_get(:@msg)) + else + raise NotImplementedError + end end - IMPLEMENTATION = case ENV['PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION'] - when nil, 'native', '' - require 'google/protobuf_native' - :NATIVE - when 'ffi' - require 'google/protobuf_ffi' - :FFI - else - raise RuntimeError.new("Unsupported value of PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION environment variable `#{ENV['PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION']}`. Must be one of `native` or `ffi`. Defaults to `native`.") + def self.discard_unknown(message) + raise FrozenError if message.frozen? + raise ArgumentError.new "Expected message, got #{message.class} instead." if message.instance_variable_get(:@msg).nil? + unless Google::Protobuf::FFI.message_discard_unknown(message.instance_variable_get(:@msg), message.class.descriptor, 128) + raise RuntimeError.new "Messages nested too deeply." + end + nil end end -end +end \ No newline at end of file diff --git a/ruby/lib/google/protobuf_native.rb b/ruby/lib/google/protobuf_native.rb index b7a671105158..f55d6c48ae6e 100644 --- a/ruby/lib/google/protobuf_native.rb +++ b/ruby/lib/google/protobuf_native.rb @@ -1,5 +1,5 @@ # Protocol Buffers - Google's data interchange format -# Copyright 2008 Google Inc. All rights reserved. +# Copyright 2021 Google Inc. All rights reserved. # https://developers.google.com/protocol-buffers/ # # Redistribution and use in source and binary forms, with or without @@ -28,19 +28,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# require mixins before we hook them into the java & c code -require 'google/protobuf/message_exts' - -# We define these before requiring the platform-specific modules. -# That way the module init can grab references to these. -module Google - module Protobuf - class Error < StandardError; end - class ParseError < Error; end - class TypeError < ::TypeError; end - end -end - if RUBY_PLATFORM == "java" require 'json' require 'google/protobuf_java' @@ -50,30 +37,7 @@ class TypeError < ::TypeError; end rescue LoadError require 'google/protobuf_c' end - end require 'google/protobuf/descriptor_dsl' require 'google/protobuf/repeated_field' - -module Google - module Protobuf - - def self.encode(msg, options = {}) - msg.to_proto(options) - end - - def self.encode_json(msg, options = {}) - msg.to_json(options) - end - - def self.decode(klass, proto, options = {}) - klass.decode(proto, options) - end - - def self.decode_json(klass, json, options = {}) - klass.decode_json(json, options) - end - - end -end From 1b1e64ef9a99ddbac62fcccd8440cb65b5ab8ea9 Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Fri, 6 Jan 2023 14:48:06 -0500 Subject: [PATCH 20/54] Fix spelling `Unsupport` -> `Unsupported` --- ruby/lib/google/protobuf.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ruby/lib/google/protobuf.rb b/ruby/lib/google/protobuf.rb index 5dfd91f5dd44..29b367352f19 100644 --- a/ruby/lib/google/protobuf.rb +++ b/ruby/lib/google/protobuf.rb @@ -60,10 +60,10 @@ def self.decode_json(klass, json, options = {}) require 'google/protobuf_native' :NATIVE when 'ffi' - require 'google/protobuf_native' + require 'google/protobuf_ffi' :FFI else - raise RuntimeError("Unsupport value of PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION environment variable `#{ENV['PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION']}`. Must be one of `native` or `ffi`. Defaults to `native`.") + raise RuntimeError("Unsupported value of PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION environment variable `#{ENV['PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION']}`. Must be one of `native` or `ffi`. Defaults to `native`.") end end end From 4cbcd56dd1ca9bd04fbce24a8d72cfac809d85ab Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Fri, 6 Jan 2023 17:41:01 -0500 Subject: [PATCH 21/54] Fix exception raising by properly instantiating the exception. --- ruby/lib/google/protobuf.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ruby/lib/google/protobuf.rb b/ruby/lib/google/protobuf.rb index 29b367352f19..1dd898e3b44e 100644 --- a/ruby/lib/google/protobuf.rb +++ b/ruby/lib/google/protobuf.rb @@ -63,7 +63,7 @@ def self.decode_json(klass, json, options = {}) require 'google/protobuf_ffi' :FFI else - raise RuntimeError("Unsupported value of PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION environment variable `#{ENV['PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION']}`. Must be one of `native` or `ffi`. Defaults to `native`.") + raise RuntimeError.new("Unsupported value of PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION environment variable `#{ENV['PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION']}`. Must be one of `native` or `ffi`. Defaults to `native`.") end end end From 5d714370a326528a56c2f207813db540abcf556b Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Thu, 6 Jul 2023 16:42:36 -0400 Subject: [PATCH 22/54] Stop ignoring shared source files; ignore FFI Compiler output directory and protoc outputs instead. --- .gitignore | 11 +++++++++++ ruby/.gitignore | 5 +---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index df64e362181d..b7e56b6516ea 100644 --- a/.gitignore +++ b/.gitignore @@ -165,6 +165,17 @@ vendor/ /bazel-* # ruby test output +ruby/lib/google/protobuf/any_pb.rb +ruby/lib/google/protobuf/api_pb.rb +ruby/lib/google/protobuf/descriptor_pb.rb +ruby/lib/google/protobuf/duration_pb.rb +ruby/lib/google/protobuf/empty_pb.rb +ruby/lib/google/protobuf/field_mask_pb.rb +ruby/lib/google/protobuf/source_context_pb.rb +ruby/lib/google/protobuf/struct_pb.rb +ruby/lib/google/protobuf/timestamp_pb.rb +ruby/lib/google/protobuf/type_pb.rb +ruby/lib/google/protobuf/wrappers_pb.rb ruby/tests/basic_test_pb.rb ruby/tests/basic_test_proto2_pb.rb ruby/tests/generated_code_pb.rb diff --git a/ruby/.gitignore b/ruby/.gitignore index 8a688943bed4..0dffaf138ace 100644 --- a/ruby/.gitignore +++ b/ruby/.gitignore @@ -7,11 +7,8 @@ target/ pkg/ tmp/ tests/google/ -/ext/google/protobuf_c/shared_convert.c -/ext/google/protobuf_c/shared_convert.h /ext/google/protobuf_c/utf8_range.h /ext/google/protobuf_c/range2-sse.c /ext/google/protobuf_c/range2-neon.c /ext/google/protobuf_c/naive.c -/ext/google/protobuf_c/shared_message.c -/ext/google/protobuf_c/shared_message.h +/ext/google/x86_64-darwin/ From b40211991c3899623a000972ad64e8ea21402961 Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Thu, 6 Jul 2023 16:45:52 -0400 Subject: [PATCH 23/54] Update function names and arguments. --- ruby/lib/google/protobuf/ffi/descriptor.rb | 6 ++++-- ruby/lib/google/protobuf/ffi/ffi.rb | 21 +++++++++++++++------ 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/ruby/lib/google/protobuf/ffi/descriptor.rb b/ruby/lib/google/protobuf/ffi/descriptor.rb index fa30c1693146..b9432fdbf9ea 100644 --- a/ruby/lib/google/protobuf/ffi/descriptor.rb +++ b/ruby/lib/google/protobuf/ffi/descriptor.rb @@ -755,8 +755,10 @@ def self.encode_internal(msg, encoding_options = 0) mini_table_ptr = Google::Protobuf::FFI.get_mini_table(descriptor) size_ptr = ::FFI::MemoryPointer.new(:size_t, 1) - encoding = Google::Protobuf::FFI.encode_message(msg, mini_table_ptr, encoding_options, temporary_arena, size_ptr) - yield encoding, size_ptr.read(:size_t), mini_table_ptr + pointer_ptr = ::FFI::MemoryPointer.new(:pointer, 1) + encoding_status = Google::Protobuf::FFI.encode_message(msg, mini_table_ptr, encoding_options, temporary_arena, pointer_ptr.to_ptr, size_ptr) + raise "Encoding failed due to #{encoding_status}" unless encoding_status == :Ok + yield pointer_ptr.read(:pointer), size_ptr.read(:size_t), mini_table_ptr end end end diff --git a/ruby/lib/google/protobuf/ffi/ffi.rb b/ruby/lib/google/protobuf/ffi/ffi.rb index 9a9ac0639420..0ebcb791797a 100644 --- a/ruby/lib/google/protobuf/ffi/ffi.rb +++ b/ruby/lib/google/protobuf/ffi/ffi.rb @@ -197,6 +197,15 @@ class MutableMessageValue < ::FFI::Union :MissingRequired, ) + EncodeStatus = enum( + :Ok, + :OutOfMemory, # Arena alloc failed + :MaxDepthExceeded, # Exceeded UPB_DECODE_MAXDEPTH + + # CheckRequired failed, but the parse otherwise succeeded. + :MissingRequired, + ) + class MiniTable < ::FFI::Struct layout :subs, :pointer, :fields, :pointer, @@ -329,10 +338,10 @@ def initialize # Message attach_function :clear_message_field, :upb_Message_ClearField, [:Message, FieldDescriptor], :void - attach_function :get_message_value, :upb_Message_Get, [:Message, FieldDescriptor], MessageValue.by_value - attach_function :get_message_has, :upb_Message_Has, [:Message, FieldDescriptor], :bool - attach_function :set_message_field, :upb_Message_Set, [:Message, FieldDescriptor, MessageValue.by_value, Arena], :bool - attach_function :encode_message, :upb_Encode, [:Message, MiniTable.by_ref, :size_t, Arena, :pointer], :binary_string + attach_function :get_message_value, :upb_Message_GetFieldByDef, [:Message, FieldDescriptor], MessageValue.by_value + attach_function :get_message_has, :upb_Message_HasFieldByDef, [:Message, FieldDescriptor], :bool + attach_function :set_message_field, :upb_Message_SetFieldByDef, [:Message, FieldDescriptor, MessageValue.by_value, Arena], :bool + attach_function :encode_message, :upb_Encode, [:Message, MiniTable.by_ref, :size_t, Arena, :pointer, :pointer], EncodeStatus attach_function :json_decode_message, :upb_JsonDecode, [:binary_string, :size_t, :Message, MessageDef, :DefPool, :int, Arena, Status.by_ref], :bool attach_function :json_encode_message, :upb_JsonEncode, [:Message, MessageDef, :DefPool, :int, :binary_string, :size_t, Status.by_ref], :size_t attach_function :decode_message, :upb_Decode, [:binary_string, :size_t, :Message, MiniTable.by_ref, :ExtensionRegistry, :int, Arena], DecodeStatus @@ -341,8 +350,8 @@ def initialize attach_function :message_discard_unknown, :upb_Message_DiscardUnknown, [:Message, MessageDef, :int], :bool # MessageValue - attach_function :message_value_equal, :Msgval_IsEqual, [MessageValue.by_value, MessageValue.by_value, CType, MessageDef], :bool - attach_function :message_value_hash, :Msgval_GetHash, [MessageValue.by_value, CType, MessageDef, :uint64_t], :uint64_t + attach_function :message_value_equal, :shared_Msgval_IsEqual, [MessageValue.by_value, MessageValue.by_value, CType, MessageDef], :bool + attach_function :message_value_hash, :shared_Msgval_GetHash, [MessageValue.by_value, CType, MessageDef, :uint64_t], :uint64_t # OneofDescriptor attach_function :get_oneof_name, :upb_OneofDef_Name, [OneofDef], :string From f7083f13c70f2cc002c693dc8fc8bbabff394355 Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Thu, 6 Jul 2023 16:49:58 -0400 Subject: [PATCH 24/54] Remove references to `PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION`. --- ruby/Rakefile | 160 ++++++++++++++++++++++------------- ruby/google-protobuf.gemspec | 26 +++--- ruby/lib/google/protobuf.rb | 28 ++++-- 3 files changed, 133 insertions(+), 81 deletions(-) diff --git a/ruby/Rakefile b/ruby/Rakefile index 0e02f7c4dff9..5a8fd5e3c1a0 100644 --- a/ruby/Rakefile +++ b/ruby/Rakefile @@ -2,10 +2,13 @@ require "rubygems" require "rubygems/package_task" require "rake/extensiontask" unless RUBY_PLATFORM == "java" require "rake/testtask" - -USE_FFI = ENV['PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION'] == 'ffi' - -require "ffi-compiler/compile_task" if USE_FFI +begin + require 'ffi' + require "ffi-compiler/compile_task" + USE_FFI = true +rescue LoadError + USE_FFI = false +end spec = Gem::Specification.load("google-protobuf.gemspec") @@ -51,6 +54,85 @@ end genproto_output = [] +if USE_FFI + common_excluded_source = [ + :convert, :defs, :map, :message, :protobuf, :repeated_field, :wrap_memcpy + ] + common_c_flags = "-std=gnu99 -O3 -DNDEBUG" + # Compile Ruby UPB separately in order to limit use of -DUPB_BUILD_API to one + # compilation unit. + desc "compiler tasks" + namespace "ffi-upb" do + FFI::Compiler::CompileTask.new('ruby-upb') do |c| + c.cflags << common_c_flags + c.cflags << "-DUPB_BUILD_API" + common_excluded_source.each { |file| c.exclude << "/#{file}.c" } + c.exclude << "/glue.c" + c.exclude << "/shared_message.c" + c.exclude << "/shared_convert.c" + c.ext_dir = File.dirname('ext/google/protobuf_c') + c.source_dirs = [c.ext_dir] + if RbConfig::CONFIG['target_os'] =~ /darwin|linux/ + c.cflags << "-fvisibility=hidden -Wall -Wsign-compare -Wno-declaration-after-statement" + end + end + end + # FFI::CompilerTask's constructor walks the filesystem at initialization time + # in order to create subtasks for each source file, so the files from third_party + # must be copied into place before the task is defined. + ffi_compiler_config_block = Proc.new do |c| + c.cflags << common_c_flags + common_excluded_source.each { | file | c.exclude << "/#{file}.c" } + # Ruby UPB was already compiled with different flags. + c.exclude << "/range2-neon.c" + c.exclude << "/range2-sse.c" + c.exclude << "/naive.c" + c.exclude << "/ruby-upb.c" + # ffi-upb:default's only dependency is the platform-specific path of the + # shared library. Ignore the shared library (or figure out how to pass the + # right flags to build one shared lib from another?) but explicitly add the + # object files that make it up as ldflags so that their symbols are + # available. + # Rake::Task[Rake::Task[:"ffi-upb:default"].prereqs.first].prereqs.each do |prereq| + # c.ldflags << prereq + # end + if RbConfig::CONFIG['target_os'] =~ /darwin|linux/ + c.cflags << "-Wall -Wsign-compare -Wno-declaration-after-statement" + end + end + + if File.exist? 'ext/google/protobuf_c/utf8_range.h' + desc "compiler tasks" + namespace "ffi-compiler" do + FFI::Compiler::CompileTask.new('ext/google/protobuf_c', &ffi_compiler_config_block) + protobuf_c_task = Rake::Task[:"ffi-compiler:default"] + protobuf_c_shared_lib_task = Rake::Task[protobuf_c_task.prereqs.first] + Rake::Task[Rake::Task[:"ffi-upb:default"].prereqs.first].prereqs.each do |dependency| + protobuf_c_shared_lib_task.prereqs.prepend dependency + end + end + else + # Define a placeholder task that triggers copying third_party source before + # redefining the task. + task "ffi-compiler:default" do + Rake::Task[:copy_third_party].invoke + desc "compiler tasks" + namespace "ffi-compiler" do + FFI::Compiler::CompileTask.new('ext/google/protobuf_c', &ffi_compiler_config_block) + protobuf_c_task = Rake::Task[:"ffi-compiler:default"] + Rake::Task[:"ffi-upb:default"].prereqs.each do |dependency| + protobuf_c_task.prereqs.prepend dependency + end + protobuf_c_task.prereqs.each do |dependency| + Rake::Task[dependency].invoke + end + end + end + end + + task :compile => ["ffi-compiler:default"] +end + # We won't have access to .. from within docker, but the proto files # will be there, thanks to the :genproto rule dependency for gem:native. unless ENV['IN_DOCKER'] == 'true' or ENV['BAZEL'] == 'true' @@ -73,8 +155,7 @@ unless ENV['IN_DOCKER'] == 'true' or ENV['BAZEL'] == 'true' end task :copy_third_party do - unless File.exist? 'ext/google/protobuf_c/third_party/utf8_range' - puts "copy_third_party was called and had actual work to do" + unless File.exist? 'ext/google/protobuf_c/utf8_range.h' # We need utf8_range in-tree. if ENV['BAZEL'] == 'true' utf8_root = '../external/utf8_range' @@ -88,59 +169,23 @@ task :copy_third_party do end end -task :copy_shared do - FileUtils.cp(Dir.glob("ext/google/shared/*.{c,h}"), "ext/google/protobuf_c") -end - -if USE_FFI - # FFI::CompilerTask's constructor walks the filesystem at initialization time - # in order to create subtasks for each source file, so the files from third_party - # must be copied into place before the task is defined. - ffi_compiler_config_block = Proc.new do |c| - c.cflags << "-std=gnu99 -O3 -DNDEBUG" - if RbConfig::CONFIG['target_os'] =~ /darwin|linux/ - # c.cflags << "-fvisibility=hidden -Wall -Wsign-compare -Wno-declaration-after-statement" - c.cflags << "-Wall -Wsign-compare -Wno-declaration-after-statement" - end - if RbConfig::CONFIG['target_os'] =~ /linux/ - # Instruct the linker to point memcpy calls at our __wrap_memcpy wrapper. - c.ldflags << "-Wl,-wrap,memcpy" - end +if RUBY_PLATFORM == "java" + task :clean => :require_mvn do + system("mvn --batch-mode clean") end - if File.exist? 'ext/google/protobuf_c/third_party/utf8_range' - desc "compiler tasks" - namespace "ffi-compiler" do - FFI::Compiler::CompileTask.new('ext/google/protobuf_c', &ffi_compiler_config_block) - end - else - task "ffi-compiler:default" do - Rake::Task[:copy_third_party].invoke - desc "compiler tasks" - namespace "ffi-compiler" do - FFI::Compiler::CompileTask.new('ext/google/protobuf_c', &ffi_compiler_config_block) - Rake::Task[Rake::Task[:default].prereqs.first].invoke - end - end + task :compile => :require_mvn do + system("mvn --batch-mode package") end -else - if RUBY_PLATFORM == "java" - task :clean => :require_mvn do - system("mvn --batch-mode clean") - end - - task :compile => :require_mvn do - system("mvn --batch-mode package") - end - task :require_mvn do - raise ArgumentError, "maven needs to be installed" if `which mvn` == '' - end + task :require_mvn do + raise ArgumentError, "maven needs to be installed" if `which mvn` == '' + end - else - unless ENV['IN_DOCKER'] == 'true' - task :build => [:copy_third_party, :copy_shared] - end +else + unless ENV['IN_DOCKER'] == 'true' + task :build => [:copy_third_party] + end Rake::ExtensionTask.new("protobuf_c", spec) do |ext| unless RUBY_PLATFORM =~ /darwin/ @@ -157,7 +202,7 @@ else 'x86_64-linux', 'x86-linux', 'x86_64-darwin', 'arm64-darwin', ] - end + end unless USE_FFI task 'gem:java' do sh "rm Gemfile.lock" @@ -193,7 +238,6 @@ else task 'gem:native' => [:genproto, 'gem:windows', 'gem:java'] end end -end task :compile => ["ffi-compiler:default"] if USE_FFI @@ -201,8 +245,10 @@ task :genproto => genproto_output task :clean do sh "rm -f #{genproto_output.join(' ')}" - sh "rm -rf `find ext/google/protobuf_c/* -maxdepth 0 -type d`" + sh "rm -rf ext/google/x86_64-darwin" sh "rm -rf pkg" + sh "rm -rf tmp" + sh "rm -f lib/google/protobuf_c.bundle" end Gem::PackageTask.new(spec) do |pkg| diff --git a/ruby/google-protobuf.gemspec b/ruby/google-protobuf.gemspec index f57b98a84198..698553f03803 100644 --- a/ruby/google-protobuf.gemspec +++ b/ruby/google-protobuf.gemspec @@ -12,22 +12,16 @@ Gem::Specification.new do |s| s.require_paths = ["lib"] s.files = Dir.glob('lib/**/*.rb') s.test_files = %w[tests/basic.rb tests/stress.rb tests/generated_code_test.rb] - case ENV['PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION'] - when nil, 'native', '' - if RUBY_PLATFORM == "java" - s.platform = "java" - s.files += ["lib/google/protobuf_java.jar"] - else - s.files += Dir.glob('ext/**/*') - s.extensions= ["ext/google/protobuf_c/extconf.rb"] - s.add_development_dependency "rake-compiler-dock", "= 1.2.1" - end - when 'ffi' - s.add_dependency "ffi", "~>1" - s.add_dependency "ffi-compiler", "~>1" - s.extensions = Dir.glob(%w[Rakefile ext/**/*.{c,h}]) - else - raise RuntimeError.new("Unsupported value of PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION environment variable `#{ENV['PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION']}`. If set, must be one of `native` or `ffi`. Defaults to `native`.") + if RUBY_PLATFORM == "java" + s.platform = "java" + s.files += ["lib/google/protobuf_java.jar"] + s.add_dependency "ffi", "~>1" + s.add_dependency "ffi-compiler", "~>1" + s.extensions = Dir.glob(%w[Rakefile ext/**/*.{c,h}]) + else + s.files += Dir.glob('ext/**/*') + s.extensions= ["ext/google/protobuf_c/extconf.rb"] + s.add_development_dependency "rake-compiler-dock", "= 1.2.1" end s.required_ruby_version = '>= 2.3' s.add_development_dependency "ffi", "~>1" diff --git a/ruby/lib/google/protobuf.rb b/ruby/lib/google/protobuf.rb index 1dd898e3b44e..855ab605e95b 100644 --- a/ruby/lib/google/protobuf.rb +++ b/ruby/lib/google/protobuf.rb @@ -39,6 +39,13 @@ class Error < StandardError; end class ParseError < Error; end class TypeError < ::TypeError; end + begin + require 'ffi' + PREFER_FFI = true + rescue LoadError + PREFER_FFI = false + end + def self.encode(msg, options = {}) msg.to_proto(options) end @@ -55,15 +62,20 @@ def self.decode_json(klass, json, options = {}) klass.decode_json(json, options) end - IMPLEMENTATION = case ENV['PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION'] - when nil, 'native', '' - require 'google/protobuf_native' - :NATIVE - when 'ffi' + IMPLEMENTATION = if PREFER_FFI + begin require 'google/protobuf_ffi' - :FFI - else - raise RuntimeError.new("Unsupported value of PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION environment variable `#{ENV['PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION']}`. Must be one of `native` or `ffi`. Defaults to `native`.") + :FFI + rescue LoadError + warn "Caught exception `#{$!.message}` while loading FFI implementation of google/protobuf." + raise $! + warn "Falling back to native implementation." + require 'google/protobuf_native' + :NATIVE + end + else + require 'google/protobuf_native' + :NATIVE end end end From 039af515087f2d310d246805e243c0d409c0757f Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Thu, 6 Jul 2023 16:50:29 -0400 Subject: [PATCH 25/54] Relocate shared files. --- ruby/ext/google/{shared => protobuf_c}/shared_convert.c | 0 ruby/ext/google/{shared => protobuf_c}/shared_convert.h | 0 ruby/ext/google/{shared => protobuf_c}/shared_message.c | 0 ruby/ext/google/{shared => protobuf_c}/shared_message.h | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename ruby/ext/google/{shared => protobuf_c}/shared_convert.c (100%) rename ruby/ext/google/{shared => protobuf_c}/shared_convert.h (100%) rename ruby/ext/google/{shared => protobuf_c}/shared_message.c (100%) rename ruby/ext/google/{shared => protobuf_c}/shared_message.h (100%) diff --git a/ruby/ext/google/shared/shared_convert.c b/ruby/ext/google/protobuf_c/shared_convert.c similarity index 100% rename from ruby/ext/google/shared/shared_convert.c rename to ruby/ext/google/protobuf_c/shared_convert.c diff --git a/ruby/ext/google/shared/shared_convert.h b/ruby/ext/google/protobuf_c/shared_convert.h similarity index 100% rename from ruby/ext/google/shared/shared_convert.h rename to ruby/ext/google/protobuf_c/shared_convert.h diff --git a/ruby/ext/google/shared/shared_message.c b/ruby/ext/google/protobuf_c/shared_message.c similarity index 100% rename from ruby/ext/google/shared/shared_message.c rename to ruby/ext/google/protobuf_c/shared_message.c diff --git a/ruby/ext/google/shared/shared_message.h b/ruby/ext/google/protobuf_c/shared_message.h similarity index 100% rename from ruby/ext/google/shared/shared_message.h rename to ruby/ext/google/protobuf_c/shared_message.h From 8607cda876b05f66e40c68e39cb4b219e3047a9a Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Thu, 6 Jul 2023 16:51:58 -0400 Subject: [PATCH 26/54] Temporarily update visibility, pending update to upb repo. --- ruby/ext/google/protobuf_c/ruby-upb.h | 122 +++++++++++++------------- 1 file changed, 61 insertions(+), 61 deletions(-) diff --git a/ruby/ext/google/protobuf_c/ruby-upb.h b/ruby/ext/google/protobuf_c/ruby-upb.h index 8606a4b58d5c..b853782ee227 100755 --- a/ruby/ext/google/protobuf_c/ruby-upb.h +++ b/ruby/ext/google/protobuf_c/ruby-upb.h @@ -1035,16 +1035,16 @@ UPB_API void upb_Map_SetEntryValue(upb_Map* map, size_t iter, */ // Advances to the next entry. Returns false if no more entries are present. -bool upb_MapIterator_Next(const upb_Map* map, size_t* iter); +UPB_API_INLINE bool upb_MapIterator_Next(const upb_Map* map, size_t* iter); // Returns true if the iterator still points to a valid entry, or false if the // iterator is past the last element. It is an error to call this function with // kUpb_Map_Begin (you must call next() at least once first). -bool upb_MapIterator_Done(const upb_Map* map, size_t iter); +UPB_API_INLINE bool upb_MapIterator_Done(const upb_Map* map, size_t iter); // Returns the key and value for this entry of the map. -upb_MessageValue upb_MapIterator_Key(const upb_Map* map, size_t iter); -upb_MessageValue upb_MapIterator_Value(const upb_Map* map, size_t iter); +UPB_API_INLINE upb_MessageValue upb_MapIterator_Key(const upb_Map* map, size_t iter); +UPB_API_INLINE upb_MessageValue upb_MapIterator_Value(const upb_Map* map, size_t iter); #ifdef __cplusplus } /* extern "C" */ @@ -1224,7 +1224,7 @@ UPB_INLINE size_t upb_table_size(const upb_table* t) { UPB_INLINE bool upb_tabent_isempty(const upb_tabent* e) { return e->key == 0; } -uint32_t _upb_Hash(const void* p, size_t n, uint64_t seed); +UPB_API_INLINE uint32_t _upb_Hash(const void* p, size_t n, uint64_t seed); #ifdef __cplusplus } /* extern "C" */ @@ -3419,7 +3419,7 @@ UPB_INLINE int upb_Encode_LimitDepth(uint32_t encode_options, uint32_t limit) { return upb_EncodeOptions_MaxDepth(max_depth) | (encode_options & 0xffff); } -upb_EncodeStatus upb_Encode(const void* msg, const upb_MiniTable* l, +UPB_API_INLINE upb_EncodeStatus upb_Encode(const void* msg, const upb_MiniTable* l, int options, upb_Arena* arena, char** buf, size_t* size); @@ -10207,17 +10207,17 @@ const void* _upb_DefType_Unpack(upb_value v, upb_deftype_t type); extern "C" { #endif -void upb_DefPool_Free(upb_DefPool* s); +UPB_API_INLINE void upb_DefPool_Free(upb_DefPool* s); -upb_DefPool* upb_DefPool_New(void); +UPB_API_INLINE upb_DefPool* upb_DefPool_New(void); -const upb_MessageDef* upb_DefPool_FindMessageByName(const upb_DefPool* s, +UPB_API_INLINE const upb_MessageDef* upb_DefPool_FindMessageByName(const upb_DefPool* s, const char* sym); const upb_MessageDef* upb_DefPool_FindMessageByNameWithSize( const upb_DefPool* s, const char* sym, size_t len); -const upb_EnumDef* upb_DefPool_FindEnumByName(const upb_DefPool* s, +UPB_API_INLINE const upb_EnumDef* upb_DefPool_FindEnumByName(const upb_DefPool* s, const char* sym); const upb_EnumValueDef* upb_DefPool_FindEnumByNameval(const upb_DefPool* s, @@ -10252,7 +10252,7 @@ const upb_ServiceDef* upb_DefPool_FindServiceByNameWithSize( const upb_FileDef* upb_DefPool_FindFileContainingSymbol(const upb_DefPool* s, const char* name); -const upb_FileDef* upb_DefPool_AddFile(upb_DefPool* s, +UPB_API const upb_FileDef* upb_DefPool_AddFile(upb_DefPool* s, const UPB_DESC(FileDescriptorProto) * file_proto, upb_Status* status); @@ -10286,14 +10286,14 @@ extern "C" { bool upb_EnumDef_CheckNumber(const upb_EnumDef* e, int32_t num); const upb_MessageDef* upb_EnumDef_ContainingType(const upb_EnumDef* e); int32_t upb_EnumDef_Default(const upb_EnumDef* e); -const upb_FileDef* upb_EnumDef_File(const upb_EnumDef* e); +UPB_API_INLINE const upb_FileDef* upb_EnumDef_File(const upb_EnumDef* e); const upb_EnumValueDef* upb_EnumDef_FindValueByName(const upb_EnumDef* e, const char* name); -const upb_EnumValueDef* upb_EnumDef_FindValueByNameWithSize( +UPB_API_INLINE const upb_EnumValueDef* upb_EnumDef_FindValueByNameWithSize( const upb_EnumDef* e, const char* name, size_t size); -const upb_EnumValueDef* upb_EnumDef_FindValueByNumber(const upb_EnumDef* e, +UPB_API_INLINE const upb_EnumValueDef* upb_EnumDef_FindValueByNumber(const upb_EnumDef* e, int32_t num); -const char* upb_EnumDef_FullName(const upb_EnumDef* e); +UPB_API_INLINE const char* upb_EnumDef_FullName(const upb_EnumDef* e); bool upb_EnumDef_HasOptions(const upb_EnumDef* e); bool upb_EnumDef_IsClosed(const upb_EnumDef* e); @@ -10311,8 +10311,8 @@ const upb_EnumReservedRange* upb_EnumDef_ReservedRange(const upb_EnumDef* e, int i); int upb_EnumDef_ReservedRangeCount(const upb_EnumDef* e); -const upb_EnumValueDef* upb_EnumDef_Value(const upb_EnumDef* e, int i); -int upb_EnumDef_ValueCount(const upb_EnumDef* e); +UPB_API_INLINE const upb_EnumValueDef* upb_EnumDef_Value(const upb_EnumDef* e, int i); +UPB_API_INLINE int upb_EnumDef_ValueCount(const upb_EnumDef* e); #ifdef __cplusplus } /* extern "C" */ @@ -10337,8 +10337,8 @@ const upb_EnumDef* upb_EnumValueDef_Enum(const upb_EnumValueDef* v); const char* upb_EnumValueDef_FullName(const upb_EnumValueDef* v); bool upb_EnumValueDef_HasOptions(const upb_EnumValueDef* v); uint32_t upb_EnumValueDef_Index(const upb_EnumValueDef* v); -const char* upb_EnumValueDef_Name(const upb_EnumValueDef* v); -int32_t upb_EnumValueDef_Number(const upb_EnumValueDef* v); +UPB_API_INLINE const char* upb_EnumValueDef_Name(const upb_EnumValueDef* v); +UPB_API_INLINE int32_t upb_EnumValueDef_Number(const upb_EnumValueDef* v); const UPB_DESC(EnumValueOptions) * upb_EnumValueDef_Options(const upb_EnumValueDef* v); @@ -10392,42 +10392,42 @@ extern "C" { #endif const upb_OneofDef* upb_FieldDef_ContainingOneof(const upb_FieldDef* f); -const upb_MessageDef* upb_FieldDef_ContainingType(const upb_FieldDef* f); -upb_CType upb_FieldDef_CType(const upb_FieldDef* f); -upb_MessageValue upb_FieldDef_Default(const upb_FieldDef* f); -const upb_EnumDef* upb_FieldDef_EnumSubDef(const upb_FieldDef* f); +UPB_API_INLINE const upb_MessageDef* upb_FieldDef_ContainingType(const upb_FieldDef* f); +UPB_API_INLINE upb_CType upb_FieldDef_CType(const upb_FieldDef* f); +UPB_API_INLINE upb_MessageValue upb_FieldDef_Default(const upb_FieldDef* f); +UPB_API_INLINE const upb_EnumDef* upb_FieldDef_EnumSubDef(const upb_FieldDef* f); const upb_MessageDef* upb_FieldDef_ExtensionScope(const upb_FieldDef* f); const upb_FileDef* upb_FieldDef_File(const upb_FieldDef* f); const char* upb_FieldDef_FullName(const upb_FieldDef* f); bool upb_FieldDef_HasDefault(const upb_FieldDef* f); bool upb_FieldDef_HasJsonName(const upb_FieldDef* f); bool upb_FieldDef_HasOptions(const upb_FieldDef* f); -bool upb_FieldDef_HasPresence(const upb_FieldDef* f); +UPB_API_INLINE bool upb_FieldDef_HasPresence(const upb_FieldDef* f); bool upb_FieldDef_HasSubDef(const upb_FieldDef* f); uint32_t upb_FieldDef_Index(const upb_FieldDef* f); bool upb_FieldDef_IsExtension(const upb_FieldDef* f); -bool upb_FieldDef_IsMap(const upb_FieldDef* f); +UPB_API_INLINE bool upb_FieldDef_IsMap(const upb_FieldDef* f); bool upb_FieldDef_IsOptional(const upb_FieldDef* f); bool upb_FieldDef_IsPacked(const upb_FieldDef* f); bool upb_FieldDef_IsPrimitive(const upb_FieldDef* f); -bool upb_FieldDef_IsRepeated(const upb_FieldDef* f); +UPB_API_INLINE bool upb_FieldDef_IsRepeated(const upb_FieldDef* f); bool upb_FieldDef_IsRequired(const upb_FieldDef* f); bool upb_FieldDef_IsString(const upb_FieldDef* f); -bool upb_FieldDef_IsSubMessage(const upb_FieldDef* f); -const char* upb_FieldDef_JsonName(const upb_FieldDef* f); -upb_Label upb_FieldDef_Label(const upb_FieldDef* f); -const upb_MessageDef* upb_FieldDef_MessageSubDef(const upb_FieldDef* f); +UPB_API_INLINE bool upb_FieldDef_IsSubMessage(const upb_FieldDef* f); +UPB_API_INLINE const char* upb_FieldDef_JsonName(const upb_FieldDef* f); +UPB_API_INLINE upb_Label upb_FieldDef_Label(const upb_FieldDef* f); +UPB_API_INLINE const upb_MessageDef* upb_FieldDef_MessageSubDef(const upb_FieldDef* f); // Creates a mini descriptor string for a field, returns true on success. bool upb_FieldDef_MiniDescriptorEncode(const upb_FieldDef* f, upb_Arena* a, upb_StringView* out); const upb_MiniTableField* upb_FieldDef_MiniTable(const upb_FieldDef* f); -const char* upb_FieldDef_Name(const upb_FieldDef* f); -uint32_t upb_FieldDef_Number(const upb_FieldDef* f); +UPB_API_INLINE const char* upb_FieldDef_Name(const upb_FieldDef* f); +UPB_API_INLINE uint32_t upb_FieldDef_Number(const upb_FieldDef* f); const UPB_DESC(FieldOptions) * upb_FieldDef_Options(const upb_FieldDef* f); -const upb_OneofDef* upb_FieldDef_RealContainingOneof(const upb_FieldDef* f); -upb_FieldType upb_FieldDef_Type(const upb_FieldDef* f); +UPB_API_INLINE const upb_OneofDef* upb_FieldDef_RealContainingOneof(const upb_FieldDef* f); +UPB_API_INLINE upb_FieldType upb_FieldDef_Type(const upb_FieldDef* f); #ifdef __cplusplus } /* extern "C" */ @@ -10451,11 +10451,11 @@ extern "C" { const upb_FileDef* upb_FileDef_Dependency(const upb_FileDef* f, int i); int upb_FileDef_DependencyCount(const upb_FileDef* f); bool upb_FileDef_HasOptions(const upb_FileDef* f); -const char* upb_FileDef_Name(const upb_FileDef* f); +UPB_API_INLINE const char* upb_FileDef_Name(const upb_FileDef* f); const UPB_DESC(FileOptions) * upb_FileDef_Options(const upb_FileDef* f); const char* upb_FileDef_Package(const upb_FileDef* f); const char* upb_FileDef_Edition(const upb_FileDef* f); -const upb_DefPool* upb_FileDef_Pool(const upb_FileDef* f); +UPB_API_INLINE const upb_DefPool* upb_FileDef_Pool(const upb_FileDef* f); const upb_FileDef* upb_FileDef_PublicDependency(const upb_FileDef* f, int i); int upb_FileDef_PublicDependencyCount(const upb_FileDef* f); @@ -10463,7 +10463,7 @@ int upb_FileDef_PublicDependencyCount(const upb_FileDef* f); const upb_ServiceDef* upb_FileDef_Service(const upb_FileDef* f, int i); int upb_FileDef_ServiceCount(const upb_FileDef* f); -upb_Syntax upb_FileDef_Syntax(const upb_FileDef* f); +UPB_API_INLINE upb_Syntax upb_FileDef_Syntax(const upb_FileDef* f); const upb_EnumDef* upb_FileDef_TopLevelEnum(const upb_FileDef* f, int i); int upb_FileDef_TopLevelEnumCount(const upb_FileDef* f); @@ -10545,10 +10545,10 @@ const upb_ExtensionRange* upb_MessageDef_ExtensionRange(const upb_MessageDef* m, int i); int upb_MessageDef_ExtensionRangeCount(const upb_MessageDef* m); -const upb_FieldDef* upb_MessageDef_Field(const upb_MessageDef* m, int i); -int upb_MessageDef_FieldCount(const upb_MessageDef* m); +UPB_API_INLINE const upb_FieldDef* upb_MessageDef_Field(const upb_MessageDef* m, int i); +UPB_API_INLINE int upb_MessageDef_FieldCount(const upb_MessageDef* m); -const upb_FileDef* upb_MessageDef_File(const upb_MessageDef* m); +UPB_API_INLINE const upb_FileDef* upb_MessageDef_File(const upb_MessageDef* m); // Returns a field by either JSON name or regular proto name. const upb_FieldDef* upb_MessageDef_FindByJsonNameWithSize( @@ -10561,7 +10561,7 @@ UPB_INLINE const upb_FieldDef* upb_MessageDef_FindByJsonName( // Lookup of either field or oneof by name. Returns whether either was found. // If the return is true, then the found def will be set, and the non-found // one set to NULL. -bool upb_MessageDef_FindByNameWithSize(const upb_MessageDef* m, +UPB_API_INLINE bool upb_MessageDef_FindByNameWithSize(const upb_MessageDef* m, const char* name, size_t size, const upb_FieldDef** f, const upb_OneofDef** o); @@ -10574,15 +10574,15 @@ UPB_INLINE bool upb_MessageDef_FindByName(const upb_MessageDef* m, const upb_FieldDef* upb_MessageDef_FindFieldByName(const upb_MessageDef* m, const char* name); -const upb_FieldDef* upb_MessageDef_FindFieldByNameWithSize( +UPB_API_INLINE const upb_FieldDef* upb_MessageDef_FindFieldByNameWithSize( const upb_MessageDef* m, const char* name, size_t size); -const upb_FieldDef* upb_MessageDef_FindFieldByNumber(const upb_MessageDef* m, +UPB_API_INLINE const upb_FieldDef* upb_MessageDef_FindFieldByNumber(const upb_MessageDef* m, uint32_t i); const upb_OneofDef* upb_MessageDef_FindOneofByName(const upb_MessageDef* m, const char* name); -const upb_OneofDef* upb_MessageDef_FindOneofByNameWithSize( +UPB_API_INLINE const upb_OneofDef* upb_MessageDef_FindOneofByNameWithSize( const upb_MessageDef* m, const char* name, size_t size); -const char* upb_MessageDef_FullName(const upb_MessageDef* m); +UPB_API_INLINE const char* upb_MessageDef_FullName(const upb_MessageDef* m); bool upb_MessageDef_HasOptions(const upb_MessageDef* m); bool upb_MessageDef_IsMapEntry(const upb_MessageDef* m); bool upb_MessageDef_IsMessageSet(const upb_MessageDef* m); @@ -10591,7 +10591,7 @@ bool upb_MessageDef_IsMessageSet(const upb_MessageDef* m); bool upb_MessageDef_MiniDescriptorEncode(const upb_MessageDef* m, upb_Arena* a, upb_StringView* out); -const upb_MiniTable* upb_MessageDef_MiniTable(const upb_MessageDef* m); +UPB_API_INLINE const upb_MiniTable* upb_MessageDef_MiniTable(const upb_MessageDef* m); const char* upb_MessageDef_Name(const upb_MessageDef* m); const upb_EnumDef* upb_MessageDef_NestedEnum(const upb_MessageDef* m, int i); @@ -10604,8 +10604,8 @@ int upb_MessageDef_NestedEnumCount(const upb_MessageDef* m); int upb_MessageDef_NestedExtensionCount(const upb_MessageDef* m); int upb_MessageDef_NestedMessageCount(const upb_MessageDef* m); -const upb_OneofDef* upb_MessageDef_Oneof(const upb_MessageDef* m, int i); -int upb_MessageDef_OneofCount(const upb_MessageDef* m); +UPB_API_INLINE const upb_OneofDef* upb_MessageDef_Oneof(const upb_MessageDef* m, int i); +UPB_API_INLINE int upb_MessageDef_OneofCount(const upb_MessageDef* m); int upb_MessageDef_RealOneofCount(const upb_MessageDef* m); const UPB_DESC(MessageOptions) * @@ -10618,8 +10618,8 @@ const upb_MessageReservedRange* upb_MessageDef_ReservedRange( const upb_MessageDef* m, int i); int upb_MessageDef_ReservedRangeCount(const upb_MessageDef* m); -upb_Syntax upb_MessageDef_Syntax(const upb_MessageDef* m); -upb_WellKnown upb_MessageDef_WellKnownType(const upb_MessageDef* m); +UPB_API_INLINE upb_Syntax upb_MessageDef_Syntax(const upb_MessageDef* m); +UPB_API_INLINE upb_WellKnown upb_MessageDef_WellKnownType(const upb_MessageDef* m); #ifdef __cplusplus } /* extern "C" */ @@ -10671,8 +10671,8 @@ extern "C" { #endif const upb_MessageDef* upb_OneofDef_ContainingType(const upb_OneofDef* o); -const upb_FieldDef* upb_OneofDef_Field(const upb_OneofDef* o, int i); -int upb_OneofDef_FieldCount(const upb_OneofDef* o); +UPB_API_INLINE const upb_FieldDef* upb_OneofDef_Field(const upb_OneofDef* o, int i); +UPB_API_INLINE int upb_OneofDef_FieldCount(const upb_OneofDef* o); const char* upb_OneofDef_FullName(const upb_OneofDef* o); bool upb_OneofDef_HasOptions(const upb_OneofDef* o); uint32_t upb_OneofDef_Index(const upb_OneofDef* o); @@ -10684,7 +10684,7 @@ const upb_FieldDef* upb_OneofDef_LookupNameWithSize(const upb_OneofDef* o, size_t size); const upb_FieldDef* upb_OneofDef_LookupNumber(const upb_OneofDef* o, uint32_t num); -const char* upb_OneofDef_Name(const upb_OneofDef* o); +UPB_API_INLINE const char* upb_OneofDef_Name(const upb_OneofDef* o); int upb_OneofDef_numfields(const upb_OneofDef* o); const UPB_DESC(OneofOptions) * upb_OneofDef_Options(const upb_OneofDef* o); @@ -10736,7 +10736,7 @@ extern "C" { enum { upb_JsonDecode_IgnoreUnknown = 1 }; -bool upb_JsonDecode(const char* buf, size_t size, upb_Message* msg, +UPB_API_INLINE bool upb_JsonDecode(const char* buf, size_t size, upb_Message* msg, const upb_MessageDef* m, const upb_DefPool* symtab, int options, upb_Arena* arena, upb_Status* status); @@ -10834,12 +10834,12 @@ extern "C" { // Returns a mutable pointer to a map, array, or submessage value. If the given // arena is non-NULL this will construct a new object if it was not previously // present. May not be called for primitive fields. -upb_MutableMessageValue upb_Message_Mutable(upb_Message* msg, +UPB_API_INLINE upb_MutableMessageValue upb_Message_Mutable(upb_Message* msg, const upb_FieldDef* f, upb_Arena* a); // Returns the field that is set in the oneof, or NULL if none are set. -const upb_FieldDef* upb_Message_WhichOneof(const upb_Message* msg, +UPB_API_INLINE const upb_FieldDef* upb_Message_WhichOneof(const upb_Message* msg, const upb_OneofDef* o); // Clear all data and unknown fields. @@ -10849,10 +10849,10 @@ void upb_Message_ClearByDef(upb_Message* msg, const upb_MessageDef* m); void upb_Message_ClearFieldByDef(upb_Message* msg, const upb_FieldDef* f); // May only be called for fields where upb_FieldDef_HasPresence(f) == true. -bool upb_Message_HasFieldByDef(const upb_Message* msg, const upb_FieldDef* f); +UPB_API_INLINE bool upb_Message_HasFieldByDef(const upb_Message* msg, const upb_FieldDef* f); // Returns the value in the message associated with this field def. -upb_MessageValue upb_Message_GetFieldByDef(const upb_Message* msg, +UPB_API_INLINE upb_MessageValue upb_Message_GetFieldByDef(const upb_Message* msg, const upb_FieldDef* f); // Sets the given field to the given value. For a msg/array/map/string, the @@ -10860,7 +10860,7 @@ upb_MessageValue upb_Message_GetFieldByDef(const upb_Message* msg, // the same arena or a different arena that outlives it). // // Returns false if allocation fails. -bool upb_Message_SetFieldByDef(upb_Message* msg, const upb_FieldDef* f, +UPB_API_INLINE bool upb_Message_SetFieldByDef(upb_Message* msg, const upb_FieldDef* f, upb_MessageValue val, upb_Arena* a); // Iterate over present fields. @@ -10883,7 +10883,7 @@ bool upb_Message_Next(const upb_Message* msg, const upb_MessageDef* m, upb_MessageValue* val, size_t* iter); // Clears all unknown field data from this message and all submessages. -bool upb_Message_DiscardUnknown(upb_Message* msg, const upb_MessageDef* m, +UPB_API_INLINE bool upb_Message_DiscardUnknown(upb_Message* msg, const upb_MessageDef* m, int maxdepth); #ifdef __cplusplus @@ -10923,7 +10923,7 @@ enum { * size (excluding NULL) is returned. This means that a return value >= |size| * implies that the output was truncated. (These are the same semantics as * snprintf()). */ -size_t upb_JsonEncode(const upb_Message* msg, const upb_MessageDef* m, +UPB_API_INLINE size_t upb_JsonEncode(const upb_Message* msg, const upb_MessageDef* m, const upb_DefPool* ext_pool, int options, char* buf, size_t size, upb_Status* status); From 1a716c0bb3c0ee1a6af879a61fa57a79caee7584 Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Thu, 6 Jul 2023 16:53:01 -0400 Subject: [PATCH 27/54] Create glue. --- ruby/ext/google/protobuf_c/glue.c | 48 +++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 ruby/ext/google/protobuf_c/glue.c diff --git a/ruby/ext/google/protobuf_c/glue.c b/ruby/ext/google/protobuf_c/glue.c new file mode 100644 index 000000000000..90615042be4c --- /dev/null +++ b/ruby/ext/google/protobuf_c/glue.c @@ -0,0 +1,48 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// ----------------------------------------------------------------------------- +// Exposing inlined UPB functions. Strictly free of dependencies on +// Ruby interpreter internals. + +upb_Arena* Arena_create() { + return upb_Arena_Init(NULL, 0, &upb_alloc_global); +} + +google_protobuf_FileDescriptorProto* FileDescriptorProto_parse (const char* serialized_file_proto, size_t length) { + upb_Arena* arena = Arena_create(); + return google_protobuf_FileDescriptorProto_parse( + serialized_file_proto, + length, arena); +} + +void* upb_Arena_FastMalloc(upb_Arena* a, size_t size) { + return upb_Arena_Malloc(a, size); +} \ No newline at end of file From e7240f8cc78a394b22011c59d11b36b04fde8364 Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Fri, 7 Jul 2023 07:39:04 -0400 Subject: [PATCH 28/54] Fix unit tests. --- ruby/ext/google/protobuf_c/glue.c | 2 ++ ruby/ext/google/protobuf_c/ruby-upb.h | 2 +- ruby/lib/google/protobuf/ffi/descriptor.rb | 29 ++++++++++++------- .../google/protobuf/ffi/enum_descriptor.rb | 10 +++++-- ruby/lib/google/protobuf/ffi/ffi.rb | 6 ++-- ruby/lib/google/protobuf/ffi/map.rb | 6 ++-- 6 files changed, 35 insertions(+), 20 deletions(-) diff --git a/ruby/ext/google/protobuf_c/glue.c b/ruby/ext/google/protobuf_c/glue.c index 90615042be4c..35cef340e61e 100644 --- a/ruby/ext/google/protobuf_c/glue.c +++ b/ruby/ext/google/protobuf_c/glue.c @@ -32,6 +32,8 @@ // Exposing inlined UPB functions. Strictly free of dependencies on // Ruby interpreter internals. +#include "ruby-upb.h" + upb_Arena* Arena_create() { return upb_Arena_Init(NULL, 0, &upb_alloc_global); } diff --git a/ruby/ext/google/protobuf_c/ruby-upb.h b/ruby/ext/google/protobuf_c/ruby-upb.h index b853782ee227..4ea8b52cff78 100755 --- a/ruby/ext/google/protobuf_c/ruby-upb.h +++ b/ruby/ext/google/protobuf_c/ruby-upb.h @@ -10846,7 +10846,7 @@ UPB_API_INLINE const upb_FieldDef* upb_Message_WhichOneof(const upb_Message* msg void upb_Message_ClearByDef(upb_Message* msg, const upb_MessageDef* m); // Clears any field presence and sets the value back to its default. -void upb_Message_ClearFieldByDef(upb_Message* msg, const upb_FieldDef* f); +UPB_API_INLINE void upb_Message_ClearFieldByDef(upb_Message* msg, const upb_FieldDef* f); // May only be called for fields where upb_FieldDef_HasPresence(f) == true. UPB_API_INLINE bool upb_Message_HasFieldByDef(const upb_Message* msg, const upb_FieldDef* f); diff --git a/ruby/lib/google/protobuf/ffi/descriptor.rb b/ruby/lib/google/protobuf/ffi/descriptor.rb index b9432fdbf9ea..916453c66779 100644 --- a/ruby/lib/google/protobuf/ffi/descriptor.rb +++ b/ruby/lib/google/protobuf/ffi/descriptor.rb @@ -216,13 +216,19 @@ def eql?(other) temporary_arena = Google::Protobuf::FFI.create_arena mini_table = Google::Protobuf::FFI.get_mini_table(self.class.descriptor) size_one = ::FFI::MemoryPointer.new(:size_t, 1) - encoding_one = Google::Protobuf::FFI.encode_message(@msg, mini_table, encoding_options, temporary_arena, size_one) + encoding_one = ::FFI::MemoryPointer.new(:pointer, 1) + encoding_status = Google::Protobuf::FFI.encode_message(@msg, mini_table, encoding_options, temporary_arena, encoding_one.to_ptr, size_one) + raise ParseError.new "Error comparing messages due to #{encoding_status} while encoding LHS of `eql?()`" unless encoding_status == :Ok + size_two = ::FFI::MemoryPointer.new(:size_t, 1) - encoding_two = Google::Protobuf::FFI.encode_message(other.instance_variable_get(:@msg), mini_table, encoding_options, temporary_arena, size_two) + encoding_two = ::FFI::MemoryPointer.new(:pointer, 1) + encoding_status = Google::Protobuf::FFI.encode_message(other.instance_variable_get(:@msg), mini_table, encoding_options, temporary_arena, encoding_two.to_ptr, size_two) + raise ParseError.new "Error comparing messages due to #{encoding_status} while encoding RHS of `eql?()`" unless encoding_status == :Ok + if encoding_one.null? or encoding_two.null? raise ParseError.new "Error comparing messages" end - size_one.read(:size_t) == size_two.read(:size_t) and Google::Protobuf::FFI.memcmp(encoding_one, encoding_two, size_one.read(:size_t)).zero? + size_one.read(:size_t) == size_two.read(:size_t) and Google::Protobuf::FFI.memcmp(encoding_one.read(:pointer), encoding_two.read(:pointer), size_one.read(:size_t)).zero? end alias == eql? @@ -231,11 +237,12 @@ def hash temporary_arena = Google::Protobuf::FFI.create_arena mini_table_ptr = Google::Protobuf::FFI.get_mini_table(self.class.descriptor) size_ptr = ::FFI::MemoryPointer.new(:size_t, 1) - encoding = Google::Protobuf::FFI.encode_message(@msg, mini_table_ptr, encoding_options, temporary_arena, size_ptr) - if encoding.null? + encoding = ::FFI::MemoryPointer.new(:pointer, 1) + encoding_status = Google::Protobuf::FFI.encode_message(@msg, mini_table_ptr, encoding_options, temporary_arena, encoding.to_ptr, size_ptr) + if encoding_status != :Ok or encoding.null? raise ParseError.new "Error calculating hash" end - Google::Protobuf::FFI.hash(encoding, size_ptr.read(:size_t), 0) + Google::Protobuf::FFI.hash(encoding.read(:pointer), size_ptr.read(:size_t), 0) end def to_h @@ -388,6 +395,9 @@ def self.encode_json(message, options = {}) if options[:emit_defaults] encoding_options |= Google::Protobuf::FFI::Upb_JsonEncode_EmitDefaults end + if options[:format_enums_as_integers] + encoding_options |= Google::Protobuf::FFI::Upb_JsonEncode_FormatEnumsAsIntegers + end buffer_size = 1024 buffer = ::FFI::MemoryPointer.new(:char, buffer_size) @@ -485,11 +495,8 @@ def self.encode_json(message, options = {}) end end if field_descriptor.has_presence? - if field_descriptor.sub_message? or field_descriptor.send(:real_containing_oneof).nil? or - Google::Protobuf::FFI.message_def_syntax(Google::Protobuf::FFI.get_containing_message_def(field_descriptor)) == :Proto2 - define_method("has_#{field_name}?") do - Google::Protobuf::FFI.get_message_has(@msg, field_descriptor) - end + define_method("has_#{field_name}?") do + Google::Protobuf::FFI.get_message_has(@msg, field_descriptor) end end end diff --git a/ruby/lib/google/protobuf/ffi/enum_descriptor.rb b/ruby/lib/google/protobuf/ffi/enum_descriptor.rb index a36f2fe07ef6..a3ceec2dc398 100644 --- a/ruby/lib/google/protobuf/ffi/enum_descriptor.rb +++ b/ruby/lib/google/protobuf/ffi/enum_descriptor.rb @@ -178,12 +178,16 @@ def definition end self.each do |name, value| if name[0] < 'A' || name[0] > 'Z' - warn( + if name[0] >= 'a' and name[0] <= 'z' + name = name[0].upcase + name[1..] # auto capitalize + else + warn( "Enum value '#{name}' does not start with an uppercase letter " + "as is required for Ruby constants.") - else - dynamic_module.const_set(name.to_sym, value) + next + end end + dynamic_module.const_set(name.to_sym, value) end dynamic_module end diff --git a/ruby/lib/google/protobuf/ffi/ffi.rb b/ruby/lib/google/protobuf/ffi/ffi.rb index 0ebcb791797a..787dd042bbb6 100644 --- a/ruby/lib/google/protobuf/ffi/ffi.rb +++ b/ruby/lib/google/protobuf/ffi/ffi.rb @@ -234,6 +234,8 @@ class MiniTable < ::FFI::Struct Upb_JsonEncode_EmitDefaults = 1 # When set, use normal (snake_case) field names instead of JSON (camelCase) names. Upb_JsonEncode_UseProtoNames = 2 + # When set, emits enums as their integer values instead of as their names. + Upb_JsonEncode_FormatEnumsAsIntegers = 4 ## JSON Decoding options Upb_JsonDecode_IgnoreUnknown = 1 @@ -308,7 +310,7 @@ def initialize # Map attach_function :map_clear, :upb_Map_Clear, [:Map], :void - attach_function :map_delete, :upb_Map_Delete, [:Map, MessageValue.by_value], :bool + attach_function :map_delete, :upb_Map_Delete, [:Map, MessageValue.by_value, MessageValue.by_ref], :bool attach_function :map_get, :upb_Map_Get, [:Map, MessageValue.by_value, MessageValue.by_ref], :bool attach_function :create_map, :upb_Map_New, [Arena, CType, CType], :Map attach_function :map_size, :upb_Map_Size, [:Map], :size_t @@ -337,7 +339,7 @@ def initialize attach_function :find_msg_def_by_name, :upb_MessageDef_FindByNameWithSize, [MessageDef, :string, :size_t, :FieldDefPointer, :OneofDefPointer], :bool # Message - attach_function :clear_message_field, :upb_Message_ClearField, [:Message, FieldDescriptor], :void + attach_function :clear_message_field, :upb_Message_ClearFieldByDef, [:Message, FieldDescriptor], :void attach_function :get_message_value, :upb_Message_GetFieldByDef, [:Message, FieldDescriptor], MessageValue.by_value attach_function :get_message_has, :upb_Message_HasFieldByDef, [:Message, FieldDescriptor], :bool attach_function :set_message_field, :upb_Message_SetFieldByDef, [:Message, FieldDescriptor, MessageValue.by_value, Arena], :bool diff --git a/ruby/lib/google/protobuf/ffi/map.rb b/ruby/lib/google/protobuf/ffi/map.rb index 2a581075f59b..e4f8ee2d6a5b 100644 --- a/ruby/lib/google/protobuf/ffi/map.rb +++ b/ruby/lib/google/protobuf/ffi/map.rb @@ -145,11 +145,11 @@ def delete(key) raise FrozenError.new "can't modify frozen #{self.class}" if frozen? value = Google::Protobuf::FFI::MessageValue.new key_message_value = convert_ruby_to_upb(key, arena, key_type, nil) - return_value = if Google::Protobuf::FFI.map_get(@map_ptr, key_message_value, value) + if Google::Protobuf::FFI.map_delete(@map_ptr, key_message_value, value) convert_upb_to_ruby(value, value_type, descriptor, arena) + else + nil end - Google::Protobuf::FFI.map_delete(@map_ptr, key_message_value) - return_value end def clear From 8f95755f766b85b0e13c44647b01ba53d657ba1b Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Mon, 31 Oct 2022 14:31:39 -0400 Subject: [PATCH 29/54] Rebase --- ruby/lib/google/protobuf.rb | 81 --------------------------------- ruby/lib/google/protobuf_ffi.rb | 76 +++++++++++++++++-------------- 2 files changed, 43 insertions(+), 114 deletions(-) delete mode 100644 ruby/lib/google/protobuf.rb diff --git a/ruby/lib/google/protobuf.rb b/ruby/lib/google/protobuf.rb deleted file mode 100644 index 855ab605e95b..000000000000 --- a/ruby/lib/google/protobuf.rb +++ /dev/null @@ -1,81 +0,0 @@ -# Protocol Buffers - Google's data interchange format -# Copyright 2008 Google Inc. All rights reserved. -# https://developers.google.com/protocol-buffers/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# require mixins before we hook them into the java & c code -require 'google/protobuf/message_exts' - -# We define these before requiring the platform-specific modules. -# That way the module init can grab references to these. -module Google - module Protobuf - class Error < StandardError; end - class ParseError < Error; end - class TypeError < ::TypeError; end - - begin - require 'ffi' - PREFER_FFI = true - rescue LoadError - PREFER_FFI = false - end - - def self.encode(msg, options = {}) - msg.to_proto(options) - end - - def self.encode_json(msg, options = {}) - msg.to_json(options) - end - - def self.decode(klass, proto, options = {}) - klass.decode(proto, options) - end - - def self.decode_json(klass, json, options = {}) - klass.decode_json(json, options) - end - - IMPLEMENTATION = if PREFER_FFI - begin - require 'google/protobuf_ffi' - :FFI - rescue LoadError - warn "Caught exception `#{$!.message}` while loading FFI implementation of google/protobuf." - raise $! - warn "Falling back to native implementation." - require 'google/protobuf_native' - :NATIVE - end - else - require 'google/protobuf_native' - :NATIVE - end - end -end diff --git a/ruby/lib/google/protobuf_ffi.rb b/ruby/lib/google/protobuf_ffi.rb index 8652410247ff..855ab605e95b 100644 --- a/ruby/lib/google/protobuf_ffi.rb +++ b/ruby/lib/google/protobuf_ffi.rb @@ -28,44 +28,54 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -require 'ffi' -require 'google/protobuf/ffi/internal/type_safety' -require 'google/protobuf/ffi/internal/arena' -require 'google/protobuf/ffi/internal/convert' -require 'google/protobuf/ffi/descriptor' -require 'google/protobuf/ffi/enum_descriptor' -require 'google/protobuf/ffi/field_descriptor' -require 'google/protobuf/ffi/oneof_descriptor' -require 'google/protobuf/ffi/ffi' -require 'google/protobuf/ffi/descriptor_pool' -require 'google/protobuf/ffi/file_descriptor' -require 'google/protobuf/ffi/map' -require 'google/protobuf/ffi/object_cache' -require 'google/protobuf/ffi/repeated_field' -require 'google/protobuf/descriptor_dsl' +# require mixins before we hook them into the java & c code +require 'google/protobuf/message_exts' +# We define these before requiring the platform-specific modules. +# That way the module init can grab references to these. module Google module Protobuf - def self.deep_copy(object) - case object - when RepeatedField - RepeatedField.send(:deep_copy, object) - when Google::Protobuf::Map - Google::Protobuf::Map.deep_copy(object) - when Google::Protobuf::MessageExts - object.class.send(:deep_copy, object.instance_variable_get(:@msg)) - else - raise NotImplementedError - end + class Error < StandardError; end + class ParseError < Error; end + class TypeError < ::TypeError; end + + begin + require 'ffi' + PREFER_FFI = true + rescue LoadError + PREFER_FFI = false + end + + def self.encode(msg, options = {}) + msg.to_proto(options) + end + + def self.encode_json(msg, options = {}) + msg.to_json(options) + end + + def self.decode(klass, proto, options = {}) + klass.decode(proto, options) + end + + def self.decode_json(klass, json, options = {}) + klass.decode_json(json, options) end - def self.discard_unknown(message) - raise FrozenError if message.frozen? - raise ArgumentError.new "Expected message, got #{message.class} instead." if message.instance_variable_get(:@msg).nil? - unless Google::Protobuf::FFI.message_discard_unknown(message.instance_variable_get(:@msg), message.class.descriptor, 128) - raise RuntimeError.new "Messages nested too deeply." + IMPLEMENTATION = if PREFER_FFI + begin + require 'google/protobuf_ffi' + :FFI + rescue LoadError + warn "Caught exception `#{$!.message}` while loading FFI implementation of google/protobuf." + raise $! + warn "Falling back to native implementation." + require 'google/protobuf_native' + :NATIVE end - nil + else + require 'google/protobuf_native' + :NATIVE end end -end \ No newline at end of file +end From 1dff2099a76a69c472480cf0a1a20a7e1f0f7102 Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Mon, 31 Oct 2022 14:32:37 -0400 Subject: [PATCH 30/54] restore protobuf.rb --- ruby/lib/google/protobuf.rb | 79 +++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 ruby/lib/google/protobuf.rb diff --git a/ruby/lib/google/protobuf.rb b/ruby/lib/google/protobuf.rb new file mode 100644 index 000000000000..b7a671105158 --- /dev/null +++ b/ruby/lib/google/protobuf.rb @@ -0,0 +1,79 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# require mixins before we hook them into the java & c code +require 'google/protobuf/message_exts' + +# We define these before requiring the platform-specific modules. +# That way the module init can grab references to these. +module Google + module Protobuf + class Error < StandardError; end + class ParseError < Error; end + class TypeError < ::TypeError; end + end +end + +if RUBY_PLATFORM == "java" + require 'json' + require 'google/protobuf_java' +else + begin + require "google/#{RUBY_VERSION.sub(/\.\d+$/, '')}/protobuf_c" + rescue LoadError + require 'google/protobuf_c' + end + +end + +require 'google/protobuf/descriptor_dsl' +require 'google/protobuf/repeated_field' + +module Google + module Protobuf + + def self.encode(msg, options = {}) + msg.to_proto(options) + end + + def self.encode_json(msg, options = {}) + msg.to_json(options) + end + + def self.decode(klass, proto, options = {}) + klass.decode(proto, options) + end + + def self.decode_json(klass, json, options = {}) + klass.decode_json(json, options) + end + + end +end From f4c08b21477e69edd627510399e7bbad864a8b22 Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Mon, 31 Oct 2022 14:40:08 -0400 Subject: [PATCH 31/54] Rebase --- ruby/lib/google/protobuf.rb | 79 ------------------------------ ruby/lib/google/protobuf_native.rb | 38 +++++++++++++- 2 files changed, 37 insertions(+), 80 deletions(-) delete mode 100644 ruby/lib/google/protobuf.rb diff --git a/ruby/lib/google/protobuf.rb b/ruby/lib/google/protobuf.rb deleted file mode 100644 index b7a671105158..000000000000 --- a/ruby/lib/google/protobuf.rb +++ /dev/null @@ -1,79 +0,0 @@ -# Protocol Buffers - Google's data interchange format -# Copyright 2008 Google Inc. All rights reserved. -# https://developers.google.com/protocol-buffers/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# require mixins before we hook them into the java & c code -require 'google/protobuf/message_exts' - -# We define these before requiring the platform-specific modules. -# That way the module init can grab references to these. -module Google - module Protobuf - class Error < StandardError; end - class ParseError < Error; end - class TypeError < ::TypeError; end - end -end - -if RUBY_PLATFORM == "java" - require 'json' - require 'google/protobuf_java' -else - begin - require "google/#{RUBY_VERSION.sub(/\.\d+$/, '')}/protobuf_c" - rescue LoadError - require 'google/protobuf_c' - end - -end - -require 'google/protobuf/descriptor_dsl' -require 'google/protobuf/repeated_field' - -module Google - module Protobuf - - def self.encode(msg, options = {}) - msg.to_proto(options) - end - - def self.encode_json(msg, options = {}) - msg.to_json(options) - end - - def self.decode(klass, proto, options = {}) - klass.decode(proto, options) - end - - def self.decode_json(klass, json, options = {}) - klass.decode_json(json, options) - end - - end -end diff --git a/ruby/lib/google/protobuf_native.rb b/ruby/lib/google/protobuf_native.rb index f55d6c48ae6e..dd383b9d5586 100644 --- a/ruby/lib/google/protobuf_native.rb +++ b/ruby/lib/google/protobuf_native.rb @@ -1,5 +1,5 @@ # Protocol Buffers - Google's data interchange format -# Copyright 2021 Google Inc. All rights reserved. +# Copyright 2023 Google Inc. All rights reserved. # https://developers.google.com/protocol-buffers/ # # Redistribution and use in source and binary forms, with or without @@ -28,6 +28,19 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# require mixins before we hook them into the java & c code +require 'google/protobuf/message_exts' + +# We define these before requiring the platform-specific modules. +# That way the module init can grab references to these. +module Google + module Protobuf + class Error < StandardError; end + class ParseError < Error; end + class TypeError < ::TypeError; end + end +end + if RUBY_PLATFORM == "java" require 'json' require 'google/protobuf_java' @@ -37,7 +50,30 @@ rescue LoadError require 'google/protobuf_c' end + end require 'google/protobuf/descriptor_dsl' require 'google/protobuf/repeated_field' + +module Google + module Protobuf + + def self.encode(msg, options = {}) + msg.to_proto(options) + end + + def self.encode_json(msg, options = {}) + msg.to_json(options) + end + + def self.decode(klass, proto, options = {}) + klass.decode(proto, options) + end + + def self.decode_json(klass, json, options = {}) + klass.decode_json(json, options) + end + + end +end From e32280e8efca3bfcc17ad257fe52e1dc770829dc Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Mon, 31 Oct 2022 14:40:40 -0400 Subject: [PATCH 32/54] restore protobuf.rb --- ruby/lib/google/protobuf.rb | 79 +++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 ruby/lib/google/protobuf.rb diff --git a/ruby/lib/google/protobuf.rb b/ruby/lib/google/protobuf.rb new file mode 100644 index 000000000000..b7a671105158 --- /dev/null +++ b/ruby/lib/google/protobuf.rb @@ -0,0 +1,79 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# require mixins before we hook them into the java & c code +require 'google/protobuf/message_exts' + +# We define these before requiring the platform-specific modules. +# That way the module init can grab references to these. +module Google + module Protobuf + class Error < StandardError; end + class ParseError < Error; end + class TypeError < ::TypeError; end + end +end + +if RUBY_PLATFORM == "java" + require 'json' + require 'google/protobuf_java' +else + begin + require "google/#{RUBY_VERSION.sub(/\.\d+$/, '')}/protobuf_c" + rescue LoadError + require 'google/protobuf_c' + end + +end + +require 'google/protobuf/descriptor_dsl' +require 'google/protobuf/repeated_field' + +module Google + module Protobuf + + def self.encode(msg, options = {}) + msg.to_proto(options) + end + + def self.encode_json(msg, options = {}) + msg.to_json(options) + end + + def self.decode(klass, proto, options = {}) + klass.decode(proto, options) + end + + def self.decode_json(klass, json, options = {}) + klass.decode_json(json, options) + end + + end +end From 536f9b3681728e2b01a9390932570b747b67e851 Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Mon, 31 Oct 2022 15:10:55 -0400 Subject: [PATCH 33/54] Rebase --- ruby/lib/google/protobuf.rb | 30 ++++-------- ruby/lib/google/protobuf_ffi.rb | 78 +++++++++++++----------------- ruby/lib/google/protobuf_native.rb | 36 -------------- 3 files changed, 44 insertions(+), 100 deletions(-) diff --git a/ruby/lib/google/protobuf.rb b/ruby/lib/google/protobuf.rb index b7a671105158..5dfd91f5dd44 100644 --- a/ruby/lib/google/protobuf.rb +++ b/ruby/lib/google/protobuf.rb @@ -38,26 +38,6 @@ module Protobuf class Error < StandardError; end class ParseError < Error; end class TypeError < ::TypeError; end - end -end - -if RUBY_PLATFORM == "java" - require 'json' - require 'google/protobuf_java' -else - begin - require "google/#{RUBY_VERSION.sub(/\.\d+$/, '')}/protobuf_c" - rescue LoadError - require 'google/protobuf_c' - end - -end - -require 'google/protobuf/descriptor_dsl' -require 'google/protobuf/repeated_field' - -module Google - module Protobuf def self.encode(msg, options = {}) msg.to_proto(options) @@ -75,5 +55,15 @@ def self.decode_json(klass, json, options = {}) klass.decode_json(json, options) end + IMPLEMENTATION = case ENV['PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION'] + when nil, 'native', '' + require 'google/protobuf_native' + :NATIVE + when 'ffi' + require 'google/protobuf_native' + :FFI + else + raise RuntimeError("Unsupport value of PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION environment variable `#{ENV['PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION']}`. Must be one of `native` or `ffi`. Defaults to `native`.") + end end end diff --git a/ruby/lib/google/protobuf_ffi.rb b/ruby/lib/google/protobuf_ffi.rb index 855ab605e95b..068d4046bef2 100644 --- a/ruby/lib/google/protobuf_ffi.rb +++ b/ruby/lib/google/protobuf_ffi.rb @@ -1,5 +1,5 @@ # Protocol Buffers - Google's data interchange format -# Copyright 2008 Google Inc. All rights reserved. +# Copyright 2023 Google Inc. All rights reserved. # https://developers.google.com/protocol-buffers/ # # Redistribution and use in source and binary forms, with or without @@ -28,54 +28,44 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# require mixins before we hook them into the java & c code -require 'google/protobuf/message_exts' +require 'ffi' +require 'google/protobuf/ffi/internal/type_safety' +require 'google/protobuf/ffi/internal/arena' +require 'google/protobuf/ffi/internal/convert' +require 'google/protobuf/ffi/descriptor' +require 'google/protobuf/ffi/enum_descriptor' +require 'google/protobuf/ffi/field_descriptor' +require 'google/protobuf/ffi/oneof_descriptor' +require 'google/protobuf/ffi/ffi' +require 'google/protobuf/ffi/descriptor_pool' +require 'google/protobuf/ffi/file_descriptor' +require 'google/protobuf/ffi/map' +require 'google/protobuf/ffi/object_cache' +require 'google/protobuf/ffi/repeated_field' +require 'google/protobuf/descriptor_dsl' -# We define these before requiring the platform-specific modules. -# That way the module init can grab references to these. module Google module Protobuf - class Error < StandardError; end - class ParseError < Error; end - class TypeError < ::TypeError; end - - begin - require 'ffi' - PREFER_FFI = true - rescue LoadError - PREFER_FFI = false - end - - def self.encode(msg, options = {}) - msg.to_proto(options) - end - - def self.encode_json(msg, options = {}) - msg.to_json(options) - end - - def self.decode(klass, proto, options = {}) - klass.decode(proto, options) - end - - def self.decode_json(klass, json, options = {}) - klass.decode_json(json, options) + def self.deep_copy(object) + case object + when RepeatedField + RepeatedField.send(:deep_copy, object) + when Google::Protobuf::Map + Google::Protobuf::Map.deep_copy(object) + when Google::Protobuf::MessageExts + object.class.send(:deep_copy, object.instance_variable_get(:@msg)) + else + raise NotImplementedError + end end - IMPLEMENTATION = if PREFER_FFI - begin - require 'google/protobuf_ffi' - :FFI - rescue LoadError - warn "Caught exception `#{$!.message}` while loading FFI implementation of google/protobuf." - raise $! - warn "Falling back to native implementation." - require 'google/protobuf_native' - :NATIVE + def self.discard_unknown(message) + raise FrozenError if message.frozen? + raise ArgumentError.new "Expected message, got #{message.class} instead." if message.instance_variable_get(:@msg).nil? + unless Google::Protobuf::FFI.message_discard_unknown(message.instance_variable_get(:@msg), message.class.descriptor, 128) + raise RuntimeError.new "Messages nested too deeply." end - else - require 'google/protobuf_native' - :NATIVE + nil end end -end +end \ No newline at end of file diff --git a/ruby/lib/google/protobuf_native.rb b/ruby/lib/google/protobuf_native.rb index dd383b9d5586..009983434e86 100644 --- a/ruby/lib/google/protobuf_native.rb +++ b/ruby/lib/google/protobuf_native.rb @@ -28,19 +28,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# require mixins before we hook them into the java & c code -require 'google/protobuf/message_exts' - -# We define these before requiring the platform-specific modules. -# That way the module init can grab references to these. -module Google - module Protobuf - class Error < StandardError; end - class ParseError < Error; end - class TypeError < ::TypeError; end - end -end - if RUBY_PLATFORM == "java" require 'json' require 'google/protobuf_java' @@ -50,30 +37,7 @@ class TypeError < ::TypeError; end rescue LoadError require 'google/protobuf_c' end - end require 'google/protobuf/descriptor_dsl' require 'google/protobuf/repeated_field' - -module Google - module Protobuf - - def self.encode(msg, options = {}) - msg.to_proto(options) - end - - def self.encode_json(msg, options = {}) - msg.to_json(options) - end - - def self.decode(klass, proto, options = {}) - klass.decode(proto, options) - end - - def self.decode_json(klass, json, options = {}) - klass.decode_json(json, options) - end - - end -end From c665a79d683613808b539149dee6420df4338cfe Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Fri, 6 Jan 2023 14:48:06 -0500 Subject: [PATCH 34/54] Fix spelling `Unsupport` -> `Unsupported` --- ruby/lib/google/protobuf.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ruby/lib/google/protobuf.rb b/ruby/lib/google/protobuf.rb index 5dfd91f5dd44..29b367352f19 100644 --- a/ruby/lib/google/protobuf.rb +++ b/ruby/lib/google/protobuf.rb @@ -60,10 +60,10 @@ def self.decode_json(klass, json, options = {}) require 'google/protobuf_native' :NATIVE when 'ffi' - require 'google/protobuf_native' + require 'google/protobuf_ffi' :FFI else - raise RuntimeError("Unsupport value of PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION environment variable `#{ENV['PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION']}`. Must be one of `native` or `ffi`. Defaults to `native`.") + raise RuntimeError("Unsupported value of PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION environment variable `#{ENV['PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION']}`. Must be one of `native` or `ffi`. Defaults to `native`.") end end end From 78a292898669f4fc968a6bbab6b6590a8d8a2a62 Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Fri, 6 Jan 2023 17:41:01 -0500 Subject: [PATCH 35/54] Fix exception raising by properly instantiating the exception. --- ruby/lib/google/protobuf.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ruby/lib/google/protobuf.rb b/ruby/lib/google/protobuf.rb index 29b367352f19..1dd898e3b44e 100644 --- a/ruby/lib/google/protobuf.rb +++ b/ruby/lib/google/protobuf.rb @@ -63,7 +63,7 @@ def self.decode_json(klass, json, options = {}) require 'google/protobuf_ffi' :FFI else - raise RuntimeError("Unsupported value of PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION environment variable `#{ENV['PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION']}`. Must be one of `native` or `ffi`. Defaults to `native`.") + raise RuntimeError.new("Unsupported value of PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION environment variable `#{ENV['PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION']}`. Must be one of `native` or `ffi`. Defaults to `native`.") end end end From a7aa7aac2232845697548162fa7d2a8d56632710 Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Fri, 6 Jan 2023 17:42:35 -0500 Subject: [PATCH 36/54] Rebase --- ruby/lib/google/protobuf/ffi/descriptor.rb | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/ruby/lib/google/protobuf/ffi/descriptor.rb b/ruby/lib/google/protobuf/ffi/descriptor.rb index 916453c66779..0fb18e6b748a 100644 --- a/ruby/lib/google/protobuf/ffi/descriptor.rb +++ b/ruby/lib/google/protobuf/ffi/descriptor.rb @@ -216,19 +216,13 @@ def eql?(other) temporary_arena = Google::Protobuf::FFI.create_arena mini_table = Google::Protobuf::FFI.get_mini_table(self.class.descriptor) size_one = ::FFI::MemoryPointer.new(:size_t, 1) - encoding_one = ::FFI::MemoryPointer.new(:pointer, 1) - encoding_status = Google::Protobuf::FFI.encode_message(@msg, mini_table, encoding_options, temporary_arena, encoding_one.to_ptr, size_one) - raise ParseError.new "Error comparing messages due to #{encoding_status} while encoding LHS of `eql?()`" unless encoding_status == :Ok - + encoding_one = Google::Protobuf::FFI.encode_message(@msg, mini_table, encoding_options, temporary_arena, size_one) size_two = ::FFI::MemoryPointer.new(:size_t, 1) - encoding_two = ::FFI::MemoryPointer.new(:pointer, 1) - encoding_status = Google::Protobuf::FFI.encode_message(other.instance_variable_get(:@msg), mini_table, encoding_options, temporary_arena, encoding_two.to_ptr, size_two) - raise ParseError.new "Error comparing messages due to #{encoding_status} while encoding RHS of `eql?()`" unless encoding_status == :Ok - + encoding_two = Google::Protobuf::FFI.encode_message(other.instance_variable_get(:@msg), mini_table, encoding_options, temporary_arena, size_two) if encoding_one.null? or encoding_two.null? raise ParseError.new "Error comparing messages" end - size_one.read(:size_t) == size_two.read(:size_t) and Google::Protobuf::FFI.memcmp(encoding_one.read(:pointer), encoding_two.read(:pointer), size_one.read(:size_t)).zero? + size_one.read(:size_t) == size_two.read(:size_t) and Google::Protobuf::FFI.memcmp(encoding_one, encoding_two, size_one.read(:size_t)).zero? end alias == eql? @@ -237,12 +231,11 @@ def hash temporary_arena = Google::Protobuf::FFI.create_arena mini_table_ptr = Google::Protobuf::FFI.get_mini_table(self.class.descriptor) size_ptr = ::FFI::MemoryPointer.new(:size_t, 1) - encoding = ::FFI::MemoryPointer.new(:pointer, 1) - encoding_status = Google::Protobuf::FFI.encode_message(@msg, mini_table_ptr, encoding_options, temporary_arena, encoding.to_ptr, size_ptr) - if encoding_status != :Ok or encoding.null? + encoding = Google::Protobuf::FFI.encode_message(@msg, mini_table_ptr, encoding_options, temporary_arena, size_ptr) + if encoding.null? raise ParseError.new "Error calculating hash" end - Google::Protobuf::FFI.hash(encoding.read(:pointer), size_ptr.read(:size_t), 0) + Google::Protobuf::FFI.hash(encoding, size_ptr.read(:size_t), 0) end def to_h From bcc6f6c2645ff0300aa97db09940dab179a40ff9 Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Wed, 11 Jan 2023 09:54:35 -0500 Subject: [PATCH 37/54] Rebase --- ruby/Rakefile | 62 ++++++++++++++++++ ruby/ext/google/shared/shared_convert.c | 85 +++++++++++++++++++++++++ ruby/ext/google/shared/shared_convert.h | 41 ++++++++++++ 3 files changed, 188 insertions(+) create mode 100644 ruby/ext/google/shared/shared_convert.c create mode 100644 ruby/ext/google/shared/shared_convert.h diff --git a/ruby/Rakefile b/ruby/Rakefile index 5a8fd5e3c1a0..7b0090b8755d 100644 --- a/ruby/Rakefile +++ b/ruby/Rakefile @@ -10,6 +10,10 @@ rescue LoadError USE_FFI = false end +USE_FFI = ENV['PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION'] == 'ffi' + +require "ffi-compiler/compile_task" if USE_FFI + spec = Gem::Specification.load("google-protobuf.gemspec") well_known_protos = %w[ @@ -186,6 +190,61 @@ else unless ENV['IN_DOCKER'] == 'true' task :build => [:copy_third_party] end +end + +task :copy_shared do + FileUtils.cp(Dir.glob("ext/google/shared/*.{c,h}"), "ext/google/protobuf_c") +end + +if USE_FFI + # FFI::CompilerTask's constructor walks the filesystem at initialization time + # in order to create subtasks for each source file, so the files from third_party + # must be copied into place before the task is defined. + ffi_compiler_config_block = Proc.new do |c| + c.cflags << "-std=gnu99 -O3 -DNDEBUG" + if RbConfig::CONFIG['target_os'] =~ /darwin|linux/ + # c.cflags << "-fvisibility=hidden -Wall -Wsign-compare -Wno-declaration-after-statement" + c.cflags << "-Wall -Wsign-compare -Wno-declaration-after-statement" + end + if RbConfig::CONFIG['target_os'] =~ /linux/ + # Instruct the linker to point memcpy calls at our __wrap_memcpy wrapper. + c.ldflags << "-Wl,-wrap,memcpy" + end + end + + if File.exist? 'ext/google/protobuf_c/third_party/utf8_range' + desc "compiler tasks" + namespace "ffi-compiler" do + FFI::Compiler::CompileTask.new('ext/google/protobuf_c', &ffi_compiler_config_block) + end + else + task "ffi-compiler:default" do + Rake::Task[:copy_third_party].invoke + desc "compiler tasks" + namespace "ffi-compiler" do + FFI::Compiler::CompileTask.new('ext/google/protobuf_c', &ffi_compiler_config_block) + Rake::Task[Rake::Task[:default].prereqs.first].invoke + end + end + end +else + if RUBY_PLATFORM == "java" + task :clean => :require_mvn do + system("mvn --batch-mode clean") + end + + task :compile => :require_mvn do + system("mvn --batch-mode package") + end + + task :require_mvn do + raise ArgumentError, "maven needs to be installed" if `which mvn` == '' + end + + else + unless ENV['IN_DOCKER'] == 'true' + task :build => [:copy_third_party, :copy_shared] + end Rake::ExtensionTask.new("protobuf_c", spec) do |ext| unless RUBY_PLATFORM =~ /darwin/ @@ -238,6 +297,9 @@ else task 'gem:native' => [:genproto, 'gem:windows', 'gem:java'] end end +end + +task :compile => ["ffi-compiler:default"] if USE_FFI task :compile => ["ffi-compiler:default"] if USE_FFI diff --git a/ruby/ext/google/shared/shared_convert.c b/ruby/ext/google/shared/shared_convert.c new file mode 100644 index 000000000000..a19c1dfe5629 --- /dev/null +++ b/ruby/ext/google/shared/shared_convert.c @@ -0,0 +1,85 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// ----------------------------------------------------------------------------- +// Ruby <-> upb data conversion functions. Strictly free of dependencies on +// Ruby interpreter internals. + +#include "shared_convert.h" + +bool shared_Msgval_IsEqual(upb_MessageValue val1, upb_MessageValue val2, + upb_CType type, upb_MessageDef* msgdef, upb_Status* status) { + switch (type) { + case kUpb_CType_Bool: + return memcmp(&val1, &val2, 1) == 0; + case kUpb_CType_Float: + case kUpb_CType_Int32: + case kUpb_CType_UInt32: + case kUpb_CType_Enum: + return memcmp(&val1, &val2, 4) == 0; + case kUpb_CType_Double: + case kUpb_CType_Int64: + case kUpb_CType_UInt64: + return memcmp(&val1, &val2, 8) == 0; + case kUpb_CType_String: + case kUpb_CType_Bytes: + return val1.str_val.size == val2.str_val.size && + memcmp(val1.str_val.data, val2.str_val.data, val1.str_val.size) == + 0; + case kUpb_CType_Message: + return Message_Equal(val1.msg_val, val2.msg_val, msgdef); + default: + upb_Status_SetErrorMessage(status, "Internal error, unexpected type"); + } +} + +uint64_t shared_Msgval_GetHash(upb_MessageValue val, upb_CType type, upb_MessageDef* msgdef, + uint64_t seed, upb_Status* status) { + switch (type) { + case kUpb_CType_Bool: + return _upb_Hash(&val, 1, seed); + case kUpb_CType_Float: + case kUpb_CType_Int32: + case kUpb_CType_UInt32: + case kUpb_CType_Enum: + return _upb_Hash(&val, 4, seed); + case kUpb_CType_Double: + case kUpb_CType_Int64: + case kUpb_CType_UInt64: + return _upb_Hash(&val, 8, seed); + case kUpb_CType_String: + case kUpb_CType_Bytes: + return _upb_Hash(val.str_val.data, val.str_val.size, seed); + case kUpb_CType_Message: + return Message_Hash(val.msg_val, msgdef, seed); + default: + upb_Status_SetErrorMessage(status, "Internal error, unexpected type"); + } +} \ No newline at end of file diff --git a/ruby/ext/google/shared/shared_convert.h b/ruby/ext/google/shared/shared_convert.h new file mode 100644 index 000000000000..53f5ccddf013 --- /dev/null +++ b/ruby/ext/google/shared/shared_convert.h @@ -0,0 +1,41 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// ----------------------------------------------------------------------------- +// Ruby <-> upb data conversion functions. Strictly free of dependencies on +// Ruby interpreter internals. + +#include "ruby-upb.h" + +bool shared_Msgval_IsEqual(upb_MessageValue val1, upb_MessageValue val2, + upb_CType type, upb_MessageDef* msgdef, upb_Status* status); + +uint64_t shared_Msgval_GetHash(upb_MessageValue val, upb_CType type, upb_MessageDef* msgdef, + uint64_t seed, upb_Status* status); \ No newline at end of file From 8336d0aedbc189088c3edbefc439f7ba49b8512f Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Wed, 11 Jan 2023 12:59:15 -0500 Subject: [PATCH 38/54] Refactor message.c to separate CRuby-specific glue that has dependencies on Ruby headers from functionality needed by FFI. --- ruby/ext/google/shared/shared_message.c | 87 +++++++++++++++++++++++++ ruby/ext/google/shared/shared_message.h | 48 ++++++++++++++ 2 files changed, 135 insertions(+) create mode 100644 ruby/ext/google/shared/shared_message.c create mode 100644 ruby/ext/google/shared/shared_message.h diff --git a/ruby/ext/google/shared/shared_message.c b/ruby/ext/google/shared/shared_message.c new file mode 100644 index 000000000000..da9fd9636992 --- /dev/null +++ b/ruby/ext/google/shared/shared_message.c @@ -0,0 +1,87 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// ----------------------------------------------------------------------------- +// Ruby Message functions. Strictly free of dependencies on +// Ruby interpreter internals. + +#include "shared_message.h" + +// Support function for Message_Hash. Returns a hash value for the given message. +uint64_t shared_Message_Hash(const upb_Message* msg, const upb_MessageDef* m, + uint64_t seed, upb_Status* status) { + upb_Arena* arena = upb_Arena_New(); + char* data; + size_t size; + + // Hash a deterministically serialized payloads with no unknown fields. + upb_EncodeStatus encode_status = upb_Encode( + msg, upb_MessageDef_MiniTable(m), + kUpb_EncodeOption_SkipUnknown | kUpb_EncodeOption_Deterministic, arena, + &data, &size); + + if (encode_status == kUpb_EncodeStatus_Ok) { + uint64_t ret = _upb_Hash(data, size, seed); + upb_Arena_Free(arena); + return ret; + } else { + upb_Arena_Free(arena); + upb_Status_SetErrorMessage(status, "Error calculating hash"); + } +} + +// Support function for Message_Equal +bool shared_Message_Equal(const upb_Message* m1, const upb_Message* m2, + const upb_MessageDef* m, upb_Status* status) { + if (m1 == m2) return true; + + size_t size1, size2; + int encode_opts = + kUpb_EncodeOption_SkipUnknown | kUpb_EncodeOption_Deterministic; + upb_Arena* arena_tmp = upb_Arena_New(); + const upb_MiniTable* layout = upb_MessageDef_MiniTable(m); + + // Compare deterministically serialized payloads with no unknown fields. + char* data1; + char* data2; + upb_EncodeStatus status1 = + upb_Encode(m1, layout, encode_opts, arena_tmp, &data1, &size1); + upb_EncodeStatus status2 = + upb_Encode(m2, layout, encode_opts, arena_tmp, &data2, &size2); + + if (status1 == kUpb_EncodeStatus_Ok && status2 == kUpb_EncodeStatus_Ok) { + bool ret = (size1 == size2) && (memcmp(data1, data2, size1) == 0); + upb_Arena_Free(arena_tmp); + return ret; + } else { + upb_Arena_Free(arena_tmp); + upb_Status_SetErrorMessage(status, "Error comparing messages"); + } +} diff --git a/ruby/ext/google/shared/shared_message.h b/ruby/ext/google/shared/shared_message.h new file mode 100644 index 000000000000..0e46d2e6a816 --- /dev/null +++ b/ruby/ext/google/shared/shared_message.h @@ -0,0 +1,48 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// ----------------------------------------------------------------------------- +// Ruby Message functions. Strictly free of dependencies on +// Ruby interpreter internals. + +#ifndef RUBY_PROTOBUF_SHARED_MESSAGE_H_ +#define RUBY_PROTOBUF_SHARED_MESSAGE_H_ + +#include "ruby-upb.h" + +// Returns a hash value for the given message. +uint64_t shared_Message_Hash(const upb_Message* msg, const upb_MessageDef* m, + uint64_t seed, upb_Status* status); + +// Returns true if these two messages are equal. +bool shared_Message_Equal(const upb_Message* m1, const upb_Message* m2, + const upb_MessageDef* m, upb_Status* status); + +#endif // RUBY_PROTOBUF_SHARED_MESSAGE_H_ From 88f150683c3aacb125584bcf36852deca61c3fe8 Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Wed, 11 Jan 2023 13:01:44 -0500 Subject: [PATCH 39/54] Have shared_convert methods invoke shared_message methods directly. Add `ifndef` armor around shared_convert.h. --- ruby/ext/google/shared/shared_convert.c | 4 ++-- ruby/ext/google/shared/shared_convert.h | 8 +++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/ruby/ext/google/shared/shared_convert.c b/ruby/ext/google/shared/shared_convert.c index a19c1dfe5629..57a8be7501ce 100644 --- a/ruby/ext/google/shared/shared_convert.c +++ b/ruby/ext/google/shared/shared_convert.c @@ -54,7 +54,7 @@ bool shared_Msgval_IsEqual(upb_MessageValue val1, upb_MessageValue val2, memcmp(val1.str_val.data, val2.str_val.data, val1.str_val.size) == 0; case kUpb_CType_Message: - return Message_Equal(val1.msg_val, val2.msg_val, msgdef); + return shared_Message_Equal(val1.msg_val, val2.msg_val, msgdef, status); default: upb_Status_SetErrorMessage(status, "Internal error, unexpected type"); } @@ -78,7 +78,7 @@ uint64_t shared_Msgval_GetHash(upb_MessageValue val, upb_CType type, upb_Message case kUpb_CType_Bytes: return _upb_Hash(val.str_val.data, val.str_val.size, seed); case kUpb_CType_Message: - return Message_Hash(val.msg_val, msgdef, seed); + return shared_Message_Hash(val.msg_val, msgdef, seed, status); default: upb_Status_SetErrorMessage(status, "Internal error, unexpected type"); } diff --git a/ruby/ext/google/shared/shared_convert.h b/ruby/ext/google/shared/shared_convert.h index 53f5ccddf013..8659cca772a4 100644 --- a/ruby/ext/google/shared/shared_convert.h +++ b/ruby/ext/google/shared/shared_convert.h @@ -32,10 +32,16 @@ // Ruby <-> upb data conversion functions. Strictly free of dependencies on // Ruby interpreter internals. +#ifndef RUBY_PROTOBUF_SHARED_CONVERT_H_ +#define RUBY_PROTOBUF_SHARED_CONVERT_H_ + #include "ruby-upb.h" +#include "shared_message.h" bool shared_Msgval_IsEqual(upb_MessageValue val1, upb_MessageValue val2, upb_CType type, upb_MessageDef* msgdef, upb_Status* status); uint64_t shared_Msgval_GetHash(upb_MessageValue val, upb_CType type, upb_MessageDef* msgdef, - uint64_t seed, upb_Status* status); \ No newline at end of file + uint64_t seed, upb_Status* status); + +#endif // RUBY_PROTOBUF_SHARED_CONVERT_H_ From 30898183572a061daa1d93147756960a95519a11 Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Fri, 7 Jul 2023 11:18:11 -0400 Subject: [PATCH 40/54] Fix merge damage --- ruby/Rakefile | 62 --------------- ruby/ext/google/shared/shared_convert.c | 85 --------------------- ruby/ext/google/shared/shared_convert.h | 47 ------------ ruby/ext/google/shared/shared_message.c | 87 ---------------------- ruby/ext/google/shared/shared_message.h | 48 ------------ ruby/lib/google/protobuf.rb | 28 +++++-- ruby/lib/google/protobuf/ffi/descriptor.rb | 19 +++-- 7 files changed, 33 insertions(+), 343 deletions(-) delete mode 100644 ruby/ext/google/shared/shared_convert.c delete mode 100644 ruby/ext/google/shared/shared_convert.h delete mode 100644 ruby/ext/google/shared/shared_message.c delete mode 100644 ruby/ext/google/shared/shared_message.h diff --git a/ruby/Rakefile b/ruby/Rakefile index 7b0090b8755d..5a8fd5e3c1a0 100644 --- a/ruby/Rakefile +++ b/ruby/Rakefile @@ -10,10 +10,6 @@ rescue LoadError USE_FFI = false end -USE_FFI = ENV['PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION'] == 'ffi' - -require "ffi-compiler/compile_task" if USE_FFI - spec = Gem::Specification.load("google-protobuf.gemspec") well_known_protos = %w[ @@ -190,61 +186,6 @@ else unless ENV['IN_DOCKER'] == 'true' task :build => [:copy_third_party] end -end - -task :copy_shared do - FileUtils.cp(Dir.glob("ext/google/shared/*.{c,h}"), "ext/google/protobuf_c") -end - -if USE_FFI - # FFI::CompilerTask's constructor walks the filesystem at initialization time - # in order to create subtasks for each source file, so the files from third_party - # must be copied into place before the task is defined. - ffi_compiler_config_block = Proc.new do |c| - c.cflags << "-std=gnu99 -O3 -DNDEBUG" - if RbConfig::CONFIG['target_os'] =~ /darwin|linux/ - # c.cflags << "-fvisibility=hidden -Wall -Wsign-compare -Wno-declaration-after-statement" - c.cflags << "-Wall -Wsign-compare -Wno-declaration-after-statement" - end - if RbConfig::CONFIG['target_os'] =~ /linux/ - # Instruct the linker to point memcpy calls at our __wrap_memcpy wrapper. - c.ldflags << "-Wl,-wrap,memcpy" - end - end - - if File.exist? 'ext/google/protobuf_c/third_party/utf8_range' - desc "compiler tasks" - namespace "ffi-compiler" do - FFI::Compiler::CompileTask.new('ext/google/protobuf_c', &ffi_compiler_config_block) - end - else - task "ffi-compiler:default" do - Rake::Task[:copy_third_party].invoke - desc "compiler tasks" - namespace "ffi-compiler" do - FFI::Compiler::CompileTask.new('ext/google/protobuf_c', &ffi_compiler_config_block) - Rake::Task[Rake::Task[:default].prereqs.first].invoke - end - end - end -else - if RUBY_PLATFORM == "java" - task :clean => :require_mvn do - system("mvn --batch-mode clean") - end - - task :compile => :require_mvn do - system("mvn --batch-mode package") - end - - task :require_mvn do - raise ArgumentError, "maven needs to be installed" if `which mvn` == '' - end - - else - unless ENV['IN_DOCKER'] == 'true' - task :build => [:copy_third_party, :copy_shared] - end Rake::ExtensionTask.new("protobuf_c", spec) do |ext| unless RUBY_PLATFORM =~ /darwin/ @@ -297,9 +238,6 @@ else task 'gem:native' => [:genproto, 'gem:windows', 'gem:java'] end end -end - -task :compile => ["ffi-compiler:default"] if USE_FFI task :compile => ["ffi-compiler:default"] if USE_FFI diff --git a/ruby/ext/google/shared/shared_convert.c b/ruby/ext/google/shared/shared_convert.c deleted file mode 100644 index 57a8be7501ce..000000000000 --- a/ruby/ext/google/shared/shared_convert.c +++ /dev/null @@ -1,85 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2023 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// ----------------------------------------------------------------------------- -// Ruby <-> upb data conversion functions. Strictly free of dependencies on -// Ruby interpreter internals. - -#include "shared_convert.h" - -bool shared_Msgval_IsEqual(upb_MessageValue val1, upb_MessageValue val2, - upb_CType type, upb_MessageDef* msgdef, upb_Status* status) { - switch (type) { - case kUpb_CType_Bool: - return memcmp(&val1, &val2, 1) == 0; - case kUpb_CType_Float: - case kUpb_CType_Int32: - case kUpb_CType_UInt32: - case kUpb_CType_Enum: - return memcmp(&val1, &val2, 4) == 0; - case kUpb_CType_Double: - case kUpb_CType_Int64: - case kUpb_CType_UInt64: - return memcmp(&val1, &val2, 8) == 0; - case kUpb_CType_String: - case kUpb_CType_Bytes: - return val1.str_val.size == val2.str_val.size && - memcmp(val1.str_val.data, val2.str_val.data, val1.str_val.size) == - 0; - case kUpb_CType_Message: - return shared_Message_Equal(val1.msg_val, val2.msg_val, msgdef, status); - default: - upb_Status_SetErrorMessage(status, "Internal error, unexpected type"); - } -} - -uint64_t shared_Msgval_GetHash(upb_MessageValue val, upb_CType type, upb_MessageDef* msgdef, - uint64_t seed, upb_Status* status) { - switch (type) { - case kUpb_CType_Bool: - return _upb_Hash(&val, 1, seed); - case kUpb_CType_Float: - case kUpb_CType_Int32: - case kUpb_CType_UInt32: - case kUpb_CType_Enum: - return _upb_Hash(&val, 4, seed); - case kUpb_CType_Double: - case kUpb_CType_Int64: - case kUpb_CType_UInt64: - return _upb_Hash(&val, 8, seed); - case kUpb_CType_String: - case kUpb_CType_Bytes: - return _upb_Hash(val.str_val.data, val.str_val.size, seed); - case kUpb_CType_Message: - return shared_Message_Hash(val.msg_val, msgdef, seed, status); - default: - upb_Status_SetErrorMessage(status, "Internal error, unexpected type"); - } -} \ No newline at end of file diff --git a/ruby/ext/google/shared/shared_convert.h b/ruby/ext/google/shared/shared_convert.h deleted file mode 100644 index 8659cca772a4..000000000000 --- a/ruby/ext/google/shared/shared_convert.h +++ /dev/null @@ -1,47 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2023 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// ----------------------------------------------------------------------------- -// Ruby <-> upb data conversion functions. Strictly free of dependencies on -// Ruby interpreter internals. - -#ifndef RUBY_PROTOBUF_SHARED_CONVERT_H_ -#define RUBY_PROTOBUF_SHARED_CONVERT_H_ - -#include "ruby-upb.h" -#include "shared_message.h" - -bool shared_Msgval_IsEqual(upb_MessageValue val1, upb_MessageValue val2, - upb_CType type, upb_MessageDef* msgdef, upb_Status* status); - -uint64_t shared_Msgval_GetHash(upb_MessageValue val, upb_CType type, upb_MessageDef* msgdef, - uint64_t seed, upb_Status* status); - -#endif // RUBY_PROTOBUF_SHARED_CONVERT_H_ diff --git a/ruby/ext/google/shared/shared_message.c b/ruby/ext/google/shared/shared_message.c deleted file mode 100644 index da9fd9636992..000000000000 --- a/ruby/ext/google/shared/shared_message.c +++ /dev/null @@ -1,87 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2023 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// ----------------------------------------------------------------------------- -// Ruby Message functions. Strictly free of dependencies on -// Ruby interpreter internals. - -#include "shared_message.h" - -// Support function for Message_Hash. Returns a hash value for the given message. -uint64_t shared_Message_Hash(const upb_Message* msg, const upb_MessageDef* m, - uint64_t seed, upb_Status* status) { - upb_Arena* arena = upb_Arena_New(); - char* data; - size_t size; - - // Hash a deterministically serialized payloads with no unknown fields. - upb_EncodeStatus encode_status = upb_Encode( - msg, upb_MessageDef_MiniTable(m), - kUpb_EncodeOption_SkipUnknown | kUpb_EncodeOption_Deterministic, arena, - &data, &size); - - if (encode_status == kUpb_EncodeStatus_Ok) { - uint64_t ret = _upb_Hash(data, size, seed); - upb_Arena_Free(arena); - return ret; - } else { - upb_Arena_Free(arena); - upb_Status_SetErrorMessage(status, "Error calculating hash"); - } -} - -// Support function for Message_Equal -bool shared_Message_Equal(const upb_Message* m1, const upb_Message* m2, - const upb_MessageDef* m, upb_Status* status) { - if (m1 == m2) return true; - - size_t size1, size2; - int encode_opts = - kUpb_EncodeOption_SkipUnknown | kUpb_EncodeOption_Deterministic; - upb_Arena* arena_tmp = upb_Arena_New(); - const upb_MiniTable* layout = upb_MessageDef_MiniTable(m); - - // Compare deterministically serialized payloads with no unknown fields. - char* data1; - char* data2; - upb_EncodeStatus status1 = - upb_Encode(m1, layout, encode_opts, arena_tmp, &data1, &size1); - upb_EncodeStatus status2 = - upb_Encode(m2, layout, encode_opts, arena_tmp, &data2, &size2); - - if (status1 == kUpb_EncodeStatus_Ok && status2 == kUpb_EncodeStatus_Ok) { - bool ret = (size1 == size2) && (memcmp(data1, data2, size1) == 0); - upb_Arena_Free(arena_tmp); - return ret; - } else { - upb_Arena_Free(arena_tmp); - upb_Status_SetErrorMessage(status, "Error comparing messages"); - } -} diff --git a/ruby/ext/google/shared/shared_message.h b/ruby/ext/google/shared/shared_message.h deleted file mode 100644 index 0e46d2e6a816..000000000000 --- a/ruby/ext/google/shared/shared_message.h +++ /dev/null @@ -1,48 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2023 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// ----------------------------------------------------------------------------- -// Ruby Message functions. Strictly free of dependencies on -// Ruby interpreter internals. - -#ifndef RUBY_PROTOBUF_SHARED_MESSAGE_H_ -#define RUBY_PROTOBUF_SHARED_MESSAGE_H_ - -#include "ruby-upb.h" - -// Returns a hash value for the given message. -uint64_t shared_Message_Hash(const upb_Message* msg, const upb_MessageDef* m, - uint64_t seed, upb_Status* status); - -// Returns true if these two messages are equal. -bool shared_Message_Equal(const upb_Message* m1, const upb_Message* m2, - const upb_MessageDef* m, upb_Status* status); - -#endif // RUBY_PROTOBUF_SHARED_MESSAGE_H_ diff --git a/ruby/lib/google/protobuf.rb b/ruby/lib/google/protobuf.rb index 1dd898e3b44e..855ab605e95b 100644 --- a/ruby/lib/google/protobuf.rb +++ b/ruby/lib/google/protobuf.rb @@ -39,6 +39,13 @@ class Error < StandardError; end class ParseError < Error; end class TypeError < ::TypeError; end + begin + require 'ffi' + PREFER_FFI = true + rescue LoadError + PREFER_FFI = false + end + def self.encode(msg, options = {}) msg.to_proto(options) end @@ -55,15 +62,20 @@ def self.decode_json(klass, json, options = {}) klass.decode_json(json, options) end - IMPLEMENTATION = case ENV['PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION'] - when nil, 'native', '' - require 'google/protobuf_native' - :NATIVE - when 'ffi' + IMPLEMENTATION = if PREFER_FFI + begin require 'google/protobuf_ffi' - :FFI - else - raise RuntimeError.new("Unsupported value of PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION environment variable `#{ENV['PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION']}`. Must be one of `native` or `ffi`. Defaults to `native`.") + :FFI + rescue LoadError + warn "Caught exception `#{$!.message}` while loading FFI implementation of google/protobuf." + raise $! + warn "Falling back to native implementation." + require 'google/protobuf_native' + :NATIVE + end + else + require 'google/protobuf_native' + :NATIVE end end end diff --git a/ruby/lib/google/protobuf/ffi/descriptor.rb b/ruby/lib/google/protobuf/ffi/descriptor.rb index 0fb18e6b748a..916453c66779 100644 --- a/ruby/lib/google/protobuf/ffi/descriptor.rb +++ b/ruby/lib/google/protobuf/ffi/descriptor.rb @@ -216,13 +216,19 @@ def eql?(other) temporary_arena = Google::Protobuf::FFI.create_arena mini_table = Google::Protobuf::FFI.get_mini_table(self.class.descriptor) size_one = ::FFI::MemoryPointer.new(:size_t, 1) - encoding_one = Google::Protobuf::FFI.encode_message(@msg, mini_table, encoding_options, temporary_arena, size_one) + encoding_one = ::FFI::MemoryPointer.new(:pointer, 1) + encoding_status = Google::Protobuf::FFI.encode_message(@msg, mini_table, encoding_options, temporary_arena, encoding_one.to_ptr, size_one) + raise ParseError.new "Error comparing messages due to #{encoding_status} while encoding LHS of `eql?()`" unless encoding_status == :Ok + size_two = ::FFI::MemoryPointer.new(:size_t, 1) - encoding_two = Google::Protobuf::FFI.encode_message(other.instance_variable_get(:@msg), mini_table, encoding_options, temporary_arena, size_two) + encoding_two = ::FFI::MemoryPointer.new(:pointer, 1) + encoding_status = Google::Protobuf::FFI.encode_message(other.instance_variable_get(:@msg), mini_table, encoding_options, temporary_arena, encoding_two.to_ptr, size_two) + raise ParseError.new "Error comparing messages due to #{encoding_status} while encoding RHS of `eql?()`" unless encoding_status == :Ok + if encoding_one.null? or encoding_two.null? raise ParseError.new "Error comparing messages" end - size_one.read(:size_t) == size_two.read(:size_t) and Google::Protobuf::FFI.memcmp(encoding_one, encoding_two, size_one.read(:size_t)).zero? + size_one.read(:size_t) == size_two.read(:size_t) and Google::Protobuf::FFI.memcmp(encoding_one.read(:pointer), encoding_two.read(:pointer), size_one.read(:size_t)).zero? end alias == eql? @@ -231,11 +237,12 @@ def hash temporary_arena = Google::Protobuf::FFI.create_arena mini_table_ptr = Google::Protobuf::FFI.get_mini_table(self.class.descriptor) size_ptr = ::FFI::MemoryPointer.new(:size_t, 1) - encoding = Google::Protobuf::FFI.encode_message(@msg, mini_table_ptr, encoding_options, temporary_arena, size_ptr) - if encoding.null? + encoding = ::FFI::MemoryPointer.new(:pointer, 1) + encoding_status = Google::Protobuf::FFI.encode_message(@msg, mini_table_ptr, encoding_options, temporary_arena, encoding.to_ptr, size_ptr) + if encoding_status != :Ok or encoding.null? raise ParseError.new "Error calculating hash" end - Google::Protobuf::FFI.hash(encoding, size_ptr.read(:size_t), 0) + Google::Protobuf::FFI.hash(encoding.read(:pointer), size_ptr.read(:size_t), 0) end def to_h From 95c4535a205bc891f58ee0ad52b7eaac67e06f44 Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Fri, 7 Jul 2023 11:57:21 -0400 Subject: [PATCH 41/54] Call `upb_Arena_Malloc()` directly. --- ruby/ext/google/protobuf_c/glue.c | 4 ---- ruby/lib/google/protobuf/ffi/ffi.rb | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/ruby/ext/google/protobuf_c/glue.c b/ruby/ext/google/protobuf_c/glue.c index 35cef340e61e..16d53130d151 100644 --- a/ruby/ext/google/protobuf_c/glue.c +++ b/ruby/ext/google/protobuf_c/glue.c @@ -44,7 +44,3 @@ google_protobuf_FileDescriptorProto* FileDescriptorProto_parse (const char* seri serialized_file_proto, length, arena); } - -void* upb_Arena_FastMalloc(upb_Arena* a, size_t size) { - return upb_Arena_Malloc(a, size); -} \ No newline at end of file diff --git a/ruby/lib/google/protobuf/ffi/ffi.rb b/ruby/lib/google/protobuf/ffi/ffi.rb index 787dd042bbb6..611531e59a34 100644 --- a/ruby/lib/google/protobuf/ffi/ffi.rb +++ b/ruby/lib/google/protobuf/ffi/ffi.rb @@ -256,7 +256,7 @@ def initialize # Argument takes a :pointer rather than a typed Arena here due to # implementation details of FFI::AutoPointer. attach_function :free_arena, :upb_Arena_Free, [:pointer], :void - attach_function :arena_malloc, :upb_Arena_FastMalloc, [Arena, :size_t], :pointer + attach_function :arena_malloc, :upb_Arena_Malloc, [Arena, :size_t], :pointer # Array attach_function :append_array, :upb_Array_Append, [:Array, MessageValue.by_value, Arena], :bool From 0286c715850a9bd34a6b6998b83847b52b8895a0 Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Fri, 7 Jul 2023 15:28:07 -0400 Subject: [PATCH 42/54] Address PR feedback. --- ruby/ext/google/protobuf_c/ruby-upb.h | 4 +- ruby/lib/google/protobuf/ffi/descriptor.rb | 626 +---------------- .../google/protobuf/ffi/enum_descriptor.rb | 42 +- ruby/lib/google/protobuf/ffi/ffi.rb | 33 +- .../google/protobuf/ffi/field_descriptor.rb | 24 +- .../google/protobuf/ffi/internal/convert.rb | 13 - .../protobuf/ffi/internal/pointer_helper.rb | 58 ++ .../protobuf/ffi/internal/type_safety.rb | 2 +- ruby/lib/google/protobuf/ffi/message.rb | 640 ++++++++++++++++++ .../google/protobuf/ffi/oneof_descriptor.rb | 26 +- ruby/lib/google/protobuf_ffi.rb | 2 + 11 files changed, 733 insertions(+), 737 deletions(-) create mode 100644 ruby/lib/google/protobuf/ffi/internal/pointer_helper.rb create mode 100644 ruby/lib/google/protobuf/ffi/message.rb diff --git a/ruby/ext/google/protobuf_c/ruby-upb.h b/ruby/ext/google/protobuf_c/ruby-upb.h index 4ea8b52cff78..5f55c079f8e8 100755 --- a/ruby/ext/google/protobuf_c/ruby-upb.h +++ b/ruby/ext/google/protobuf_c/ruby-upb.h @@ -10397,7 +10397,7 @@ UPB_API_INLINE upb_CType upb_FieldDef_CType(const upb_FieldDef* f); UPB_API_INLINE upb_MessageValue upb_FieldDef_Default(const upb_FieldDef* f); UPB_API_INLINE const upb_EnumDef* upb_FieldDef_EnumSubDef(const upb_FieldDef* f); const upb_MessageDef* upb_FieldDef_ExtensionScope(const upb_FieldDef* f); -const upb_FileDef* upb_FieldDef_File(const upb_FieldDef* f); +UPB_API_INLINE const upb_FileDef* upb_FieldDef_File(const upb_FieldDef* f); const char* upb_FieldDef_FullName(const upb_FieldDef* f); bool upb_FieldDef_HasDefault(const upb_FieldDef* f); bool upb_FieldDef_HasJsonName(const upb_FieldDef* f); @@ -10670,7 +10670,7 @@ const upb_ServiceDef* upb_MethodDef_Service(const upb_MethodDef* m); extern "C" { #endif -const upb_MessageDef* upb_OneofDef_ContainingType(const upb_OneofDef* o); +UPB_API_INLINE const upb_MessageDef* upb_OneofDef_ContainingType(const upb_OneofDef* o); UPB_API_INLINE const upb_FieldDef* upb_OneofDef_Field(const upb_OneofDef* o, int i); UPB_API_INLINE int upb_OneofDef_FieldCount(const upb_OneofDef* o); const char* upb_OneofDef_FullName(const upb_OneofDef* o); diff --git a/ruby/lib/google/protobuf/ffi/descriptor.rb b/ruby/lib/google/protobuf/ffi/descriptor.rb index 916453c66779..06fd6136f034 100644 --- a/ruby/lib/google/protobuf/ffi/descriptor.rb +++ b/ruby/lib/google/protobuf/ffi/descriptor.rb @@ -42,6 +42,7 @@ class Descriptor class << self prepend Google::Protobuf::Internal::TypeSafety + include Google::Protobuf::Internal::PointerHelper # @param value [Descriptor] Descriptor to convert to an FFI native type # @param _ [Object] Unused @@ -57,29 +58,15 @@ def to_native(value, _ = nil) # @param _ [Object] Unused def from_native(msg_def, _ = nil) return nil if msg_def.nil? or msg_def.null? - # Calling get_message_file_def(msg_def) would create a cyclic - # dependency because FFI would then complain about passing an - # FFI::Pointer instance instead of a Descriptor. Instead, directly - # read the top of the MsgDef structure an extract the FileDef*. - # file_def = Google::Protobuf::FFI.get_message_file_def msg_def - message_def_struct = Google::Protobuf::FFI::Upb_MessageDef.new(msg_def) - file_def = message_def_struct[:file_def] - raise RuntimeError.new "FileDef is nil" if file_def.nil? - raise RuntimeError.new "FileDef is null" if file_def.null? - pool_def = Google::Protobuf::FFI.file_def_pool file_def - raise RuntimeError.new "PoolDef is nil" if pool_def.nil? - raise RuntimeError.new "PoolDef is null" if pool_def.null? - pool = Google::Protobuf::ObjectCache.get(pool_def) - raise "Cannot find pool in ObjectCache!" if pool.nil? - descriptor = pool.descriptor_class_by_def[msg_def.address] - if descriptor.nil? - pool.descriptor_class_by_def[msg_def.address] = private_constructor(msg_def, pool) - else - descriptor - end + file_def = Google::Protobuf::FFI.get_message_file_def msg_def + descriptor_from_file_def(file_def, msg_def) end end + def to_native + self.class.to_native(self) + end + ## # Great write up of this strategy: # See https://blog.appsignal.com/2018/08/07/ruby-magic-changing-the-way-ruby-creates-objects.html @@ -96,7 +83,7 @@ def inspect end def file_descriptor - @descriptor_pool.send(:get_file_descriptor, Google::Protobuf::FFI.get_message_file_def(self)) + @descriptor_pool.send(:get_file_descriptor, Google::Protobuf::FFI.get_message_file_def(@msg_def)) end def name @@ -172,603 +159,6 @@ def self.get_message(msg, descriptor, arena) def pool @descriptor_pool end - - def build_message_class - descriptor = self - Class.new(Google::Protobuf::const_get(:AbstractMessage)) do - @descriptor = descriptor - class << self - attr_accessor :descriptor - private - attr_accessor :oneof_field_names - include ::Google::Protobuf::Internal::Convert - end - - alias original_method_missing method_missing - def method_missing(method_name, *args) - method_missing_internal method_name, *args, mode: :method_missing - end - - def respond_to_missing?(method_name, include_private = false) - method_missing_internal(method_name, mode: :respond_to_missing?) || super - end - - ## - # Public constructor. Automatically allocates from a new Arena. - def self.new(initial_value = nil) - instance = allocate - instance.send(:initialize, initial_value) - instance - end - - def dup - duplicate = self.class.private_constructor(@arena) - mini_table = Google::Protobuf::FFI.get_mini_table(self.class.descriptor) - size = mini_table[:size] - duplicate.instance_variable_get(:@msg).write_string_length(@msg.read_string_length(size), size) - duplicate - end - alias clone dup - - def eql?(other) - return false unless self.class === other - encoding_options = Google::Protobuf::FFI::Upb_Encode_Deterministic | Google::Protobuf::FFI::Upb_Encode_SkipUnknown - temporary_arena = Google::Protobuf::FFI.create_arena - mini_table = Google::Protobuf::FFI.get_mini_table(self.class.descriptor) - size_one = ::FFI::MemoryPointer.new(:size_t, 1) - encoding_one = ::FFI::MemoryPointer.new(:pointer, 1) - encoding_status = Google::Protobuf::FFI.encode_message(@msg, mini_table, encoding_options, temporary_arena, encoding_one.to_ptr, size_one) - raise ParseError.new "Error comparing messages due to #{encoding_status} while encoding LHS of `eql?()`" unless encoding_status == :Ok - - size_two = ::FFI::MemoryPointer.new(:size_t, 1) - encoding_two = ::FFI::MemoryPointer.new(:pointer, 1) - encoding_status = Google::Protobuf::FFI.encode_message(other.instance_variable_get(:@msg), mini_table, encoding_options, temporary_arena, encoding_two.to_ptr, size_two) - raise ParseError.new "Error comparing messages due to #{encoding_status} while encoding RHS of `eql?()`" unless encoding_status == :Ok - - if encoding_one.null? or encoding_two.null? - raise ParseError.new "Error comparing messages" - end - size_one.read(:size_t) == size_two.read(:size_t) and Google::Protobuf::FFI.memcmp(encoding_one.read(:pointer), encoding_two.read(:pointer), size_one.read(:size_t)).zero? - end - alias == eql? - - def hash - encoding_options = Google::Protobuf::FFI::Upb_Encode_Deterministic | Google::Protobuf::FFI::Upb_Encode_SkipUnknown - temporary_arena = Google::Protobuf::FFI.create_arena - mini_table_ptr = Google::Protobuf::FFI.get_mini_table(self.class.descriptor) - size_ptr = ::FFI::MemoryPointer.new(:size_t, 1) - encoding = ::FFI::MemoryPointer.new(:pointer, 1) - encoding_status = Google::Protobuf::FFI.encode_message(@msg, mini_table_ptr, encoding_options, temporary_arena, encoding.to_ptr, size_ptr) - if encoding_status != :Ok or encoding.null? - raise ParseError.new "Error calculating hash" - end - Google::Protobuf::FFI.hash(encoding.read(:pointer), size_ptr.read(:size_t), 0) - end - - def to_h - to_h_internal @msg, self.class.descriptor - end - - ## - # call-seq: - # Message.inspect => string - # - # Returns a human-readable string representing this message. It will be - # formatted as "". Each - # field's value is represented according to its own #inspect method. - def inspect - self.class.inspect_internal @msg - end - - def to_s - self.inspect - end - - ## - # call-seq: - # Message.[](index) => value - # Accesses a field's value by field name. The provided field name - # should be a string. - def [](name) - raise TypeError.new "Expected String for name but got #{name.class}" unless name.is_a? String - index_internal name - end - - ## - # call-seq: - # Message.[]=(index, value) - # Sets a field's value by field name. The provided field name should - # be a string. - # @param name [String] Name of the field to be set - # @param value [Object] Value to set the field to - def []=(name, value) - raise TypeError.new "Expected String for name but got #{name.class}" unless name.is_a? String - index_assign_internal(value, name: name) - end - - ## - # call-seq: - # MessageClass.decode(data, options) => message - # - # Decodes the given data (as a string containing bytes in protocol buffers wire - # format) under the interpretation given by this message class's definition - # and returns a message object with the corresponding field values. - # @param data [String] Binary string in Protobuf wire format to decode - # @param options [Hash] options for the decoder - # @option options [Integer] :recursion_limit Set to maximum decoding depth for message (default is 64) - def self.decode(data, options = {}) - raise ArgumentError.new "Expected hash arguments." unless options.is_a? Hash - raise ArgumentError.new "Expected string for binary protobuf data." unless data.is_a? String - decoding_options = 0 - depth = options[:recursion_limit] - - if depth.is_a? Numeric - decoding_options |= Google::Protobuf::FFI.decode_max_depth(depth.to_i) - end - - message = new - mini_table_ptr = Google::Protobuf::FFI.get_mini_table(message.class.descriptor) - status = Google::Protobuf::FFI.decode_message(data, data.bytesize, message.instance_variable_get(:@msg), mini_table_ptr, nil, decoding_options, message.instance_variable_get(:@arena)) - raise ParseError.new "Error occurred during parsing" unless status == :Ok - message - end - - ## - # call-seq: - # MessageClass.encode(msg, options) => bytes - # - # Encodes the given message object to its serialized form in protocol buffers - # wire format. - # @param options [Hash] options for the encoder - # @option options [Integer] :recursion_limit Set to maximum encoding depth for message (default is 64) - def self.encode(message, options = {}) - raise ArgumentError.new "Message of wrong type." unless message.is_a? self - raise ArgumentError.new "Expected hash arguments." unless options.is_a? Hash - - encoding_options = 0 - depth = options[:recursion_limit] - - if depth.is_a? Numeric - encoding_options |= Google::Protobuf::FFI.decode_max_depth(depth.to_i) - end - - encode_internal(message.instance_variable_get(:@msg), encoding_options) do |encoding, size, _| - if encoding.nil? or encoding.null? - raise RuntimeError.new "Exceeded maximum depth (possibly cycle)" - else - encoding.read_string_length(size).force_encoding("ASCII-8BIT").freeze - end - end - end - - ## - # all-seq: - # MessageClass.decode_json(data, options = {}) => message - # - # Decodes the given data (as a string containing bytes in protocol buffers wire - # format) under the interpretation given by this message class's definition - # and returns a message object with the corresponding field values. - # - # @param options [Hash] options for the decoder - # @option options [Boolean] :ignore_unknown_fields Set true to ignore unknown fields (default is to raise an error) - # @return [Message] - def self.decode_json(data, options = {}) - decoding_options = 0 - unless options.is_a? Hash - if options.respond_to? :to_h - options options.to_h - else - #TODO(jatl) can this error message be improve to include what was received? - raise ArgumentError.new "Expected hash arguments" - end - end - raise ArgumentError.new "Expected string for JSON data." unless data.is_a? String - raise RuntimeError.new "Cannot parse a wrapper directly" if descriptor.send(:wrapper?) - - if options[:ignore_unknown_fields] - decoding_options |= Google::Protobuf::FFI::Upb_JsonDecode_IgnoreUnknown - end - - message = new - pool_def = pool_def_from_message_definition(message.class.descriptor) - status = Google::Protobuf::FFI::Status.new - unless Google::Protobuf::FFI.json_decode_message(data, data.bytesize, message.instance_variable_get(:@msg), message.class.descriptor, pool_def, decoding_options, message.instance_variable_get(:@arena), status) - raise ParseError.new "Error occurred during parsing: #{Google::Protobuf::FFI.error_message(status)}" - end - message - end - - def self.encode_json(message, options = {}) - encoding_options = 0 - unless options.is_a? Hash - if options.respond_to? :to_h - options = options.to_h - else - #TODO(jatl) can this error message be improve to include what was received? - raise ArgumentError.new "Expected hash arguments" - end - end - - if options[:preserve_proto_fieldnames] - encoding_options |= Google::Protobuf::FFI::Upb_JsonEncode_UseProtoNames - end - if options[:emit_defaults] - encoding_options |= Google::Protobuf::FFI::Upb_JsonEncode_EmitDefaults - end - if options[:format_enums_as_integers] - encoding_options |= Google::Protobuf::FFI::Upb_JsonEncode_FormatEnumsAsIntegers - end - - buffer_size = 1024 - buffer = ::FFI::MemoryPointer.new(:char, buffer_size) - status = Google::Protobuf::FFI::Status.new - msg = message.instance_variable_get(:@msg) - pool_def = pool_def_from_message_definition(message.class.descriptor) - size = Google::Protobuf::FFI::json_encode_message(msg, message.class.descriptor, pool_def, encoding_options, buffer, buffer_size, status) - unless status[:ok] - raise ParseError.new "Error occurred during encoding: #{Google::Protobuf::FFI.error_message(status)}" - end - - if size >= buffer_size - buffer_size = size + 1 - buffer = ::FFI::MemoryPointer.new(:char, buffer_size) - status.clear - size = Google::Protobuf::FFI::json_encode_message(msg, message.class.descriptor, pool_def, encoding_options, buffer, buffer_size, status) - unless status[:ok] - raise ParseError.new "Error occurred during encoding: #{Google::Protobuf::FFI.error_message(status)}" - end - if size >= buffer_size - raise ParseError.new "Inconsistent JSON encoding sizes - was #{buffer_size - 1}, now #{size}" - end - end - - buffer.read_string_length(size).force_encoding("UTF-8").freeze - end - - @descriptor.each do |field_descriptor| - field_name = field_descriptor.name - unless instance_methods(true).include?(field_name.to_sym) - #TODO(jatl) - at a high level, dispatching to either - # index_internal or get_field would be logically correct, but slightly slower. - if field_descriptor.map? - define_method(field_name) do - mutable_message_value = Google::Protobuf::FFI.get_mutable_message @msg, field_descriptor, @arena - get_map_field(mutable_message_value[:map], field_descriptor) - end - elsif field_descriptor.repeated? - define_method(field_name) do - mutable_message_value = Google::Protobuf::FFI.get_mutable_message @msg, field_descriptor, @arena - get_repeated_field(mutable_message_value[:array], field_descriptor) - end - elsif field_descriptor.sub_message? - define_method(field_name) do - return nil unless Google::Protobuf::FFI.get_message_has @msg, field_descriptor - mutable_message = Google::Protobuf::FFI.get_mutable_message @msg, field_descriptor, @arena - sub_message = mutable_message[:msg] - sub_message_def = Google::Protobuf::FFI.get_subtype_as_message(field_descriptor) - Descriptor.send(:get_message, sub_message, sub_message_def, @arena) - end - else - c_type = field_descriptor.send(:c_type) - if c_type == :enum - define_method(field_name) do - message_value = Google::Protobuf::FFI.get_message_value @msg, field_descriptor - convert_upb_to_ruby message_value, c_type, Google::Protobuf::FFI.get_subtype_as_enum(field_descriptor) - end - else - define_method(field_name) do - message_value = Google::Protobuf::FFI.get_message_value @msg, field_descriptor - convert_upb_to_ruby message_value, c_type - end - end - end - define_method("#{field_name}=") do |value| - index_assign_internal(value, field_descriptor: field_descriptor) - end - define_method("clear_#{field_name}") do - clear_internal(field_descriptor) - end - if field_descriptor.type == :enum - define_method("#{field_name}_const") do - if field_descriptor.repeated? - return_value = [] - get_field(field_descriptor).send(:each_msg_val) do |msg_val| - return_value << msg_val[:int32_val] - end - return_value - else - message_value = Google::Protobuf::FFI.get_message_value @msg, field_descriptor - message_value[:int32_val] - end - end - end - if !field_descriptor.repeated? and field_descriptor.wrapper? - define_method("#{field_name}_as_value") do - get_field(field_descriptor, unwrap: true) - end - define_method("#{field_name}_as_value=") do |value| - if value.nil? - clear_internal(field_descriptor) - else - index_assign_internal(value, field_descriptor: field_descriptor, wrap: true) - end - end - end - if field_descriptor.has_presence? - define_method("has_#{field_name}?") do - Google::Protobuf::FFI.get_message_has(@msg, field_descriptor) - end - end - end - end - - @oneof_field_names = [] - - @descriptor.each_oneof do |oneof_descriptor| - field_name = oneof_descriptor.name.to_sym - @oneof_field_names << field_name - unless instance_methods(true).include?(field_name) - define_method(field_name) do - field_descriptor = Google::Protobuf::FFI.get_message_which_oneof(@msg, oneof_descriptor) - if field_descriptor.nil? - return - else - return field_descriptor.name.to_sym - end - end - define_method("clear_#{field_name}") do - field_descriptor = Google::Protobuf::FFI.get_message_which_oneof(@msg, oneof_descriptor) - unless field_descriptor.nil? - clear_internal(field_descriptor) - end - end - define_method("has_#{field_name}?") do - !Google::Protobuf::FFI.get_message_which_oneof(@msg, oneof_descriptor).nil? - end - end - end - - private - # Implementation details below are subject to breaking changes without - # warning and are intended for use only within the gem. - - def method_missing_internal(method_name, *args, mode: nil) - raise ArgumentError.new "method_missing_internal called with invalid mode #{mode.inspect}" unless [:respond_to_missing?, :method_missing].include? mode - - #TODO(jatl) not being allowed is not the same thing as not responding, but this is needed to pass tests - if method_name.to_s.end_with? '=' - if self.class.send(:oneof_field_names).include? method_name.to_s[0..-2].to_sym - return false if mode == :respond_to_missing? - raise RuntimeError.new "Oneof accessors are read-only." - end - end - - original_method_missing(method_name, *args) if mode == :method_missing - end - - def self.private_constructor(arena, msg: nil, initial_value: nil) - instance = allocate - instance.send(:initialize, initial_value, arena, msg) - instance - end - - def clear_internal(field_def) - raise FrozenError.new "can't modify frozen #{self.class}" if frozen? - Google::Protobuf::FFI.clear_message_field(@msg, field_def) - end - - def index_internal(name) - field_descriptor = self.class.descriptor.lookup(name) - get_field field_descriptor unless field_descriptor.nil? - end - - #TODO(jatl) - well known types keeps us on our toes by overloading methods. - # How much of the public API needs to be defended? - def index_assign_internal(value, name: nil, field_descriptor: nil, wrap: false) - raise FrozenError.new "can't modify frozen #{self.class}" if frozen? - if field_descriptor.nil? - field_descriptor = self.class.descriptor.lookup(name) - if field_descriptor.nil? - raise ArgumentError.new "Unknown field: #{name}" - end - end - unless field_descriptor.send :set_value_on_message, value, @msg, @arena, wrap: wrap - raise RuntimeError.new "allocation failed" - end - end - - ## - # @param initial_value [Object] initial value of this Message - # @param arena [Arena] Optional; Arena where this message will be allocated - # @param msg [::FFI::Pointer] Optional; value of this message - def initialize(initial_value = nil, arena = nil, msg = nil) - @arena = arena || Google::Protobuf::FFI.create_arena - @msg = msg || Google::Protobuf::FFI.new_message_from_def(self.class.descriptor, @arena) - - unless initial_value.nil? - raise ArgumentError.new "Expected hash arguments or message, not #{initial_value.class}" unless initial_value.respond_to? :each - - field_def_ptr = ::FFI::MemoryPointer.new :pointer - oneof_def_ptr = ::FFI::MemoryPointer.new :pointer - - initial_value.each do |key, value| - raise ArgumentError.new "Expected string or symbols as hash keys when initializing proto from hash." unless [String, Symbol].include? key.class - - unless Google::Protobuf::FFI.find_msg_def_by_name self.class.descriptor, key.to_s, key.to_s.bytesize, field_def_ptr, oneof_def_ptr - raise ArgumentError.new "Unknown field name '#{key}' in initialization map entry." - end - raise NotImplementedError.new "Haven't added oneofsupport yet" unless oneof_def_ptr.get_pointer(0).null? - raise NotImplementedError.new "Expected a field def" if field_def_ptr.get_pointer(0).null? - - field_descriptor = FieldDescriptor.from_native field_def_ptr.get_pointer(0) - - next if value.nil? - if field_descriptor.map? - index_assign_internal(Google::Protobuf::Map.send(:construct_for_field, field_descriptor, @arena, value: value), name: key.to_s) - elsif field_descriptor.repeated? - index_assign_internal(RepeatedField.send(:construct_for_field, field_descriptor, @arena, values: value), name: key.to_s) - # TODO - Is it OK not trap this and just let []= convert it for me?? - # elsif field_descriptor.sub_message? - # raise NotImplementedError - else - index_assign_internal(value, name: key.to_s) - end - end - end - - # Should always be the last expression of the initializer to avoid - # leaking references to this object before construction is complete. - Google::Protobuf::ObjectCache.add @msg, self - end - - include Google::Protobuf::Internal::Convert - - - def self.inspect_field(field_descriptor, c_type, message_value) - if field_descriptor.sub_message? - sub_msg_descriptor = Google::Protobuf::FFI.get_subtype_as_message(field_descriptor) - sub_msg_descriptor.msgclass.send(:inspect_internal, message_value[:msg_val]) - else - convert_upb_to_ruby(message_value, c_type, field_descriptor.subtype).inspect - end - end - - # @param field_def [::FFI::Pointer] Pointer to the Message - def self.inspect_internal(msg) - field_output = [] - descriptor.each do |field_descriptor| - next if field_descriptor.has_presence? && !Google::Protobuf::FFI.get_message_has(msg, field_descriptor) - if field_descriptor.map? - # TODO(jatl) Adapted - from map#each_msg_val and map#inspect- can this be refactored to reduce echo without introducing a arena allocation? - message_descriptor = field_descriptor.subtype - key_field_def = Google::Protobuf::FFI.get_field_by_number(message_descriptor, 1) - key_field_type = Google::Protobuf::FFI.get_type(key_field_def) - - value_field_def = Google::Protobuf::FFI.get_field_by_number(message_descriptor, 2) - value_field_type = Google::Protobuf::FFI.get_type(value_field_def) - - message_value = Google::Protobuf::FFI.get_message_value(msg, field_descriptor) - iter = ::FFI::MemoryPointer.new(:size_t, 1) - iter.write(:size_t, Google::Protobuf::FFI::Upb_Map_Begin) - key_value_pairs = [] - while Google::Protobuf::FFI.map_next(message_value[:map_val], iter) do - iter_size_t = iter.read(:size_t) - key_message_value = Google::Protobuf::FFI.map_key(message_value[:map_val], iter_size_t) - value_message_value = Google::Protobuf::FFI.map_value(message_value[:map_val], iter_size_t) - key_string = convert_upb_to_ruby(key_message_value, key_field_type).inspect - value_string = inspect_field(value_field_def, value_field_type, value_message_value) - key_value_pairs << "#{key_string}=>#{value_string}" - end - field_output << "#{field_descriptor.name}: {#{key_value_pairs.join(", ")}}" - elsif field_descriptor.repeated? - # TODO(jatl) Adapted - from repeated_field#each - can this be refactored to reduce echo? - repeated_field_output = [] - message_value = Google::Protobuf::FFI.get_message_value(msg, field_descriptor) - array = message_value[:array_val] - n = array.null? ? 0 : Google::Protobuf::FFI.array_size(array) - 0.upto(n - 1) do |i| - element = Google::Protobuf::FFI.get_msgval_at(array, i) - repeated_field_output << inspect_field(field_descriptor, field_descriptor.send(:c_type), element) - end - field_output << "#{field_descriptor.name}: [#{repeated_field_output.join(", ")}]" - else - message_value = Google::Protobuf::FFI.get_message_value msg, field_descriptor - rendered_value = inspect_field(field_descriptor, field_descriptor.send(:c_type), message_value) - field_output << "#{field_descriptor.name}: #{rendered_value}" - end - end - "<#{name}: #{field_output.join(', ')}>" - end - - ## - # Gets a field of this message identified by the argument definition. - # - # @param field [FieldDescriptor] Descriptor of the field to get - def get_field(field, unwrap: false) - if field.map? - mutable_message_value = Google::Protobuf::FFI.get_mutable_message @msg, field, @arena - get_map_field(mutable_message_value[:map], field) - elsif field.repeated? - mutable_message_value = Google::Protobuf::FFI.get_mutable_message @msg, field, @arena - get_repeated_field(mutable_message_value[:array], field) - elsif field.sub_message? - return nil unless Google::Protobuf::FFI.get_message_has @msg, field - sub_message_def = Google::Protobuf::FFI.get_subtype_as_message(field) - if unwrap - if field.has?(self) - wrapper_message_value = Google::Protobuf::FFI.get_message_value @msg, field - fields = Google::Protobuf::FFI.field_count(sub_message_def) - raise "Sub message has #{fields} fields! Expected exactly 1." unless fields == 1 - value_field_def = Google::Protobuf::FFI.get_field_by_number sub_message_def, 1 - message_value = Google::Protobuf::FFI.get_message_value wrapper_message_value[:msg_val], value_field_def - convert_upb_to_ruby message_value, Google::Protobuf::FFI.get_c_type(value_field_def) - else - nil - end - else - mutable_message = Google::Protobuf::FFI.get_mutable_message @msg, field, @arena - sub_message = mutable_message[:msg] - Descriptor.send(:get_message, sub_message, sub_message_def, @arena) - end - else - c_type = field.send(:c_type) - message_value = Google::Protobuf::FFI.get_message_value @msg, field - if c_type == :enum - convert_upb_to_ruby message_value, c_type, Google::Protobuf::FFI.get_subtype_as_enum(field) - else - convert_upb_to_ruby message_value, c_type - end - end - end - - ## - # @param array [::FFI::Pointer] Pointer to the Array - # @param field [Google::Protobuf::FieldDescriptor] Type of the repeated field - def get_repeated_field(array, field) - return nil if array.nil? or array.null? - repeated_field = ObjectCache.get(array) - if repeated_field.nil? - repeated_field = RepeatedField.send(:construct_for_field, field, @arena, array: array) - end - repeated_field - end - - - ## - # @param map [::FFI::Pointer] Pointer to the Map - # @param field [Google::Protobuf::FieldDescriptor] Type of the map field - def get_map_field(map, field) - return nil if map.nil? or map.null? - map_field = ObjectCache.get(map) - if map_field.nil? - map_field = Google::Protobuf::Map.send(:construct_for_field, field, @arena, map: map) - end - map_field - end - - def self.deep_copy(msg, arena = nil) - arena ||= Google::Protobuf::FFI.create_arena - encode_internal(msg) do |encoding, size, mini_table_ptr| - message = private_constructor(arena) - if encoding.nil? or encoding.null? or Google::Protobuf::FFI.decode_message(encoding, size, message.instance_variable_get(:@msg), mini_table_ptr, nil, 0, arena) != :Ok - raise ParseError.new "Error occurred copying proto" - end - message - end - end - - def self.encode_internal(msg, encoding_options = 0) - temporary_arena = Google::Protobuf::FFI.create_arena - - mini_table_ptr = Google::Protobuf::FFI.get_mini_table(descriptor) - size_ptr = ::FFI::MemoryPointer.new(:size_t, 1) - pointer_ptr = ::FFI::MemoryPointer.new(:pointer, 1) - encoding_status = Google::Protobuf::FFI.encode_message(msg, mini_table_ptr, encoding_options, temporary_arena, pointer_ptr.to_ptr, size_ptr) - raise "Encoding failed due to #{encoding_status}" unless encoding_status == :Ok - yield pointer_ptr.read(:pointer), size_ptr.read(:size_t), mini_table_ptr - end - end - end end end end diff --git a/ruby/lib/google/protobuf/ffi/enum_descriptor.rb b/ruby/lib/google/protobuf/ffi/enum_descriptor.rb index a3ceec2dc398..b31c4bbc066d 100644 --- a/ruby/lib/google/protobuf/ffi/enum_descriptor.rb +++ b/ruby/lib/google/protobuf/ffi/enum_descriptor.rb @@ -40,6 +40,7 @@ class EnumDescriptor class << self prepend Google::Protobuf::Internal::TypeSafety + include Google::Protobuf::Internal::PointerHelper # @param value [Arena] Arena to convert to an FFI native type # @param _ [Object] Unused @@ -52,26 +53,8 @@ def to_native(value, _) # @param _ [Object] Unused def from_native(enum_def, _) return nil if enum_def.nil? or enum_def.null? - # Calling get_message_file_def(enum_def) would create a cyclic - # dependency because FFI would then complain about passing an - # FFI::Pointer instance instead of a Descriptor. Instead, directly - # read the top of the MsgDef structure an extract the FileDef*. - # file_def = Google::Protobuf::FFI.get_message_file_def enum_def - enum_def_struct = Google::Protobuf::FFI::Upb_EnumDef.new(enum_def) - file_def = enum_def_struct[:file_def] - raise RuntimeError.new "FileDef is nil" if file_def.nil? - raise RuntimeError.new "FileDef is null" if file_def.null? - pool_def = Google::Protobuf::FFI.file_def_pool file_def - raise RuntimeError.new "PoolDef is nil" if pool_def.nil? - raise RuntimeError.new "PoolDef is null" if pool_def.null? - pool = Google::Protobuf::ObjectCache.get(pool_def) - raise "Cannot find pool in ObjectCache!" if pool.nil? - descriptor = pool.descriptor_class_by_def[enum_def.address] - if descriptor.nil? - pool.descriptor_class_by_def[enum_def.address] = private_constructor(enum_def, pool) - else - descriptor - end + file_def = Google::Protobuf::FFI.get_message_file_def enum_def + descriptor_from_file_def(file_def, enum_def) end end @@ -105,7 +88,7 @@ def lookup_value(number) def each &block n = Google::Protobuf::FFI.enum_value_count(self) - 0.upto(n-1) do |i| + 0.upto(n - 1) do |i| enum_value = Google::Protobuf::FFI.enum_value_by_index(self, i) yield(Google::Protobuf::FFI.enum_name(enum_value).to_sym, Google::Protobuf::FFI.enum_number(enum_value)) end @@ -151,14 +134,11 @@ def self.lookup_name(enum_def, name) end end - def definition - @enum_def - end - def build_enum_module descriptor = self dynamic_module = Module.new do @descriptor = descriptor + class << self attr_accessor :descriptor end @@ -166,24 +146,20 @@ class << self def self.lookup(number) descriptor.lookup_value number end + def self.resolve(name) descriptor.lookup_name name end - - private - def definition - self.class.send(:descriptor) - end - end + self.each do |name, value| if name[0] < 'A' || name[0] > 'Z' if name[0] >= 'a' and name[0] <= 'z' name = name[0].upcase + name[1..] # auto capitalize else warn( - "Enum value '#{name}' does not start with an uppercase letter " + - "as is required for Ruby constants.") + "Enum value '#{name}' does not start with an uppercase letter " + + "as is required for Ruby constants.") next end end diff --git a/ruby/lib/google/protobuf/ffi/ffi.rb b/ruby/lib/google/protobuf/ffi/ffi.rb index 611531e59a34..dd49f8a74d69 100644 --- a/ruby/lib/google/protobuf/ffi/ffi.rb +++ b/ruby/lib/google/protobuf/ffi/ffi.rb @@ -43,29 +43,7 @@ class FFI ffi_lib ::FFI::Compiler::Loader.find 'protobuf_c' end - Arena = Google::Protobuf::Internal::Arena - - # Partial definitions of the top of structs used for bootstrapping FFI. - class Upb_MessageDef < ::FFI::Struct - layout :opts, :pointer, - :mini_table, :pointer, - :file_def, :pointer - end - class Upb_EnumDef < ::FFI::Struct - layout :opts, :pointer, - :mini_table, :pointer, - :file_def, :pointer - end - class Upb_FieldDef < ::FFI::Struct - layout :opts, :pointer, - :file_def, :pointer - end - class Upb_OneofDef < ::FFI::Struct - layout :opts, :pointer, - :parent, :pointer - end - MessageDef = Google::Protobuf::Descriptor EnumDef = Google::Protobuf::EnumDescriptor FieldDef = Google::Protobuf::FieldDescriptor @@ -307,7 +285,7 @@ def initialize attach_function :get_number, :upb_FieldDef_Number, [FieldDescriptor], :uint32_t attach_function :real_containing_oneof, :upb_FieldDef_RealContainingOneof,[FieldDescriptor], OneofDef attach_function :get_type, :upb_FieldDef_Type, [FieldDescriptor], FieldType - + attach_function :file_def_by_raw_field_def, :upb_FieldDef_File, [:pointer], :FileDef # Map attach_function :map_clear, :upb_Map_Clear, [:Map], :void attach_function :map_delete, :upb_Map_Delete, [:Map, MessageValue.by_value, MessageValue.by_ref], :bool @@ -326,7 +304,7 @@ def initialize attach_function :new_message_from_def, :upb_Message_New, [MessageDef, Arena], :Message attach_function :get_field_by_index, :upb_MessageDef_Field, [MessageDef, :int], FieldDescriptor attach_function :field_count, :upb_MessageDef_FieldCount, [MessageDef], :int - attach_function :get_message_file_def, :upb_MessageDef_File, [MessageDef], :FileDef + attach_function :get_message_file_def, :upb_MessageDef_File, [:pointer], :FileDef attach_function :get_field_by_name, :upb_MessageDef_FindFieldByNameWithSize, [MessageDef, :string, :size_t], FieldDescriptor attach_function :get_field_by_number, :upb_MessageDef_FindFieldByNumber, [MessageDef, :uint32_t], FieldDescriptor attach_function :get_oneof_by_name, :upb_MessageDef_FindOneofByNameWithSize, [MessageDef, :string, :size_t], OneofDef @@ -356,9 +334,10 @@ def initialize attach_function :message_value_hash, :shared_Msgval_GetHash, [MessageValue.by_value, CType, MessageDef, :uint64_t], :uint64_t # OneofDescriptor - attach_function :get_oneof_name, :upb_OneofDef_Name, [OneofDef], :string - attach_function :get_oneof_field_count, :upb_OneofDef_FieldCount, [OneofDef], :int - attach_function :get_oneof_field_by_index, :upb_OneofDef_Field, [OneofDef, :int], FieldDescriptor + attach_function :get_oneof_name, :upb_OneofDef_Name, [OneofDef], :string + attach_function :get_oneof_field_count, :upb_OneofDef_FieldCount, [OneofDef], :int + attach_function :get_oneof_field_by_index, :upb_OneofDef_Field, [OneofDef, :int], FieldDescriptor + attach_function :get_oneof_containing_type,:upb_OneofDef_ContainingType,[:pointer], MessageDef # RepeatableField diff --git a/ruby/lib/google/protobuf/ffi/field_descriptor.rb b/ruby/lib/google/protobuf/ffi/field_descriptor.rb index b163d1bfdc3c..1424b8d4f7e7 100644 --- a/ruby/lib/google/protobuf/ffi/field_descriptor.rb +++ b/ruby/lib/google/protobuf/ffi/field_descriptor.rb @@ -41,6 +41,7 @@ class FieldDescriptor class << self prepend Google::Protobuf::Internal::TypeSafety + include Google::Protobuf::Internal::PointerHelper # @param value [FieldDescriptor] FieldDescriptor to convert to an FFI native type # @param _ [Object] Unused @@ -48,7 +49,6 @@ def to_native(value, _) field_def_ptr = value.instance_variable_get(:@field_def) warn "Underlying field_def was nil!" if field_def_ptr.nil? raise "Underlying field_def was null!" if !field_def_ptr.nil? and field_def_ptr.null? - # || ::FFI::Pointer::NULL field_def_ptr end @@ -57,26 +57,8 @@ def to_native(value, _) # @param _ [Object] Unused def from_native(field_def, _ = nil) return nil if field_def.nil? or field_def.null? - # Calling upb_FieldDef_File(field_def) would create a cyclic - # dependency because either 1) we'd have to define the method to accept - # an untyped pointer or 2) FFI would complain about passing a - # FFI::Pointer instance instead of a FieldDescriptor. Instead, directly - # read the top of the FieldDef structure and extract the FileDef*. - field_def_struct = Google::Protobuf::FFI::Upb_FieldDef.new(field_def) - file_def = field_def_struct[:file_def] - raise RuntimeError.new "FileDef is nil" if file_def.nil? - raise RuntimeError.new "FileDef is null" if file_def.null? - pool_def = Google::Protobuf::FFI.file_def_pool file_def - raise RuntimeError.new "PoolDef is nil" if pool_def.nil? - raise RuntimeError.new "PoolDef is null" if pool_def.null? - pool = Google::Protobuf::ObjectCache.get(pool_def) - raise "Cannot find pool in ObjectCache!" if pool.nil? - descriptor = pool.descriptor_class_by_def[field_def.address] - if descriptor.nil? - pool.descriptor_class_by_def[field_def.address] = private_constructor(field_def, pool) - else - descriptor - end + file_def = Google::Protobuf::FFI.file_def_by_raw_field_def(field_def) + descriptor_from_file_def(file_def, field_def) end end diff --git a/ruby/lib/google/protobuf/ffi/internal/convert.rb b/ruby/lib/google/protobuf/ffi/internal/convert.rb index aa1dc4a8958e..e3b6d52274bd 100644 --- a/ruby/lib/google/protobuf/ffi/internal/convert.rb +++ b/ruby/lib/google/protobuf/ffi/internal/convert.rb @@ -210,19 +210,6 @@ def convert_upb_to_ruby(message_value, c_type, msg_or_enum_def = nil, arena = ni end end - # @param message_descriptor [Descriptor] Message Descriptor - # @return [::FFI::Pointer] PoolDef pointer - def pool_def_from_message_definition(message_descriptor) - raise RuntimeError.new "Descriptor is nil" if message_descriptor.nil? - file_def = Google::Protobuf::FFI.get_message_file_def message_descriptor - raise RuntimeError.new "FileDef is nil" if file_def.nil? - raise RuntimeError.new "FileDef is null" if file_def.null? - pool_def = Google::Protobuf::FFI.file_def_pool file_def - raise RuntimeError.new "PoolDef is nil" if pool_def.nil? - raise RuntimeError.new "PoolDef is null" if pool_def.null? - pool_def - end - def to_h_internal(msg, message_descriptor) return nil if msg.nil? or msg.null? hash = {} diff --git a/ruby/lib/google/protobuf/ffi/internal/pointer_helper.rb b/ruby/lib/google/protobuf/ffi/internal/pointer_helper.rb new file mode 100644 index 000000000000..5984edd50dd7 --- /dev/null +++ b/ruby/lib/google/protobuf/ffi/internal/pointer_helper.rb @@ -0,0 +1,58 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2023 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +module Google + module Protobuf + module Internal + module PointerHelper + # Utility code to defensively find walk the object graph from a file_def + # to the pool, and either retrieve the wrapper object for the given + # pointer or create one. Assumes that the caller is the wrapper class + # for the given pointer and that it implements `private_constructor`. + def descriptor_from_file_def(file_def, pointer) + raise RuntimeError.new "FileDef is nil" if file_def.nil? + raise RuntimeError.new "FileDef is null" if file_def.null? + pool_def = Google::Protobuf::FFI.file_def_pool file_def + raise RuntimeError.new "PoolDef is nil" if pool_def.nil? + raise RuntimeError.new "PoolDef is null" if pool_def.null? + pool = Google::Protobuf::ObjectCache.get(pool_def) + raise "Cannot find pool in ObjectCache!" if pool.nil? + descriptor = pool.descriptor_class_by_def[pointer.address] + if descriptor.nil? + pool.descriptor_class_by_def[pointer.address] = private_constructor(pointer, pool) + else + descriptor + end + end + end + end + end +end + diff --git a/ruby/lib/google/protobuf/ffi/internal/type_safety.rb b/ruby/lib/google/protobuf/ffi/internal/type_safety.rb index 82903a7bc5a3..1752c2939836 100644 --- a/ruby/lib/google/protobuf/ffi/internal/type_safety.rb +++ b/ruby/lib/google/protobuf/ffi/internal/type_safety.rb @@ -34,7 +34,7 @@ module Google module Protobuf module Internal module TypeSafety - def to_native(value, ctx) + def to_native(value, ctx = nil) if value.kind_of?(self) or value.nil? super else diff --git a/ruby/lib/google/protobuf/ffi/message.rb b/ruby/lib/google/protobuf/ffi/message.rb new file mode 100644 index 000000000000..0c8b209fb332 --- /dev/null +++ b/ruby/lib/google/protobuf/ffi/message.rb @@ -0,0 +1,640 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2023 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +# Decorates Descriptor with the `build_messsage_class` method that defines +# Message classes. +module Google + module Protobuf + class Descriptor + def build_message_class + descriptor = self + Class.new(Google::Protobuf::const_get(:AbstractMessage)) do + @descriptor = descriptor + class << self + attr_accessor :descriptor + private + attr_accessor :oneof_field_names + include ::Google::Protobuf::Internal::Convert + end + + alias original_method_missing method_missing + def method_missing(method_name, *args) + method_missing_internal method_name, *args, mode: :method_missing + end + + def respond_to_missing?(method_name, include_private = false) + method_missing_internal(method_name, mode: :respond_to_missing?) || super + end + + ## + # Public constructor. Automatically allocates from a new Arena. + def self.new(initial_value = nil) + instance = allocate + instance.send(:initialize, initial_value) + instance + end + + def dup + duplicate = self.class.private_constructor(@arena) + mini_table = Google::Protobuf::FFI.get_mini_table(self.class.descriptor) + size = mini_table[:size] + duplicate.instance_variable_get(:@msg).write_string_length(@msg.read_string_length(size), size) + duplicate + end + alias clone dup + + def eql?(other) + return false unless self.class === other + encoding_options = Google::Protobuf::FFI::Upb_Encode_Deterministic | Google::Protobuf::FFI::Upb_Encode_SkipUnknown + temporary_arena = Google::Protobuf::FFI.create_arena + mini_table = Google::Protobuf::FFI.get_mini_table(self.class.descriptor) + size_one = ::FFI::MemoryPointer.new(:size_t, 1) + encoding_one = ::FFI::MemoryPointer.new(:pointer, 1) + encoding_status = Google::Protobuf::FFI.encode_message(@msg, mini_table, encoding_options, temporary_arena, encoding_one.to_ptr, size_one) + raise ParseError.new "Error comparing messages due to #{encoding_status} while encoding LHS of `eql?()`" unless encoding_status == :Ok + + size_two = ::FFI::MemoryPointer.new(:size_t, 1) + encoding_two = ::FFI::MemoryPointer.new(:pointer, 1) + encoding_status = Google::Protobuf::FFI.encode_message(other.instance_variable_get(:@msg), mini_table, encoding_options, temporary_arena, encoding_two.to_ptr, size_two) + raise ParseError.new "Error comparing messages due to #{encoding_status} while encoding RHS of `eql?()`" unless encoding_status == :Ok + + if encoding_one.null? or encoding_two.null? + raise ParseError.new "Error comparing messages" + end + size_one.read(:size_t) == size_two.read(:size_t) and Google::Protobuf::FFI.memcmp(encoding_one.read(:pointer), encoding_two.read(:pointer), size_one.read(:size_t)).zero? + end + alias == eql? + + def hash + encoding_options = Google::Protobuf::FFI::Upb_Encode_Deterministic | Google::Protobuf::FFI::Upb_Encode_SkipUnknown + temporary_arena = Google::Protobuf::FFI.create_arena + mini_table_ptr = Google::Protobuf::FFI.get_mini_table(self.class.descriptor) + size_ptr = ::FFI::MemoryPointer.new(:size_t, 1) + encoding = ::FFI::MemoryPointer.new(:pointer, 1) + encoding_status = Google::Protobuf::FFI.encode_message(@msg, mini_table_ptr, encoding_options, temporary_arena, encoding.to_ptr, size_ptr) + if encoding_status != :Ok or encoding.null? + raise ParseError.new "Error calculating hash" + end + Google::Protobuf::FFI.hash(encoding.read(:pointer), size_ptr.read(:size_t), 0) + end + + def to_h + to_h_internal @msg, self.class.descriptor + end + + ## + # call-seq: + # Message.inspect => string + # + # Returns a human-readable string representing this message. It will be + # formatted as "". Each + # field's value is represented according to its own #inspect method. + def inspect + self.class.inspect_internal @msg + end + + def to_s + self.inspect + end + + ## + # call-seq: + # Message.[](index) => value + # Accesses a field's value by field name. The provided field name + # should be a string. + def [](name) + raise TypeError.new "Expected String for name but got #{name.class}" unless name.is_a? String + index_internal name + end + + ## + # call-seq: + # Message.[]=(index, value) + # Sets a field's value by field name. The provided field name should + # be a string. + # @param name [String] Name of the field to be set + # @param value [Object] Value to set the field to + def []=(name, value) + raise TypeError.new "Expected String for name but got #{name.class}" unless name.is_a? String + index_assign_internal(value, name: name) + end + + ## + # call-seq: + # MessageClass.decode(data, options) => message + # + # Decodes the given data (as a string containing bytes in protocol buffers wire + # format) under the interpretation given by this message class's definition + # and returns a message object with the corresponding field values. + # @param data [String] Binary string in Protobuf wire format to decode + # @param options [Hash] options for the decoder + # @option options [Integer] :recursion_limit Set to maximum decoding depth for message (default is 64) + def self.decode(data, options = {}) + raise ArgumentError.new "Expected hash arguments." unless options.is_a? Hash + raise ArgumentError.new "Expected string for binary protobuf data." unless data.is_a? String + decoding_options = 0 + depth = options[:recursion_limit] + + if depth.is_a? Numeric + decoding_options |= Google::Protobuf::FFI.decode_max_depth(depth.to_i) + end + + message = new + mini_table_ptr = Google::Protobuf::FFI.get_mini_table(message.class.descriptor) + status = Google::Protobuf::FFI.decode_message(data, data.bytesize, message.instance_variable_get(:@msg), mini_table_ptr, nil, decoding_options, message.instance_variable_get(:@arena)) + raise ParseError.new "Error occurred during parsing" unless status == :Ok + message + end + + ## + # call-seq: + # MessageClass.encode(msg, options) => bytes + # + # Encodes the given message object to its serialized form in protocol buffers + # wire format. + # @param options [Hash] options for the encoder + # @option options [Integer] :recursion_limit Set to maximum encoding depth for message (default is 64) + def self.encode(message, options = {}) + raise ArgumentError.new "Message of wrong type." unless message.is_a? self + raise ArgumentError.new "Expected hash arguments." unless options.is_a? Hash + + encoding_options = 0 + depth = options[:recursion_limit] + + if depth.is_a? Numeric + encoding_options |= Google::Protobuf::FFI.decode_max_depth(depth.to_i) + end + + encode_internal(message.instance_variable_get(:@msg), encoding_options) do |encoding, size, _| + if encoding.nil? or encoding.null? + raise RuntimeError.new "Exceeded maximum depth (possibly cycle)" + else + encoding.read_string_length(size).force_encoding("ASCII-8BIT").freeze + end + end + end + + ## + # all-seq: + # MessageClass.decode_json(data, options = {}) => message + # + # Decodes the given data (as a string containing bytes in protocol buffers wire + # format) under the interpretation given by this message class's definition + # and returns a message object with the corresponding field values. + # + # @param options [Hash] options for the decoder + # @option options [Boolean] :ignore_unknown_fields Set true to ignore unknown fields (default is to raise an error) + # @return [Message] + def self.decode_json(data, options = {}) + decoding_options = 0 + unless options.is_a? Hash + if options.respond_to? :to_h + options options.to_h + else + #TODO(jatl) can this error message be improve to include what was received? + raise ArgumentError.new "Expected hash arguments" + end + end + raise ArgumentError.new "Expected string for JSON data." unless data.is_a? String + raise RuntimeError.new "Cannot parse a wrapper directly" if descriptor.send(:wrapper?) + + if options[:ignore_unknown_fields] + decoding_options |= Google::Protobuf::FFI::Upb_JsonDecode_IgnoreUnknown + end + + message = new + pool_def = message.class.descriptor.instance_variable_get(:@descriptor_pool).descriptor_pool + status = Google::Protobuf::FFI::Status.new + unless Google::Protobuf::FFI.json_decode_message(data, data.bytesize, message.instance_variable_get(:@msg), message.class.descriptor, pool_def, decoding_options, message.instance_variable_get(:@arena), status) + raise ParseError.new "Error occurred during parsing: #{Google::Protobuf::FFI.error_message(status)}" + end + message + end + + def self.encode_json(message, options = {}) + encoding_options = 0 + unless options.is_a? Hash + if options.respond_to? :to_h + options = options.to_h + else + #TODO(jatl) can this error message be improve to include what was received? + raise ArgumentError.new "Expected hash arguments" + end + end + + if options[:preserve_proto_fieldnames] + encoding_options |= Google::Protobuf::FFI::Upb_JsonEncode_UseProtoNames + end + if options[:emit_defaults] + encoding_options |= Google::Protobuf::FFI::Upb_JsonEncode_EmitDefaults + end + if options[:format_enums_as_integers] + encoding_options |= Google::Protobuf::FFI::Upb_JsonEncode_FormatEnumsAsIntegers + end + + buffer_size = 1024 + buffer = ::FFI::MemoryPointer.new(:char, buffer_size) + status = Google::Protobuf::FFI::Status.new + msg = message.instance_variable_get(:@msg) + pool_def = message.class.descriptor.instance_variable_get(:@descriptor_pool).descriptor_pool + size = Google::Protobuf::FFI::json_encode_message(msg, message.class.descriptor, pool_def, encoding_options, buffer, buffer_size, status) + unless status[:ok] + raise ParseError.new "Error occurred during encoding: #{Google::Protobuf::FFI.error_message(status)}" + end + + if size >= buffer_size + buffer_size = size + 1 + buffer = ::FFI::MemoryPointer.new(:char, buffer_size) + status.clear + size = Google::Protobuf::FFI::json_encode_message(msg, message.class.descriptor, pool_def, encoding_options, buffer, buffer_size, status) + unless status[:ok] + raise ParseError.new "Error occurred during encoding: #{Google::Protobuf::FFI.error_message(status)}" + end + if size >= buffer_size + raise ParseError.new "Inconsistent JSON encoding sizes - was #{buffer_size - 1}, now #{size}" + end + end + + buffer.read_string_length(size).force_encoding("UTF-8").freeze + end + + private + # Implementation details below are subject to breaking changes without + # warning and are intended for use only within the gem. + + include Google::Protobuf::Internal::Convert + + def self.setup_accessors! + @descriptor.each do |field_descriptor| + field_name = field_descriptor.name + unless instance_methods(true).include?(field_name.to_sym) + #TODO(jatl) - at a high level, dispatching to either + # index_internal or get_field would be logically correct, but slightly slower. + if field_descriptor.map? + define_method(field_name) do + mutable_message_value = Google::Protobuf::FFI.get_mutable_message @msg, field_descriptor, @arena + get_map_field(mutable_message_value[:map], field_descriptor) + end + elsif field_descriptor.repeated? + define_method(field_name) do + mutable_message_value = Google::Protobuf::FFI.get_mutable_message @msg, field_descriptor, @arena + get_repeated_field(mutable_message_value[:array], field_descriptor) + end + elsif field_descriptor.sub_message? + define_method(field_name) do + return nil unless Google::Protobuf::FFI.get_message_has @msg, field_descriptor + mutable_message = Google::Protobuf::FFI.get_mutable_message @msg, field_descriptor, @arena + sub_message = mutable_message[:msg] + sub_message_def = Google::Protobuf::FFI.get_subtype_as_message(field_descriptor) + Descriptor.send(:get_message, sub_message, sub_message_def, @arena) + end + else + c_type = field_descriptor.send(:c_type) + if c_type == :enum + define_method(field_name) do + message_value = Google::Protobuf::FFI.get_message_value @msg, field_descriptor + convert_upb_to_ruby message_value, c_type, Google::Protobuf::FFI.get_subtype_as_enum(field_descriptor) + end + else + define_method(field_name) do + message_value = Google::Protobuf::FFI.get_message_value @msg, field_descriptor + convert_upb_to_ruby message_value, c_type + end + end + end + define_method("#{field_name}=") do |value| + index_assign_internal(value, field_descriptor: field_descriptor) + end + define_method("clear_#{field_name}") do + clear_internal(field_descriptor) + end + if field_descriptor.type == :enum + define_method("#{field_name}_const") do + if field_descriptor.repeated? + return_value = [] + get_field(field_descriptor).send(:each_msg_val) do |msg_val| + return_value << msg_val[:int32_val] + end + return_value + else + message_value = Google::Protobuf::FFI.get_message_value @msg, field_descriptor + message_value[:int32_val] + end + end + end + if !field_descriptor.repeated? and field_descriptor.wrapper? + define_method("#{field_name}_as_value") do + get_field(field_descriptor, unwrap: true) + end + define_method("#{field_name}_as_value=") do |value| + if value.nil? + clear_internal(field_descriptor) + else + index_assign_internal(value, field_descriptor: field_descriptor, wrap: true) + end + end + end + if field_descriptor.has_presence? + define_method("has_#{field_name}?") do + Google::Protobuf::FFI.get_message_has(@msg, field_descriptor) + end + end + end + end + end + + def self.setup_oneof_accessors! + @oneof_field_names = [] + @descriptor.each_oneof do |oneof_descriptor| + self.add_oneof_accessors_for! oneof_descriptor + end + end + def self.add_oneof_accessors_for!(oneof_descriptor) + field_name = oneof_descriptor.name.to_sym + @oneof_field_names << field_name + unless instance_methods(true).include?(field_name) + define_method(field_name) do + field_descriptor = Google::Protobuf::FFI.get_message_which_oneof(@msg, oneof_descriptor) + if field_descriptor.nil? + return + else + return field_descriptor.name.to_sym + end + end + define_method("clear_#{field_name}") do + field_descriptor = Google::Protobuf::FFI.get_message_which_oneof(@msg, oneof_descriptor) + unless field_descriptor.nil? + clear_internal(field_descriptor) + end + end + define_method("has_#{field_name}?") do + !Google::Protobuf::FFI.get_message_which_oneof(@msg, oneof_descriptor).nil? + end + end + end + + setup_accessors! + setup_oneof_accessors! + + def self.private_constructor(arena, msg: nil, initial_value: nil) + instance = allocate + instance.send(:initialize, initial_value, arena, msg) + instance + end + + def self.inspect_field(field_descriptor, c_type, message_value) + if field_descriptor.sub_message? + sub_msg_descriptor = Google::Protobuf::FFI.get_subtype_as_message(field_descriptor) + sub_msg_descriptor.msgclass.send(:inspect_internal, message_value[:msg_val]) + else + convert_upb_to_ruby(message_value, c_type, field_descriptor.subtype).inspect + end + end + + # @param msg [::FFI::Pointer] Pointer to the Message + def self.inspect_internal(msg) + field_output = [] + descriptor.each do |field_descriptor| + next if field_descriptor.has_presence? && !Google::Protobuf::FFI.get_message_has(msg, field_descriptor) + if field_descriptor.map? + # TODO(jatl) Adapted - from map#each_msg_val and map#inspect- can this be refactored to reduce echo without introducing a arena allocation? + message_descriptor = field_descriptor.subtype + key_field_def = Google::Protobuf::FFI.get_field_by_number(message_descriptor, 1) + key_field_type = Google::Protobuf::FFI.get_type(key_field_def) + + value_field_def = Google::Protobuf::FFI.get_field_by_number(message_descriptor, 2) + value_field_type = Google::Protobuf::FFI.get_type(value_field_def) + + message_value = Google::Protobuf::FFI.get_message_value(msg, field_descriptor) + iter = ::FFI::MemoryPointer.new(:size_t, 1) + iter.write(:size_t, Google::Protobuf::FFI::Upb_Map_Begin) + key_value_pairs = [] + while Google::Protobuf::FFI.map_next(message_value[:map_val], iter) do + iter_size_t = iter.read(:size_t) + key_message_value = Google::Protobuf::FFI.map_key(message_value[:map_val], iter_size_t) + value_message_value = Google::Protobuf::FFI.map_value(message_value[:map_val], iter_size_t) + key_string = convert_upb_to_ruby(key_message_value, key_field_type).inspect + value_string = inspect_field(value_field_def, value_field_type, value_message_value) + key_value_pairs << "#{key_string}=>#{value_string}" + end + field_output << "#{field_descriptor.name}: {#{key_value_pairs.join(", ")}}" + elsif field_descriptor.repeated? + # TODO(jatl) Adapted - from repeated_field#each - can this be refactored to reduce echo? + repeated_field_output = [] + message_value = Google::Protobuf::FFI.get_message_value(msg, field_descriptor) + array = message_value[:array_val] + n = array.null? ? 0 : Google::Protobuf::FFI.array_size(array) + 0.upto(n - 1) do |i| + element = Google::Protobuf::FFI.get_msgval_at(array, i) + repeated_field_output << inspect_field(field_descriptor, field_descriptor.send(:c_type), element) + end + field_output << "#{field_descriptor.name}: [#{repeated_field_output.join(", ")}]" + else + message_value = Google::Protobuf::FFI.get_message_value msg, field_descriptor + rendered_value = inspect_field(field_descriptor, field_descriptor.send(:c_type), message_value) + field_output << "#{field_descriptor.name}: #{rendered_value}" + end + end + "<#{name}: #{field_output.join(', ')}>" + end + + def self.deep_copy(msg, arena = nil) + arena ||= Google::Protobuf::FFI.create_arena + encode_internal(msg) do |encoding, size, mini_table_ptr| + message = private_constructor(arena) + if encoding.nil? or encoding.null? or Google::Protobuf::FFI.decode_message(encoding, size, message.instance_variable_get(:@msg), mini_table_ptr, nil, 0, arena) != :Ok + raise ParseError.new "Error occurred copying proto" + end + message + end + end + + def self.encode_internal(msg, encoding_options = 0) + temporary_arena = Google::Protobuf::FFI.create_arena + + mini_table_ptr = Google::Protobuf::FFI.get_mini_table(descriptor) + size_ptr = ::FFI::MemoryPointer.new(:size_t, 1) + pointer_ptr = ::FFI::MemoryPointer.new(:pointer, 1) + encoding_status = Google::Protobuf::FFI.encode_message(msg, mini_table_ptr, encoding_options, temporary_arena, pointer_ptr.to_ptr, size_ptr) + raise "Encoding failed due to #{encoding_status}" unless encoding_status == :Ok + yield pointer_ptr.read(:pointer), size_ptr.read(:size_t), mini_table_ptr + end + + def method_missing_internal(method_name, *args, mode: nil) + raise ArgumentError.new "method_missing_internal called with invalid mode #{mode.inspect}" unless [:respond_to_missing?, :method_missing].include? mode + + #TODO(jatl) not being allowed is not the same thing as not responding, but this is needed to pass tests + if method_name.to_s.end_with? '=' + if self.class.send(:oneof_field_names).include? method_name.to_s[0..-2].to_sym + return false if mode == :respond_to_missing? + raise RuntimeError.new "Oneof accessors are read-only." + end + end + + original_method_missing(method_name, *args) if mode == :method_missing + end + + def clear_internal(field_def) + raise FrozenError.new "can't modify frozen #{self.class}" if frozen? + Google::Protobuf::FFI.clear_message_field(@msg, field_def) + end + + def index_internal(name) + field_descriptor = self.class.descriptor.lookup(name) + get_field field_descriptor unless field_descriptor.nil? + end + + #TODO(jatl) - well known types keeps us on our toes by overloading methods. + # How much of the public API needs to be defended? + def index_assign_internal(value, name: nil, field_descriptor: nil, wrap: false) + raise FrozenError.new "can't modify frozen #{self.class}" if frozen? + if field_descriptor.nil? + field_descriptor = self.class.descriptor.lookup(name) + if field_descriptor.nil? + raise ArgumentError.new "Unknown field: #{name}" + end + end + unless field_descriptor.send :set_value_on_message, value, @msg, @arena, wrap: wrap + raise RuntimeError.new "allocation failed" + end + end + + ## + # @param initial_value [Object] initial value of this Message + # @param arena [Arena] Optional; Arena where this message will be allocated + # @param msg [::FFI::Pointer] Optional; Message to initialize; creates + # one if omitted or nil. + def initialize(initial_value = nil, arena = nil, msg = nil) + @arena = arena || Google::Protobuf::FFI.create_arena + @msg = msg || Google::Protobuf::FFI.new_message_from_def(self.class.descriptor, @arena) + + unless initial_value.nil? + raise ArgumentError.new "Expected hash arguments or message, not #{initial_value.class}" unless initial_value.respond_to? :each + + field_def_ptr = ::FFI::MemoryPointer.new :pointer + oneof_def_ptr = ::FFI::MemoryPointer.new :pointer + + initial_value.each do |key, value| + raise ArgumentError.new "Expected string or symbols as hash keys when initializing proto from hash." unless [String, Symbol].include? key.class + + unless Google::Protobuf::FFI.find_msg_def_by_name self.class.descriptor, key.to_s, key.to_s.bytesize, field_def_ptr, oneof_def_ptr + raise ArgumentError.new "Unknown field name '#{key}' in initialization map entry." + end + raise NotImplementedError.new "Haven't added oneofsupport yet" unless oneof_def_ptr.get_pointer(0).null? + raise NotImplementedError.new "Expected a field def" if field_def_ptr.get_pointer(0).null? + + field_descriptor = FieldDescriptor.from_native field_def_ptr.get_pointer(0) + + next if value.nil? + if field_descriptor.map? + index_assign_internal(Google::Protobuf::Map.send(:construct_for_field, field_descriptor, @arena, value: value), name: key.to_s) + elsif field_descriptor.repeated? + index_assign_internal(RepeatedField.send(:construct_for_field, field_descriptor, @arena, values: value), name: key.to_s) + else + index_assign_internal(value, name: key.to_s) + end + end + end + + # Should always be the last expression of the initializer to avoid + # leaking references to this object before construction is complete. + Google::Protobuf::ObjectCache.add @msg, self + end + + ## + # Gets a field of this message identified by the argument definition. + # + # @param field [FieldDescriptor] Descriptor of the field to get + def get_field(field, unwrap: false) + if field.map? + mutable_message_value = Google::Protobuf::FFI.get_mutable_message @msg, field, @arena + get_map_field(mutable_message_value[:map], field) + elsif field.repeated? + mutable_message_value = Google::Protobuf::FFI.get_mutable_message @msg, field, @arena + get_repeated_field(mutable_message_value[:array], field) + elsif field.sub_message? + return nil unless Google::Protobuf::FFI.get_message_has @msg, field + sub_message_def = Google::Protobuf::FFI.get_subtype_as_message(field) + if unwrap + if field.has?(self) + wrapper_message_value = Google::Protobuf::FFI.get_message_value @msg, field + fields = Google::Protobuf::FFI.field_count(sub_message_def) + raise "Sub message has #{fields} fields! Expected exactly 1." unless fields == 1 + value_field_def = Google::Protobuf::FFI.get_field_by_number sub_message_def, 1 + message_value = Google::Protobuf::FFI.get_message_value wrapper_message_value[:msg_val], value_field_def + convert_upb_to_ruby message_value, Google::Protobuf::FFI.get_c_type(value_field_def) + else + nil + end + else + mutable_message = Google::Protobuf::FFI.get_mutable_message @msg, field, @arena + sub_message = mutable_message[:msg] + Descriptor.send(:get_message, sub_message, sub_message_def, @arena) + end + else + c_type = field.send(:c_type) + message_value = Google::Protobuf::FFI.get_message_value @msg, field + if c_type == :enum + convert_upb_to_ruby message_value, c_type, Google::Protobuf::FFI.get_subtype_as_enum(field) + else + convert_upb_to_ruby message_value, c_type + end + end + end + + ## + # @param array [::FFI::Pointer] Pointer to the Array + # @param field [Google::Protobuf::FieldDescriptor] Type of the repeated field + def get_repeated_field(array, field) + return nil if array.nil? or array.null? + repeated_field = ObjectCache.get(array) + if repeated_field.nil? + repeated_field = RepeatedField.send(:construct_for_field, field, @arena, array: array) + end + repeated_field + end + + ## + # @param map [::FFI::Pointer] Pointer to the Map + # @param field [Google::Protobuf::FieldDescriptor] Type of the map field + def get_map_field(map, field) + return nil if map.nil? or map.null? + map_field = ObjectCache.get(map) + if map_field.nil? + map_field = Google::Protobuf::Map.send(:construct_for_field, field, @arena, map: map) + end + map_field + end + end + end + end + end +end diff --git a/ruby/lib/google/protobuf/ffi/oneof_descriptor.rb b/ruby/lib/google/protobuf/ffi/oneof_descriptor.rb index 366c5e8030a1..5a622f98b05c 100644 --- a/ruby/lib/google/protobuf/ffi/oneof_descriptor.rb +++ b/ruby/lib/google/protobuf/ffi/oneof_descriptor.rb @@ -40,6 +40,7 @@ class OneofDescriptor class << self prepend Google::Protobuf::Internal::TypeSafety + include Google::Protobuf::Internal::PointerHelper # @param value [OneofDescriptor] FieldDescriptor to convert to an FFI native type # @param _ [Object] Unused @@ -47,7 +48,6 @@ def to_native(value, _ = nil) oneof_def_ptr = value.instance_variable_get(:@oneof_def) warn "Underlying oneof_def was nil!" if oneof_def_ptr.nil? raise "Underlying oneof_def was null!" if !oneof_def_ptr.nil? and oneof_def_ptr.null? - # || ::FFI::Pointer::NULL oneof_def_ptr end @@ -56,28 +56,10 @@ def to_native(value, _ = nil) # @param _ [Object] Unused def from_native(oneof_def, _ = nil) return nil if oneof_def.nil? or oneof_def.null? - # Calling upb_OneofDef_ContainingType(oneof_def) would create a cyclic - # dependency because either 1) we'd have to define the method to accept - # an untyped pointer or 2) FFI would complain about passing a - # FFI::Pointer instance instead of a OneofDescriptor. Instead, directly - # read the top of the OneDef structure and extract the MsgDef*. - oneof_def_struct = Google::Protobuf::FFI::Upb_OneofDef.new(oneof_def) - message_descriptor = Descriptor.from_native(oneof_def_struct[:parent]) + message_descriptor = Google::Protobuf::FFI.get_oneof_containing_type oneof_def raise RuntimeError.new "Message Descriptor is nil" if message_descriptor.nil? - file_def = Google::Protobuf::FFI.get_message_file_def message_descriptor - raise RuntimeError.new "FileDef is nil" if file_def.nil? - raise RuntimeError.new "FileDef is null" if file_def.null? - pool_def = Google::Protobuf::FFI.file_def_pool file_def - raise RuntimeError.new "PoolDef is nil" if pool_def.nil? - raise RuntimeError.new "PoolDef is null" if pool_def.null? - pool = Google::Protobuf::ObjectCache.get(pool_def) - raise "Cannot find pool in ObjectCache!" if pool.nil? - descriptor = pool.descriptor_class_by_def[oneof_def.address] - if descriptor.nil? - pool.descriptor_class_by_def[oneof_def.address] = private_constructor(oneof_def, pool) - else - descriptor - end + file_def = Google::Protobuf::FFI.get_message_file_def message_descriptor.to_native + descriptor_from_file_def(file_def, oneof_def) end end diff --git a/ruby/lib/google/protobuf_ffi.rb b/ruby/lib/google/protobuf_ffi.rb index 068d4046bef2..99f440befff1 100644 --- a/ruby/lib/google/protobuf_ffi.rb +++ b/ruby/lib/google/protobuf_ffi.rb @@ -30,9 +30,11 @@ require 'ffi' require 'google/protobuf/ffi/internal/type_safety' +require 'google/protobuf/ffi/internal/pointer_helper' require 'google/protobuf/ffi/internal/arena' require 'google/protobuf/ffi/internal/convert' require 'google/protobuf/ffi/descriptor' +require 'google/protobuf/ffi/message' require 'google/protobuf/ffi/enum_descriptor' require 'google/protobuf/ffi/field_descriptor' require 'google/protobuf/ffi/oneof_descriptor' From 2fd1eac488d1c38badadc6ca13dd320dd1fe7a00 Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Fri, 7 Jul 2023 16:29:08 -0400 Subject: [PATCH 43/54] Cleanup debug print. --- ruby/lib/google/protobuf/ffi/internal/convert.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/ruby/lib/google/protobuf/ffi/internal/convert.rb b/ruby/lib/google/protobuf/ffi/internal/convert.rb index e3b6d52274bd..414598648a1e 100644 --- a/ruby/lib/google/protobuf/ffi/internal/convert.rb +++ b/ruby/lib/google/protobuf/ffi/internal/convert.rb @@ -277,7 +277,6 @@ def map_create_hash(map_ptr, field_descriptor) def repeated_field_create_array(array, field_descriptor, type) return_value = [] - # puts "JATL - about to call FFI method array_size(#{array}); stacktrace:\n#{caller.join("\n")}" n = (array.nil? || array.null?) ? 0 : Google::Protobuf::FFI.array_size(array) 0.upto(n - 1) do |i| message_value = Google::Protobuf::FFI.get_msgval_at(array, i) From 3d1d1fece3343e8bbb677f11e4c82490867c80d6 Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Mon, 10 Jul 2023 14:40:52 -0400 Subject: [PATCH 44/54] Revert changes to ruby-upb.h. Remove uses of _upb_Hash directly from Ruby. --- ruby/ext/google/protobuf_c/glue.c | 222 ++++++++++++++++++++++++ ruby/ext/google/protobuf_c/ruby-upb.h | 128 +++++++------- ruby/lib/google/protobuf/ffi/ffi.rb | 129 +++++++------- ruby/lib/google/protobuf/ffi/message.rb | 2 +- 4 files changed, 350 insertions(+), 131 deletions(-) diff --git a/ruby/ext/google/protobuf_c/glue.c b/ruby/ext/google/protobuf_c/glue.c index 16d53130d151..f1005b6bc994 100644 --- a/ruby/ext/google/protobuf_c/glue.c +++ b/ruby/ext/google/protobuf_c/glue.c @@ -44,3 +44,225 @@ google_protobuf_FileDescriptorProto* FileDescriptorProto_parse (const char* seri serialized_file_proto, length, arena); } + +bool API_PENDING_upb_MapIterator_Next(const upb_Map* map, size_t* iter) { + return upb_MapIterator_Next(map, iter); +} +bool API_PENDING_upb_MapIterator_Done(const upb_Map* map, size_t iter) { + return upb_MapIterator_Done(map, iter); +} +upb_MessageValue API_PENDING_upb_MapIterator_Key(const upb_Map* map, size_t iter) { + return upb_MapIterator_Key(map, iter); +} +upb_MessageValue API_PENDING_upb_MapIterator_Value(const upb_Map* map, size_t iter) { + return upb_MapIterator_Value(map, iter); +} +upb_EncodeStatus API_PENDING_upb_Encode(const void* msg, const upb_MiniTable* l, + int options, upb_Arena* arena, char** buf, + size_t* size) { + return upb_Encode(msg, l, options, arena, buf, size); +} + +void API_PENDING_upb_DefPool_Free(upb_DefPool* s) { + return upb_DefPool_Free(s); +} + +upb_DefPool* API_PENDING_upb_DefPool_New(void) { + return upb_DefPool_New(); +} + +const upb_MessageDef* API_PENDING_upb_DefPool_FindMessageByName(const upb_DefPool* s, + const char* sym) { + return upb_DefPool_FindMessageByName(s, sym); +} + +const upb_EnumDef* API_PENDING_upb_DefPool_FindEnumByName(const upb_DefPool* s, + const char* sym) { + return upb_DefPool_FindEnumByName(s, sym); +} +const upb_FileDef* API_PENDING_upb_DefPool_AddFile(upb_DefPool* s, + void * + file_proto, + upb_Status* status) { + return upb_DefPool_AddFile(s, file_proto, status); +} +const upb_FileDef* API_PENDING_upb_EnumDef_File(const upb_EnumDef* e) { + return upb_EnumDef_File(e); +} +const upb_EnumValueDef* API_PENDING_upb_EnumDef_FindValueByNameWithSize( + const upb_EnumDef* e, const char* name, size_t size) { + return upb_EnumDef_FindValueByNameWithSize(e, name, size); +} +const upb_EnumValueDef* API_PENDING_upb_EnumDef_FindValueByNumber(const upb_EnumDef* e, + int32_t num) { + return upb_EnumDef_FindValueByNumber(e, num); +} +const char* API_PENDING_upb_EnumDef_FullName(const upb_EnumDef* e) { + return upb_EnumDef_FullName(e); +} +const upb_EnumValueDef* API_PENDING_upb_EnumDef_Value(const upb_EnumDef* e, int i) { + return upb_EnumDef_Value(e, i); +} +int API_PENDING_upb_EnumDef_ValueCount(const upb_EnumDef* e) { + return upb_EnumDef_ValueCount(e); +} +const char* API_PENDING_upb_EnumValueDef_Name(const upb_EnumValueDef* v) { + return upb_EnumValueDef_Name(v); +} +int32_t API_PENDING_upb_EnumValueDef_Number(const upb_EnumValueDef* v) { + return upb_EnumValueDef_Number(v); +} +const upb_MessageDef* API_PENDING_upb_FieldDef_ContainingType(const upb_FieldDef* f) { + return upb_FieldDef_ContainingType(f); +} +upb_CType API_PENDING_upb_FieldDef_CType(const upb_FieldDef* f) { + return upb_FieldDef_CType(f); +} +upb_MessageValue API_PENDING_upb_FieldDef_Default(const upb_FieldDef* f) { + return upb_FieldDef_Default(f); +} +const upb_EnumDef* API_PENDING_upb_FieldDef_EnumSubDef(const upb_FieldDef* f) { + return upb_FieldDef_EnumSubDef(f); +} +const upb_FileDef* API_PENDING_upb_FieldDef_File(const upb_FieldDef* f) { + return upb_FieldDef_File(f); +} +bool API_PENDING_upb_FieldDef_HasPresence(const upb_FieldDef* f) { + return upb_FieldDef_HasPresence(f); +} +bool API_PENDING_upb_FieldDef_IsMap(const upb_FieldDef* f) { + return upb_FieldDef_IsMap(f); +} +bool API_PENDING_upb_FieldDef_IsRepeated(const upb_FieldDef* f) { + return upb_FieldDef_IsRepeated(f); +} +bool API_PENDING_upb_FieldDef_IsSubMessage(const upb_FieldDef* f) { + return upb_FieldDef_IsSubMessage(f); +} +const char* API_PENDING_upb_FieldDef_JsonName(const upb_FieldDef* f) { + return upb_FieldDef_JsonName(f); +} +upb_Label API_PENDING_upb_FieldDef_Label(const upb_FieldDef* f) { + return upb_FieldDef_Label(f); +} +const upb_MessageDef* API_PENDING_upb_FieldDef_MessageSubDef(const upb_FieldDef* f) { + return upb_FieldDef_MessageSubDef(f); +} +const char* API_PENDING_upb_FieldDef_Name(const upb_FieldDef* f) { + return upb_FieldDef_Name(f); +} +uint32_t API_PENDING_upb_FieldDef_Number(const upb_FieldDef* f) { + return upb_FieldDef_Number(f); +} +const upb_OneofDef* API_PENDING_upb_FieldDef_RealContainingOneof(const upb_FieldDef* f) { + return upb_FieldDef_RealContainingOneof(f); +} +upb_FieldType API_PENDING_upb_FieldDef_Type(const upb_FieldDef* f) { + return upb_FieldDef_Type(f); +} +const char* API_PENDING_upb_FileDef_Name(const upb_FileDef* f) { + return upb_FileDef_Name(f); +} +const upb_DefPool* API_PENDING_upb_FileDef_Pool(const upb_FileDef* f) { + return upb_FileDef_Pool(f); +} +upb_Syntax API_PENDING_upb_FileDef_Syntax(const upb_FileDef* f) { + return upb_FileDef_Syntax(f); +} +const upb_FieldDef* API_PENDING_upb_MessageDef_Field(const upb_MessageDef* m, int i) { + return upb_MessageDef_Field(m, i); +} +int API_PENDING_upb_MessageDef_FieldCount(const upb_MessageDef* m) { + return upb_MessageDef_FieldCount(m); +} +const upb_FileDef* API_PENDING_upb_MessageDef_File(const upb_MessageDef* m) { + return upb_MessageDef_File(m); +} +bool API_PENDING_upb_MessageDef_FindByNameWithSize(const upb_MessageDef* m, + const char* name, size_t size, + const upb_FieldDef** f, + const upb_OneofDef** o) { + return upb_MessageDef_FindByNameWithSize(m, name, size, f, o); +} +const upb_FieldDef* API_PENDING_upb_MessageDef_FindFieldByNameWithSize( + const upb_MessageDef* m, const char* name, size_t size) { + return upb_MessageDef_FindFieldByNameWithSize(m, name, size); +} +const upb_FieldDef* API_PENDING_upb_MessageDef_FindFieldByNumber(const upb_MessageDef* m, + uint32_t i) { + return upb_MessageDef_FindFieldByNumber(m, i); +} +const upb_OneofDef* API_PENDING_upb_MessageDef_FindOneofByNameWithSize( + const upb_MessageDef* m, const char* name, size_t size) { + return upb_MessageDef_FindOneofByNameWithSize(m, name, size); +} +const char* API_PENDING_upb_MessageDef_FullName(const upb_MessageDef* m) { + + return upb_MessageDef_FullName(m); +} +const upb_MiniTable* API_PENDING_upb_MessageDef_MiniTable(const upb_MessageDef* m) { + return upb_MessageDef_MiniTable(m); +} +const upb_OneofDef* API_PENDING_upb_MessageDef_Oneof(const upb_MessageDef* m, int i) { + return upb_MessageDef_Oneof(m, i); +} +int API_PENDING_upb_MessageDef_OneofCount(const upb_MessageDef* m) { + return upb_MessageDef_OneofCount(m); +} +upb_Syntax API_PENDING_upb_MessageDef_Syntax(const upb_MessageDef* m) { + return upb_MessageDef_Syntax(m); +} +upb_WellKnown API_PENDING_upb_MessageDef_WellKnownType(const upb_MessageDef* m) { + return upb_MessageDef_WellKnownType(m); +} +const upb_MessageDef* API_PENDING_upb_OneofDef_ContainingType(const upb_OneofDef* o) { + return upb_OneofDef_ContainingType(o); +} +const upb_FieldDef* API_PENDING_upb_OneofDef_Field(const upb_OneofDef* o, int i) { + return upb_OneofDef_Field(o, i); +} +int API_PENDING_upb_OneofDef_FieldCount(const upb_OneofDef* o) { + return upb_OneofDef_FieldCount(o); +} +const char* API_PENDING_upb_OneofDef_Name(const upb_OneofDef* o) { + return upb_OneofDef_Name(o); +} +bool API_PENDING_upb_JsonDecode(const char* buf, size_t size, upb_Message* msg, + const upb_MessageDef* m, const upb_DefPool* symtab, + int options, upb_Arena* arena, upb_Status* status) { + return upb_JsonDecode(buf, size, msg, m, symtab, options, arena, status); +} +upb_MutableMessageValue API_PENDING_upb_Message_Mutable(upb_Message* msg, + const upb_FieldDef* f, + upb_Arena* a) { + return upb_Message_Mutable(msg, f, a); +} +const upb_FieldDef* API_PENDING_upb_Message_WhichOneof(const upb_Message* msg, + const upb_OneofDef* o) { + return upb_Message_WhichOneof(msg, o); +} +void API_PENDING_upb_Message_ClearFieldByDef(upb_Message* msg, const upb_FieldDef* f) { + return upb_Message_ClearFieldByDef(msg, f); +} + +bool API_PENDING_upb_Message_HasFieldByDef(const upb_Message* msg, const upb_FieldDef* f) { + return upb_Message_HasFieldByDef(msg, f); +} + +upb_MessageValue API_PENDING_upb_Message_GetFieldByDef(const upb_Message* msg, + const upb_FieldDef* f) { + return upb_Message_GetFieldByDef(msg, f); +} +bool API_PENDING_upb_Message_SetFieldByDef(upb_Message* msg, const upb_FieldDef* f, + upb_MessageValue val, upb_Arena* a) { + return upb_Message_SetFieldByDef(msg, f, val, a); +} +bool API_PENDING_upb_Message_DiscardUnknown(upb_Message* msg, const upb_MessageDef* m, + int maxdepth) { + return upb_Message_DiscardUnknown(msg, m, maxdepth); +} +size_t API_PENDING_upb_JsonEncode(const upb_Message* msg, const upb_MessageDef* m, + const upb_DefPool* ext_pool, int options, char* buf, + size_t size, upb_Status* status) { + return upb_JsonEncode(msg, m, ext_pool, options, buf, size, status); +} diff --git a/ruby/ext/google/protobuf_c/ruby-upb.h b/ruby/ext/google/protobuf_c/ruby-upb.h index 5f55c079f8e8..8606a4b58d5c 100755 --- a/ruby/ext/google/protobuf_c/ruby-upb.h +++ b/ruby/ext/google/protobuf_c/ruby-upb.h @@ -1035,16 +1035,16 @@ UPB_API void upb_Map_SetEntryValue(upb_Map* map, size_t iter, */ // Advances to the next entry. Returns false if no more entries are present. -UPB_API_INLINE bool upb_MapIterator_Next(const upb_Map* map, size_t* iter); +bool upb_MapIterator_Next(const upb_Map* map, size_t* iter); // Returns true if the iterator still points to a valid entry, or false if the // iterator is past the last element. It is an error to call this function with // kUpb_Map_Begin (you must call next() at least once first). -UPB_API_INLINE bool upb_MapIterator_Done(const upb_Map* map, size_t iter); +bool upb_MapIterator_Done(const upb_Map* map, size_t iter); // Returns the key and value for this entry of the map. -UPB_API_INLINE upb_MessageValue upb_MapIterator_Key(const upb_Map* map, size_t iter); -UPB_API_INLINE upb_MessageValue upb_MapIterator_Value(const upb_Map* map, size_t iter); +upb_MessageValue upb_MapIterator_Key(const upb_Map* map, size_t iter); +upb_MessageValue upb_MapIterator_Value(const upb_Map* map, size_t iter); #ifdef __cplusplus } /* extern "C" */ @@ -1224,7 +1224,7 @@ UPB_INLINE size_t upb_table_size(const upb_table* t) { UPB_INLINE bool upb_tabent_isempty(const upb_tabent* e) { return e->key == 0; } -UPB_API_INLINE uint32_t _upb_Hash(const void* p, size_t n, uint64_t seed); +uint32_t _upb_Hash(const void* p, size_t n, uint64_t seed); #ifdef __cplusplus } /* extern "C" */ @@ -3419,7 +3419,7 @@ UPB_INLINE int upb_Encode_LimitDepth(uint32_t encode_options, uint32_t limit) { return upb_EncodeOptions_MaxDepth(max_depth) | (encode_options & 0xffff); } -UPB_API_INLINE upb_EncodeStatus upb_Encode(const void* msg, const upb_MiniTable* l, +upb_EncodeStatus upb_Encode(const void* msg, const upb_MiniTable* l, int options, upb_Arena* arena, char** buf, size_t* size); @@ -10207,17 +10207,17 @@ const void* _upb_DefType_Unpack(upb_value v, upb_deftype_t type); extern "C" { #endif -UPB_API_INLINE void upb_DefPool_Free(upb_DefPool* s); +void upb_DefPool_Free(upb_DefPool* s); -UPB_API_INLINE upb_DefPool* upb_DefPool_New(void); +upb_DefPool* upb_DefPool_New(void); -UPB_API_INLINE const upb_MessageDef* upb_DefPool_FindMessageByName(const upb_DefPool* s, +const upb_MessageDef* upb_DefPool_FindMessageByName(const upb_DefPool* s, const char* sym); const upb_MessageDef* upb_DefPool_FindMessageByNameWithSize( const upb_DefPool* s, const char* sym, size_t len); -UPB_API_INLINE const upb_EnumDef* upb_DefPool_FindEnumByName(const upb_DefPool* s, +const upb_EnumDef* upb_DefPool_FindEnumByName(const upb_DefPool* s, const char* sym); const upb_EnumValueDef* upb_DefPool_FindEnumByNameval(const upb_DefPool* s, @@ -10252,7 +10252,7 @@ const upb_ServiceDef* upb_DefPool_FindServiceByNameWithSize( const upb_FileDef* upb_DefPool_FindFileContainingSymbol(const upb_DefPool* s, const char* name); -UPB_API const upb_FileDef* upb_DefPool_AddFile(upb_DefPool* s, +const upb_FileDef* upb_DefPool_AddFile(upb_DefPool* s, const UPB_DESC(FileDescriptorProto) * file_proto, upb_Status* status); @@ -10286,14 +10286,14 @@ extern "C" { bool upb_EnumDef_CheckNumber(const upb_EnumDef* e, int32_t num); const upb_MessageDef* upb_EnumDef_ContainingType(const upb_EnumDef* e); int32_t upb_EnumDef_Default(const upb_EnumDef* e); -UPB_API_INLINE const upb_FileDef* upb_EnumDef_File(const upb_EnumDef* e); +const upb_FileDef* upb_EnumDef_File(const upb_EnumDef* e); const upb_EnumValueDef* upb_EnumDef_FindValueByName(const upb_EnumDef* e, const char* name); -UPB_API_INLINE const upb_EnumValueDef* upb_EnumDef_FindValueByNameWithSize( +const upb_EnumValueDef* upb_EnumDef_FindValueByNameWithSize( const upb_EnumDef* e, const char* name, size_t size); -UPB_API_INLINE const upb_EnumValueDef* upb_EnumDef_FindValueByNumber(const upb_EnumDef* e, +const upb_EnumValueDef* upb_EnumDef_FindValueByNumber(const upb_EnumDef* e, int32_t num); -UPB_API_INLINE const char* upb_EnumDef_FullName(const upb_EnumDef* e); +const char* upb_EnumDef_FullName(const upb_EnumDef* e); bool upb_EnumDef_HasOptions(const upb_EnumDef* e); bool upb_EnumDef_IsClosed(const upb_EnumDef* e); @@ -10311,8 +10311,8 @@ const upb_EnumReservedRange* upb_EnumDef_ReservedRange(const upb_EnumDef* e, int i); int upb_EnumDef_ReservedRangeCount(const upb_EnumDef* e); -UPB_API_INLINE const upb_EnumValueDef* upb_EnumDef_Value(const upb_EnumDef* e, int i); -UPB_API_INLINE int upb_EnumDef_ValueCount(const upb_EnumDef* e); +const upb_EnumValueDef* upb_EnumDef_Value(const upb_EnumDef* e, int i); +int upb_EnumDef_ValueCount(const upb_EnumDef* e); #ifdef __cplusplus } /* extern "C" */ @@ -10337,8 +10337,8 @@ const upb_EnumDef* upb_EnumValueDef_Enum(const upb_EnumValueDef* v); const char* upb_EnumValueDef_FullName(const upb_EnumValueDef* v); bool upb_EnumValueDef_HasOptions(const upb_EnumValueDef* v); uint32_t upb_EnumValueDef_Index(const upb_EnumValueDef* v); -UPB_API_INLINE const char* upb_EnumValueDef_Name(const upb_EnumValueDef* v); -UPB_API_INLINE int32_t upb_EnumValueDef_Number(const upb_EnumValueDef* v); +const char* upb_EnumValueDef_Name(const upb_EnumValueDef* v); +int32_t upb_EnumValueDef_Number(const upb_EnumValueDef* v); const UPB_DESC(EnumValueOptions) * upb_EnumValueDef_Options(const upb_EnumValueDef* v); @@ -10392,42 +10392,42 @@ extern "C" { #endif const upb_OneofDef* upb_FieldDef_ContainingOneof(const upb_FieldDef* f); -UPB_API_INLINE const upb_MessageDef* upb_FieldDef_ContainingType(const upb_FieldDef* f); -UPB_API_INLINE upb_CType upb_FieldDef_CType(const upb_FieldDef* f); -UPB_API_INLINE upb_MessageValue upb_FieldDef_Default(const upb_FieldDef* f); -UPB_API_INLINE const upb_EnumDef* upb_FieldDef_EnumSubDef(const upb_FieldDef* f); +const upb_MessageDef* upb_FieldDef_ContainingType(const upb_FieldDef* f); +upb_CType upb_FieldDef_CType(const upb_FieldDef* f); +upb_MessageValue upb_FieldDef_Default(const upb_FieldDef* f); +const upb_EnumDef* upb_FieldDef_EnumSubDef(const upb_FieldDef* f); const upb_MessageDef* upb_FieldDef_ExtensionScope(const upb_FieldDef* f); -UPB_API_INLINE const upb_FileDef* upb_FieldDef_File(const upb_FieldDef* f); +const upb_FileDef* upb_FieldDef_File(const upb_FieldDef* f); const char* upb_FieldDef_FullName(const upb_FieldDef* f); bool upb_FieldDef_HasDefault(const upb_FieldDef* f); bool upb_FieldDef_HasJsonName(const upb_FieldDef* f); bool upb_FieldDef_HasOptions(const upb_FieldDef* f); -UPB_API_INLINE bool upb_FieldDef_HasPresence(const upb_FieldDef* f); +bool upb_FieldDef_HasPresence(const upb_FieldDef* f); bool upb_FieldDef_HasSubDef(const upb_FieldDef* f); uint32_t upb_FieldDef_Index(const upb_FieldDef* f); bool upb_FieldDef_IsExtension(const upb_FieldDef* f); -UPB_API_INLINE bool upb_FieldDef_IsMap(const upb_FieldDef* f); +bool upb_FieldDef_IsMap(const upb_FieldDef* f); bool upb_FieldDef_IsOptional(const upb_FieldDef* f); bool upb_FieldDef_IsPacked(const upb_FieldDef* f); bool upb_FieldDef_IsPrimitive(const upb_FieldDef* f); -UPB_API_INLINE bool upb_FieldDef_IsRepeated(const upb_FieldDef* f); +bool upb_FieldDef_IsRepeated(const upb_FieldDef* f); bool upb_FieldDef_IsRequired(const upb_FieldDef* f); bool upb_FieldDef_IsString(const upb_FieldDef* f); -UPB_API_INLINE bool upb_FieldDef_IsSubMessage(const upb_FieldDef* f); -UPB_API_INLINE const char* upb_FieldDef_JsonName(const upb_FieldDef* f); -UPB_API_INLINE upb_Label upb_FieldDef_Label(const upb_FieldDef* f); -UPB_API_INLINE const upb_MessageDef* upb_FieldDef_MessageSubDef(const upb_FieldDef* f); +bool upb_FieldDef_IsSubMessage(const upb_FieldDef* f); +const char* upb_FieldDef_JsonName(const upb_FieldDef* f); +upb_Label upb_FieldDef_Label(const upb_FieldDef* f); +const upb_MessageDef* upb_FieldDef_MessageSubDef(const upb_FieldDef* f); // Creates a mini descriptor string for a field, returns true on success. bool upb_FieldDef_MiniDescriptorEncode(const upb_FieldDef* f, upb_Arena* a, upb_StringView* out); const upb_MiniTableField* upb_FieldDef_MiniTable(const upb_FieldDef* f); -UPB_API_INLINE const char* upb_FieldDef_Name(const upb_FieldDef* f); -UPB_API_INLINE uint32_t upb_FieldDef_Number(const upb_FieldDef* f); +const char* upb_FieldDef_Name(const upb_FieldDef* f); +uint32_t upb_FieldDef_Number(const upb_FieldDef* f); const UPB_DESC(FieldOptions) * upb_FieldDef_Options(const upb_FieldDef* f); -UPB_API_INLINE const upb_OneofDef* upb_FieldDef_RealContainingOneof(const upb_FieldDef* f); -UPB_API_INLINE upb_FieldType upb_FieldDef_Type(const upb_FieldDef* f); +const upb_OneofDef* upb_FieldDef_RealContainingOneof(const upb_FieldDef* f); +upb_FieldType upb_FieldDef_Type(const upb_FieldDef* f); #ifdef __cplusplus } /* extern "C" */ @@ -10451,11 +10451,11 @@ extern "C" { const upb_FileDef* upb_FileDef_Dependency(const upb_FileDef* f, int i); int upb_FileDef_DependencyCount(const upb_FileDef* f); bool upb_FileDef_HasOptions(const upb_FileDef* f); -UPB_API_INLINE const char* upb_FileDef_Name(const upb_FileDef* f); +const char* upb_FileDef_Name(const upb_FileDef* f); const UPB_DESC(FileOptions) * upb_FileDef_Options(const upb_FileDef* f); const char* upb_FileDef_Package(const upb_FileDef* f); const char* upb_FileDef_Edition(const upb_FileDef* f); -UPB_API_INLINE const upb_DefPool* upb_FileDef_Pool(const upb_FileDef* f); +const upb_DefPool* upb_FileDef_Pool(const upb_FileDef* f); const upb_FileDef* upb_FileDef_PublicDependency(const upb_FileDef* f, int i); int upb_FileDef_PublicDependencyCount(const upb_FileDef* f); @@ -10463,7 +10463,7 @@ int upb_FileDef_PublicDependencyCount(const upb_FileDef* f); const upb_ServiceDef* upb_FileDef_Service(const upb_FileDef* f, int i); int upb_FileDef_ServiceCount(const upb_FileDef* f); -UPB_API_INLINE upb_Syntax upb_FileDef_Syntax(const upb_FileDef* f); +upb_Syntax upb_FileDef_Syntax(const upb_FileDef* f); const upb_EnumDef* upb_FileDef_TopLevelEnum(const upb_FileDef* f, int i); int upb_FileDef_TopLevelEnumCount(const upb_FileDef* f); @@ -10545,10 +10545,10 @@ const upb_ExtensionRange* upb_MessageDef_ExtensionRange(const upb_MessageDef* m, int i); int upb_MessageDef_ExtensionRangeCount(const upb_MessageDef* m); -UPB_API_INLINE const upb_FieldDef* upb_MessageDef_Field(const upb_MessageDef* m, int i); -UPB_API_INLINE int upb_MessageDef_FieldCount(const upb_MessageDef* m); +const upb_FieldDef* upb_MessageDef_Field(const upb_MessageDef* m, int i); +int upb_MessageDef_FieldCount(const upb_MessageDef* m); -UPB_API_INLINE const upb_FileDef* upb_MessageDef_File(const upb_MessageDef* m); +const upb_FileDef* upb_MessageDef_File(const upb_MessageDef* m); // Returns a field by either JSON name or regular proto name. const upb_FieldDef* upb_MessageDef_FindByJsonNameWithSize( @@ -10561,7 +10561,7 @@ UPB_INLINE const upb_FieldDef* upb_MessageDef_FindByJsonName( // Lookup of either field or oneof by name. Returns whether either was found. // If the return is true, then the found def will be set, and the non-found // one set to NULL. -UPB_API_INLINE bool upb_MessageDef_FindByNameWithSize(const upb_MessageDef* m, +bool upb_MessageDef_FindByNameWithSize(const upb_MessageDef* m, const char* name, size_t size, const upb_FieldDef** f, const upb_OneofDef** o); @@ -10574,15 +10574,15 @@ UPB_INLINE bool upb_MessageDef_FindByName(const upb_MessageDef* m, const upb_FieldDef* upb_MessageDef_FindFieldByName(const upb_MessageDef* m, const char* name); -UPB_API_INLINE const upb_FieldDef* upb_MessageDef_FindFieldByNameWithSize( +const upb_FieldDef* upb_MessageDef_FindFieldByNameWithSize( const upb_MessageDef* m, const char* name, size_t size); -UPB_API_INLINE const upb_FieldDef* upb_MessageDef_FindFieldByNumber(const upb_MessageDef* m, +const upb_FieldDef* upb_MessageDef_FindFieldByNumber(const upb_MessageDef* m, uint32_t i); const upb_OneofDef* upb_MessageDef_FindOneofByName(const upb_MessageDef* m, const char* name); -UPB_API_INLINE const upb_OneofDef* upb_MessageDef_FindOneofByNameWithSize( +const upb_OneofDef* upb_MessageDef_FindOneofByNameWithSize( const upb_MessageDef* m, const char* name, size_t size); -UPB_API_INLINE const char* upb_MessageDef_FullName(const upb_MessageDef* m); +const char* upb_MessageDef_FullName(const upb_MessageDef* m); bool upb_MessageDef_HasOptions(const upb_MessageDef* m); bool upb_MessageDef_IsMapEntry(const upb_MessageDef* m); bool upb_MessageDef_IsMessageSet(const upb_MessageDef* m); @@ -10591,7 +10591,7 @@ bool upb_MessageDef_IsMessageSet(const upb_MessageDef* m); bool upb_MessageDef_MiniDescriptorEncode(const upb_MessageDef* m, upb_Arena* a, upb_StringView* out); -UPB_API_INLINE const upb_MiniTable* upb_MessageDef_MiniTable(const upb_MessageDef* m); +const upb_MiniTable* upb_MessageDef_MiniTable(const upb_MessageDef* m); const char* upb_MessageDef_Name(const upb_MessageDef* m); const upb_EnumDef* upb_MessageDef_NestedEnum(const upb_MessageDef* m, int i); @@ -10604,8 +10604,8 @@ int upb_MessageDef_NestedEnumCount(const upb_MessageDef* m); int upb_MessageDef_NestedExtensionCount(const upb_MessageDef* m); int upb_MessageDef_NestedMessageCount(const upb_MessageDef* m); -UPB_API_INLINE const upb_OneofDef* upb_MessageDef_Oneof(const upb_MessageDef* m, int i); -UPB_API_INLINE int upb_MessageDef_OneofCount(const upb_MessageDef* m); +const upb_OneofDef* upb_MessageDef_Oneof(const upb_MessageDef* m, int i); +int upb_MessageDef_OneofCount(const upb_MessageDef* m); int upb_MessageDef_RealOneofCount(const upb_MessageDef* m); const UPB_DESC(MessageOptions) * @@ -10618,8 +10618,8 @@ const upb_MessageReservedRange* upb_MessageDef_ReservedRange( const upb_MessageDef* m, int i); int upb_MessageDef_ReservedRangeCount(const upb_MessageDef* m); -UPB_API_INLINE upb_Syntax upb_MessageDef_Syntax(const upb_MessageDef* m); -UPB_API_INLINE upb_WellKnown upb_MessageDef_WellKnownType(const upb_MessageDef* m); +upb_Syntax upb_MessageDef_Syntax(const upb_MessageDef* m); +upb_WellKnown upb_MessageDef_WellKnownType(const upb_MessageDef* m); #ifdef __cplusplus } /* extern "C" */ @@ -10670,9 +10670,9 @@ const upb_ServiceDef* upb_MethodDef_Service(const upb_MethodDef* m); extern "C" { #endif -UPB_API_INLINE const upb_MessageDef* upb_OneofDef_ContainingType(const upb_OneofDef* o); -UPB_API_INLINE const upb_FieldDef* upb_OneofDef_Field(const upb_OneofDef* o, int i); -UPB_API_INLINE int upb_OneofDef_FieldCount(const upb_OneofDef* o); +const upb_MessageDef* upb_OneofDef_ContainingType(const upb_OneofDef* o); +const upb_FieldDef* upb_OneofDef_Field(const upb_OneofDef* o, int i); +int upb_OneofDef_FieldCount(const upb_OneofDef* o); const char* upb_OneofDef_FullName(const upb_OneofDef* o); bool upb_OneofDef_HasOptions(const upb_OneofDef* o); uint32_t upb_OneofDef_Index(const upb_OneofDef* o); @@ -10684,7 +10684,7 @@ const upb_FieldDef* upb_OneofDef_LookupNameWithSize(const upb_OneofDef* o, size_t size); const upb_FieldDef* upb_OneofDef_LookupNumber(const upb_OneofDef* o, uint32_t num); -UPB_API_INLINE const char* upb_OneofDef_Name(const upb_OneofDef* o); +const char* upb_OneofDef_Name(const upb_OneofDef* o); int upb_OneofDef_numfields(const upb_OneofDef* o); const UPB_DESC(OneofOptions) * upb_OneofDef_Options(const upb_OneofDef* o); @@ -10736,7 +10736,7 @@ extern "C" { enum { upb_JsonDecode_IgnoreUnknown = 1 }; -UPB_API_INLINE bool upb_JsonDecode(const char* buf, size_t size, upb_Message* msg, +bool upb_JsonDecode(const char* buf, size_t size, upb_Message* msg, const upb_MessageDef* m, const upb_DefPool* symtab, int options, upb_Arena* arena, upb_Status* status); @@ -10834,25 +10834,25 @@ extern "C" { // Returns a mutable pointer to a map, array, or submessage value. If the given // arena is non-NULL this will construct a new object if it was not previously // present. May not be called for primitive fields. -UPB_API_INLINE upb_MutableMessageValue upb_Message_Mutable(upb_Message* msg, +upb_MutableMessageValue upb_Message_Mutable(upb_Message* msg, const upb_FieldDef* f, upb_Arena* a); // Returns the field that is set in the oneof, or NULL if none are set. -UPB_API_INLINE const upb_FieldDef* upb_Message_WhichOneof(const upb_Message* msg, +const upb_FieldDef* upb_Message_WhichOneof(const upb_Message* msg, const upb_OneofDef* o); // Clear all data and unknown fields. void upb_Message_ClearByDef(upb_Message* msg, const upb_MessageDef* m); // Clears any field presence and sets the value back to its default. -UPB_API_INLINE void upb_Message_ClearFieldByDef(upb_Message* msg, const upb_FieldDef* f); +void upb_Message_ClearFieldByDef(upb_Message* msg, const upb_FieldDef* f); // May only be called for fields where upb_FieldDef_HasPresence(f) == true. -UPB_API_INLINE bool upb_Message_HasFieldByDef(const upb_Message* msg, const upb_FieldDef* f); +bool upb_Message_HasFieldByDef(const upb_Message* msg, const upb_FieldDef* f); // Returns the value in the message associated with this field def. -UPB_API_INLINE upb_MessageValue upb_Message_GetFieldByDef(const upb_Message* msg, +upb_MessageValue upb_Message_GetFieldByDef(const upb_Message* msg, const upb_FieldDef* f); // Sets the given field to the given value. For a msg/array/map/string, the @@ -10860,7 +10860,7 @@ UPB_API_INLINE upb_MessageValue upb_Message_GetFieldByDef(const upb_Message* msg // the same arena or a different arena that outlives it). // // Returns false if allocation fails. -UPB_API_INLINE bool upb_Message_SetFieldByDef(upb_Message* msg, const upb_FieldDef* f, +bool upb_Message_SetFieldByDef(upb_Message* msg, const upb_FieldDef* f, upb_MessageValue val, upb_Arena* a); // Iterate over present fields. @@ -10883,7 +10883,7 @@ bool upb_Message_Next(const upb_Message* msg, const upb_MessageDef* m, upb_MessageValue* val, size_t* iter); // Clears all unknown field data from this message and all submessages. -UPB_API_INLINE bool upb_Message_DiscardUnknown(upb_Message* msg, const upb_MessageDef* m, +bool upb_Message_DiscardUnknown(upb_Message* msg, const upb_MessageDef* m, int maxdepth); #ifdef __cplusplus @@ -10923,7 +10923,7 @@ enum { * size (excluding NULL) is returned. This means that a return value >= |size| * implies that the output was truncated. (These are the same semantics as * snprintf()). */ -UPB_API_INLINE size_t upb_JsonEncode(const upb_Message* msg, const upb_MessageDef* m, +size_t upb_JsonEncode(const upb_Message* msg, const upb_MessageDef* m, const upb_DefPool* ext_pool, int options, char* buf, size_t size, upb_Status* status); diff --git a/ruby/lib/google/protobuf/ffi/ffi.rb b/ruby/lib/google/protobuf/ffi/ffi.rb index dd49f8a74d69..afa1a3f78fe9 100644 --- a/ruby/lib/google/protobuf/ffi/ffi.rb +++ b/ruby/lib/google/protobuf/ffi/ffi.rb @@ -245,47 +245,47 @@ def initialize attach_function :array_size, :upb_Array_Size, [:Array], :size_t # DefPool - attach_function :add_serialized_file, :upb_DefPool_AddFile, [:DefPool, :FileDescriptorProto, Status.by_ref], :FileDef - attach_function :free_DescriptorPool, :upb_DefPool_Free, [:DefPool], :void - attach_function :create_DescriptorPool, :upb_DefPool_New, [], :DefPool - attach_function :lookup_enum, :upb_DefPool_FindEnumByName, [:DefPool, :string], EnumDef - attach_function :lookup_msg, :upb_DefPool_FindMessageByName, [:DefPool, :string], MessageDef + attach_function :add_serialized_file, :API_PENDING_upb_DefPool_AddFile, [:DefPool, :FileDescriptorProto, Status.by_ref], :FileDef + attach_function :free_DescriptorPool, :API_PENDING_upb_DefPool_Free, [:DefPool], :void + attach_function :create_DescriptorPool, :API_PENDING_upb_DefPool_New, [], :DefPool + attach_function :lookup_enum, :API_PENDING_upb_DefPool_FindEnumByName, [:DefPool, :string], EnumDef + attach_function :lookup_msg, :API_PENDING_upb_DefPool_FindMessageByName, [:DefPool, :string], MessageDef # EnumDescriptor - attach_function :get_enum_file_descriptor, :upb_EnumDef_File, [EnumDef], :FileDef - attach_function :enum_value_by_name, :upb_EnumDef_FindValueByNameWithSize,[EnumDef, :string, :size_t], :EnumValueDef - attach_function :enum_value_by_number, :upb_EnumDef_FindValueByNumber, [EnumDef, :int], :EnumValueDef - attach_function :get_enum_fullname, :upb_EnumDef_FullName, [EnumDef], :string - attach_function :enum_value_by_index, :upb_EnumDef_Value, [EnumDef, :int], :EnumValueDef - attach_function :enum_value_count, :upb_EnumDef_ValueCount, [EnumDef], :int - attach_function :enum_name, :upb_EnumValueDef_Name, [:EnumValueDef], :string - attach_function :enum_number, :upb_EnumValueDef_Number, [:EnumValueDef], :int + attach_function :get_enum_file_descriptor, :API_PENDING_upb_EnumDef_File, [EnumDef], :FileDef + attach_function :enum_value_by_name, :API_PENDING_upb_EnumDef_FindValueByNameWithSize,[EnumDef, :string, :size_t], :EnumValueDef + attach_function :enum_value_by_number, :API_PENDING_upb_EnumDef_FindValueByNumber, [EnumDef, :int], :EnumValueDef + attach_function :get_enum_fullname, :API_PENDING_upb_EnumDef_FullName, [EnumDef], :string + attach_function :enum_value_by_index, :API_PENDING_upb_EnumDef_Value, [EnumDef, :int], :EnumValueDef + attach_function :enum_value_count, :API_PENDING_upb_EnumDef_ValueCount, [EnumDef], :int + attach_function :enum_name, :API_PENDING_upb_EnumValueDef_Name, [:EnumValueDef], :string + attach_function :enum_number, :API_PENDING_upb_EnumValueDef_Number, [:EnumValueDef], :int # FileDescriptor - attach_function :file_def_name, :upb_FileDef_Name, [:FileDef], :string - attach_function :file_def_syntax, :upb_FileDef_Syntax, [:FileDef], Syntax - attach_function :file_def_pool, :upb_FileDef_Pool, [:FileDef], :DefPool + attach_function :file_def_name, :API_PENDING_upb_FileDef_Name, [:FileDef], :string + attach_function :file_def_syntax, :API_PENDING_upb_FileDef_Syntax, [:FileDef], Syntax + attach_function :file_def_pool, :API_PENDING_upb_FileDef_Pool, [:FileDef], :DefPool # FileDescriptorProto attach_function :parse, :FileDescriptorProto_parse, [:binary_string, :size_t], :FileDescriptorProto # FieldDescriptor - attach_function :get_containing_message_def, :upb_FieldDef_ContainingType, [FieldDescriptor], MessageDef - attach_function :get_c_type, :upb_FieldDef_CType, [FieldDescriptor], CType - attach_function :get_default, :upb_FieldDef_Default, [FieldDescriptor], MessageValue.by_value - attach_function :get_subtype_as_enum, :upb_FieldDef_EnumSubDef, [FieldDescriptor], EnumDef - attach_function :get_has_presence, :upb_FieldDef_HasPresence, [FieldDescriptor], :bool - attach_function :is_map, :upb_FieldDef_IsMap, [FieldDescriptor], :bool - attach_function :is_repeated, :upb_FieldDef_IsRepeated, [FieldDescriptor], :bool - attach_function :is_sub_message, :upb_FieldDef_IsSubMessage, [FieldDescriptor], :bool - attach_function :get_json_name, :upb_FieldDef_JsonName, [FieldDescriptor], :string - attach_function :get_label, :upb_FieldDef_Label, [FieldDescriptor], Label - attach_function :get_subtype_as_message, :upb_FieldDef_MessageSubDef, [FieldDescriptor], MessageDef - attach_function :get_full_name, :upb_FieldDef_Name, [FieldDescriptor], :string - attach_function :get_number, :upb_FieldDef_Number, [FieldDescriptor], :uint32_t - attach_function :real_containing_oneof, :upb_FieldDef_RealContainingOneof,[FieldDescriptor], OneofDef - attach_function :get_type, :upb_FieldDef_Type, [FieldDescriptor], FieldType - attach_function :file_def_by_raw_field_def, :upb_FieldDef_File, [:pointer], :FileDef + attach_function :get_containing_message_def, :API_PENDING_upb_FieldDef_ContainingType, [FieldDescriptor], MessageDef + attach_function :get_c_type, :API_PENDING_upb_FieldDef_CType, [FieldDescriptor], CType + attach_function :get_default, :API_PENDING_upb_FieldDef_Default, [FieldDescriptor], MessageValue.by_value + attach_function :get_subtype_as_enum, :API_PENDING_upb_FieldDef_EnumSubDef, [FieldDescriptor], EnumDef + attach_function :get_has_presence, :API_PENDING_upb_FieldDef_HasPresence, [FieldDescriptor], :bool + attach_function :is_map, :API_PENDING_upb_FieldDef_IsMap, [FieldDescriptor], :bool + attach_function :is_repeated, :API_PENDING_upb_FieldDef_IsRepeated, [FieldDescriptor], :bool + attach_function :is_sub_message, :API_PENDING_upb_FieldDef_IsSubMessage, [FieldDescriptor], :bool + attach_function :get_json_name, :API_PENDING_upb_FieldDef_JsonName, [FieldDescriptor], :string + attach_function :get_label, :API_PENDING_upb_FieldDef_Label, [FieldDescriptor], Label + attach_function :get_subtype_as_message, :API_PENDING_upb_FieldDef_MessageSubDef, [FieldDescriptor], MessageDef + attach_function :get_full_name, :API_PENDING_upb_FieldDef_Name, [FieldDescriptor], :string + attach_function :get_number, :API_PENDING_upb_FieldDef_Number, [FieldDescriptor], :uint32_t + attach_function :real_containing_oneof, :API_PENDING_upb_FieldDef_RealContainingOneof,[FieldDescriptor], OneofDef + attach_function :get_type, :API_PENDING_upb_FieldDef_Type, [FieldDescriptor], FieldType + attach_function :file_def_by_raw_field_def, :API_PENDING_upb_FieldDef_File, [:pointer], :FileDef # Map attach_function :map_clear, :upb_Map_Clear, [:Map], :void attach_function :map_delete, :upb_Map_Delete, [:Map, MessageValue.by_value, MessageValue.by_ref], :bool @@ -295,49 +295,49 @@ def initialize attach_function :map_set, :upb_Map_Set, [:Map, MessageValue.by_value, MessageValue.by_value, Arena], :bool # MapIterator - attach_function :map_next, :upb_MapIterator_Next, [:Map, :pointer], :bool - attach_function :map_done, :upb_MapIterator_Done, [:Map, :size_t], :bool - attach_function :map_key, :upb_MapIterator_Key, [:Map, :size_t], MessageValue.by_value - attach_function :map_value, :upb_MapIterator_Value, [:Map, :size_t], MessageValue.by_value + attach_function :map_next, :API_PENDING_upb_MapIterator_Next, [:Map, :pointer], :bool + attach_function :map_done, :API_PENDING_upb_MapIterator_Done, [:Map, :size_t], :bool + attach_function :map_key, :API_PENDING_upb_MapIterator_Key, [:Map, :size_t], MessageValue.by_value + attach_function :map_value, :API_PENDING_upb_MapIterator_Value, [:Map, :size_t], MessageValue.by_value # MessageDef attach_function :new_message_from_def, :upb_Message_New, [MessageDef, Arena], :Message - attach_function :get_field_by_index, :upb_MessageDef_Field, [MessageDef, :int], FieldDescriptor - attach_function :field_count, :upb_MessageDef_FieldCount, [MessageDef], :int - attach_function :get_message_file_def, :upb_MessageDef_File, [:pointer], :FileDef - attach_function :get_field_by_name, :upb_MessageDef_FindFieldByNameWithSize, [MessageDef, :string, :size_t], FieldDescriptor - attach_function :get_field_by_number, :upb_MessageDef_FindFieldByNumber, [MessageDef, :uint32_t], FieldDescriptor - attach_function :get_oneof_by_name, :upb_MessageDef_FindOneofByNameWithSize, [MessageDef, :string, :size_t], OneofDef - attach_function :get_message_fullname, :upb_MessageDef_FullName, [MessageDef], :string - attach_function :get_mini_table, :upb_MessageDef_MiniTable, [MessageDef], MiniTable.ptr - attach_function :get_oneof_by_index, :upb_MessageDef_Oneof, [MessageDef, :int], OneofDef - attach_function :oneof_count, :upb_MessageDef_OneofCount, [MessageDef], :int - attach_function :get_well_known_type, :upb_MessageDef_WellKnownType, [MessageDef], WellKnown - attach_function :message_def_syntax, :upb_MessageDef_Syntax, [MessageDef], Syntax - attach_function :find_msg_def_by_name, :upb_MessageDef_FindByNameWithSize, [MessageDef, :string, :size_t, :FieldDefPointer, :OneofDefPointer], :bool + attach_function :get_field_by_index, :API_PENDING_upb_MessageDef_Field, [MessageDef, :int], FieldDescriptor + attach_function :field_count, :API_PENDING_upb_MessageDef_FieldCount, [MessageDef], :int + attach_function :get_message_file_def, :API_PENDING_upb_MessageDef_File, [:pointer], :FileDef + attach_function :get_field_by_name, :API_PENDING_upb_MessageDef_FindFieldByNameWithSize, [MessageDef, :string, :size_t], FieldDescriptor + attach_function :get_field_by_number, :API_PENDING_upb_MessageDef_FindFieldByNumber, [MessageDef, :uint32_t], FieldDescriptor + attach_function :get_oneof_by_name, :API_PENDING_upb_MessageDef_FindOneofByNameWithSize, [MessageDef, :string, :size_t], OneofDef + attach_function :get_message_fullname, :API_PENDING_upb_MessageDef_FullName, [MessageDef], :string + attach_function :get_mini_table, :API_PENDING_upb_MessageDef_MiniTable, [MessageDef], MiniTable.ptr + attach_function :get_oneof_by_index, :API_PENDING_upb_MessageDef_Oneof, [MessageDef, :int], OneofDef + attach_function :oneof_count, :API_PENDING_upb_MessageDef_OneofCount, [MessageDef], :int + attach_function :get_well_known_type, :API_PENDING_upb_MessageDef_WellKnownType, [MessageDef], WellKnown + attach_function :message_def_syntax, :API_PENDING_upb_MessageDef_Syntax, [MessageDef], Syntax + attach_function :find_msg_def_by_name, :API_PENDING_upb_MessageDef_FindByNameWithSize, [MessageDef, :string, :size_t, :FieldDefPointer, :OneofDefPointer], :bool # Message - attach_function :clear_message_field, :upb_Message_ClearFieldByDef, [:Message, FieldDescriptor], :void - attach_function :get_message_value, :upb_Message_GetFieldByDef, [:Message, FieldDescriptor], MessageValue.by_value - attach_function :get_message_has, :upb_Message_HasFieldByDef, [:Message, FieldDescriptor], :bool - attach_function :set_message_field, :upb_Message_SetFieldByDef, [:Message, FieldDescriptor, MessageValue.by_value, Arena], :bool - attach_function :encode_message, :upb_Encode, [:Message, MiniTable.by_ref, :size_t, Arena, :pointer, :pointer], EncodeStatus - attach_function :json_decode_message, :upb_JsonDecode, [:binary_string, :size_t, :Message, MessageDef, :DefPool, :int, Arena, Status.by_ref], :bool - attach_function :json_encode_message, :upb_JsonEncode, [:Message, MessageDef, :DefPool, :int, :binary_string, :size_t, Status.by_ref], :size_t + attach_function :clear_message_field, :API_PENDING_upb_Message_ClearFieldByDef, [:Message, FieldDescriptor], :void + attach_function :get_message_value, :API_PENDING_upb_Message_GetFieldByDef, [:Message, FieldDescriptor], MessageValue.by_value + attach_function :get_message_has, :API_PENDING_upb_Message_HasFieldByDef, [:Message, FieldDescriptor], :bool + attach_function :set_message_field, :API_PENDING_upb_Message_SetFieldByDef, [:Message, FieldDescriptor, MessageValue.by_value, Arena], :bool + attach_function :encode_message, :API_PENDING_upb_Encode, [:Message, MiniTable.by_ref, :size_t, Arena, :pointer, :pointer], EncodeStatus + attach_function :json_decode_message, :API_PENDING_upb_JsonDecode, [:binary_string, :size_t, :Message, MessageDef, :DefPool, :int, Arena, Status.by_ref], :bool + attach_function :json_encode_message, :API_PENDING_upb_JsonEncode, [:Message, MessageDef, :DefPool, :int, :binary_string, :size_t, Status.by_ref], :size_t attach_function :decode_message, :upb_Decode, [:binary_string, :size_t, :Message, MiniTable.by_ref, :ExtensionRegistry, :int, Arena], DecodeStatus - attach_function :get_mutable_message, :upb_Message_Mutable, [:Message, FieldDescriptor, Arena], MutableMessageValue.by_value - attach_function :get_message_which_oneof, :upb_Message_WhichOneof, [:Message, OneofDef], FieldDescriptor - attach_function :message_discard_unknown, :upb_Message_DiscardUnknown, [:Message, MessageDef, :int], :bool + attach_function :get_mutable_message, :API_PENDING_upb_Message_Mutable, [:Message, FieldDescriptor, Arena], MutableMessageValue.by_value + attach_function :get_message_which_oneof, :API_PENDING_upb_Message_WhichOneof, [:Message, OneofDef], FieldDescriptor + attach_function :message_discard_unknown, :API_PENDING_upb_Message_DiscardUnknown, [:Message, MessageDef, :int], :bool # MessageValue attach_function :message_value_equal, :shared_Msgval_IsEqual, [MessageValue.by_value, MessageValue.by_value, CType, MessageDef], :bool attach_function :message_value_hash, :shared_Msgval_GetHash, [MessageValue.by_value, CType, MessageDef, :uint64_t], :uint64_t # OneofDescriptor - attach_function :get_oneof_name, :upb_OneofDef_Name, [OneofDef], :string - attach_function :get_oneof_field_count, :upb_OneofDef_FieldCount, [OneofDef], :int - attach_function :get_oneof_field_by_index, :upb_OneofDef_Field, [OneofDef, :int], FieldDescriptor - attach_function :get_oneof_containing_type,:upb_OneofDef_ContainingType,[:pointer], MessageDef + attach_function :get_oneof_name, :API_PENDING_upb_OneofDef_Name, [OneofDef], :string + attach_function :get_oneof_field_count, :API_PENDING_upb_OneofDef_FieldCount, [OneofDef], :int + attach_function :get_oneof_field_by_index, :API_PENDING_upb_OneofDef_Field, [OneofDef, :int], FieldDescriptor + attach_function :get_oneof_containing_type,:API_PENDING_upb_OneofDef_ContainingType,[:pointer], MessageDef # RepeatableField @@ -349,9 +349,6 @@ def initialize attach_function :memcmp, [:pointer, :pointer, :size_t], :int attach_function :memcpy, [:pointer, :pointer, :size_t], :int - # Misc - attach_function :hash, :_upb_Hash, [:pointer, :size_t, :uint64_t], :uint32_t - # Alternatives to pre-processor macros def self.decode_max_depth(i) i << 16 diff --git a/ruby/lib/google/protobuf/ffi/message.rb b/ruby/lib/google/protobuf/ffi/message.rb index 0c8b209fb332..0058e09d473a 100644 --- a/ruby/lib/google/protobuf/ffi/message.rb +++ b/ruby/lib/google/protobuf/ffi/message.rb @@ -103,7 +103,7 @@ def hash if encoding_status != :Ok or encoding.null? raise ParseError.new "Error calculating hash" end - Google::Protobuf::FFI.hash(encoding.read(:pointer), size_ptr.read(:size_t), 0) + encoding.read(:pointer).read_string(size_ptr.read(:size_t)).hash end def to_h From 8920f9d98c47c51268fb05707c9e0dcff4343e0f Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Mon, 10 Jul 2023 18:38:28 -0400 Subject: [PATCH 45/54] Update to the new Ruby-native ObjectCache. Drop test sources from the gemspec which was breaking tests. --- ruby/google-protobuf.gemspec | 1 - ruby/lib/google/protobuf/ffi/descriptor.rb | 2 +- .../google/protobuf/ffi/descriptor_pool.rb | 2 +- .../protobuf/ffi/internal/pointer_helper.rb | 10 +-- ruby/lib/google/protobuf/ffi/map.rb | 2 +- ruby/lib/google/protobuf/ffi/message.rb | 6 +- ruby/lib/google/protobuf/ffi/object_cache.rb | 71 ++++--------------- .../lib/google/protobuf/ffi/repeated_field.rb | 2 +- 8 files changed, 27 insertions(+), 69 deletions(-) diff --git a/ruby/google-protobuf.gemspec b/ruby/google-protobuf.gemspec index 698553f03803..bdc6815d418d 100644 --- a/ruby/google-protobuf.gemspec +++ b/ruby/google-protobuf.gemspec @@ -11,7 +11,6 @@ Gem::Specification.new do |s| s.metadata = { "source_code_uri" => "https://github.com/protocolbuffers/protobuf/tree/#{git_tag}/ruby" } s.require_paths = ["lib"] s.files = Dir.glob('lib/**/*.rb') - s.test_files = %w[tests/basic.rb tests/stress.rb tests/generated_code_test.rb] if RUBY_PLATFORM == "java" s.platform = "java" s.files += ["lib/google/protobuf_java.jar"] diff --git a/ruby/lib/google/protobuf/ffi/descriptor.rb b/ruby/lib/google/protobuf/ffi/descriptor.rb index 06fd6136f034..31a4596ded7b 100644 --- a/ruby/lib/google/protobuf/ffi/descriptor.rb +++ b/ruby/lib/google/protobuf/ffi/descriptor.rb @@ -149,7 +149,7 @@ def wrapper? def self.get_message(msg, descriptor, arena) return nil if msg.nil? or msg.null? - message = ObjectCache.get(msg) + message = OBJECT_CACHE.get(msg.address) if message.nil? message = descriptor.msgclass.send(:private_constructor, arena, msg: msg) end diff --git a/ruby/lib/google/protobuf/ffi/descriptor_pool.rb b/ruby/lib/google/protobuf/ffi/descriptor_pool.rb index c12e7293f374..356ad15549a8 100644 --- a/ruby/lib/google/protobuf/ffi/descriptor_pool.rb +++ b/ruby/lib/google/protobuf/ffi/descriptor_pool.rb @@ -40,7 +40,7 @@ def initialize # Should always be the last expression of the initializer to avoid # leaking references to this object before construction is complete. - Google::Protobuf::ObjectCache.add @descriptor_pool, self + Google::Protobuf::OBJECT_CACHE.try_add @descriptor_pool.address, self end def add_serialized_file(file_contents) diff --git a/ruby/lib/google/protobuf/ffi/internal/pointer_helper.rb b/ruby/lib/google/protobuf/ffi/internal/pointer_helper.rb index 5984edd50dd7..a1cc15964427 100644 --- a/ruby/lib/google/protobuf/ffi/internal/pointer_helper.rb +++ b/ruby/lib/google/protobuf/ffi/internal/pointer_helper.rb @@ -32,17 +32,17 @@ module Google module Protobuf module Internal module PointerHelper - # Utility code to defensively find walk the object graph from a file_def - # to the pool, and either retrieve the wrapper object for the given - # pointer or create one. Assumes that the caller is the wrapper class - # for the given pointer and that it implements `private_constructor`. + # Utility code to defensively walk the object graph from a file_def to + # the pool, and either retrieve the wrapper object for the given pointer + # or create one. Assumes that the caller is the wrapper class for the + # given pointer and that it implements `private_constructor`. def descriptor_from_file_def(file_def, pointer) raise RuntimeError.new "FileDef is nil" if file_def.nil? raise RuntimeError.new "FileDef is null" if file_def.null? pool_def = Google::Protobuf::FFI.file_def_pool file_def raise RuntimeError.new "PoolDef is nil" if pool_def.nil? raise RuntimeError.new "PoolDef is null" if pool_def.null? - pool = Google::Protobuf::ObjectCache.get(pool_def) + pool = Google::Protobuf::OBJECT_CACHE.get(pool_def.address) raise "Cannot find pool in ObjectCache!" if pool.nil? descriptor = pool.descriptor_class_by_def[pointer.address] if descriptor.nil? diff --git a/ruby/lib/google/protobuf/ffi/map.rb b/ruby/lib/google/protobuf/ffi/map.rb index e4f8ee2d6a5b..51c87387a6bf 100644 --- a/ruby/lib/google/protobuf/ffi/map.rb +++ b/ruby/lib/google/protobuf/ffi/map.rb @@ -363,7 +363,7 @@ def initialize(key_type, value_type, value_type_class: nil, initial_values: nil, # Should always be the last expression of the initializer to avoid # leaking references to this object before construction is complete. - ObjectCache.add(@map_ptr, self) + OBJECT_CACHE.try_add(@map_ptr.address, self) end # @param field [FieldDescriptor] Descriptor of the field where the RepeatedField will be assigned diff --git a/ruby/lib/google/protobuf/ffi/message.rb b/ruby/lib/google/protobuf/ffi/message.rb index 0058e09d473a..15083352cbc1 100644 --- a/ruby/lib/google/protobuf/ffi/message.rb +++ b/ruby/lib/google/protobuf/ffi/message.rb @@ -566,7 +566,7 @@ def initialize(initial_value = nil, arena = nil, msg = nil) # Should always be the last expression of the initializer to avoid # leaking references to this object before construction is complete. - Google::Protobuf::ObjectCache.add @msg, self + Google::Protobuf::OBJECT_CACHE.try_add @msg.address, self end ## @@ -615,7 +615,7 @@ def get_field(field, unwrap: false) # @param field [Google::Protobuf::FieldDescriptor] Type of the repeated field def get_repeated_field(array, field) return nil if array.nil? or array.null? - repeated_field = ObjectCache.get(array) + repeated_field = OBJECT_CACHE.get(array.address) if repeated_field.nil? repeated_field = RepeatedField.send(:construct_for_field, field, @arena, array: array) end @@ -627,7 +627,7 @@ def get_repeated_field(array, field) # @param field [Google::Protobuf::FieldDescriptor] Type of the map field def get_map_field(map, field) return nil if map.nil? or map.null? - map_field = ObjectCache.get(map) + map_field = OBJECT_CACHE.get(map.address) if map_field.nil? map_field = Google::Protobuf::Map.send(:construct_for_field, field, @arena, map: map) end diff --git a/ruby/lib/google/protobuf/ffi/object_cache.rb b/ruby/lib/google/protobuf/ffi/object_cache.rb index 8ad0de109bee..b462963af01d 100644 --- a/ruby/lib/google/protobuf/ffi/object_cache.rb +++ b/ruby/lib/google/protobuf/ffi/object_cache.rb @@ -28,67 +28,26 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -require 'weakref' - module Google module Protobuf - module ObjectCache - @@lock = Mutex.new - @@cache = {} - def self.drop(key) - @@lock.synchronize do - @@cache.delete(key.address) - end - end + private - def self.get(key) - @@lock.synchronize do - value = @@cache[key.address] - begin - if value.nil? - return nil - else - if value.weakref_alive? - return value.__getobj__ - else - @@cache.delete(key.address) - return nil - end - end - rescue WeakRef::RefError - @@cache.delete(key.address) - return nil - end - end - nil - end + SIZEOF_LONG = ::FFI::MemoryPointer.new(:long).size + SIZEOF_VALUE = ::FFI::Pointer::SIZE - def self.add(key, value) - raise ArgumentError.new "WeakRef values are not allowed" if value.is_a? WeakRef - @@lock.synchronize do - if @@cache.include? key.address - existing_value = @@cache[key.address] - begin - if existing_value.nil? - raise ArgumentError.new "Key already exists in ObjectCache but has nil value" - else - if existing_value.weakref_alive? - original = existing_value.__getobj__ - raise ArgumentError.new "Key already exists in ObjectCache for different value" unless original.object_id == value.object_id - else - @@cache.delete(key.address) - nil - end - end - rescue WeakRef::RefError - @@cache.delete(key.address) - nil - end - end - @@cache[key.address] = WeakRef.new value - nil - end + def self.interpreter_supports_non_finalized_keys_in_weak_map? + Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('2.7.0') + end + + def self.cache_implementation + if interpreter_supports_non_finalized_keys_in_weak_map? and SIZEOF_LONG >= SIZEOF_VALUE + Google::Protobuf::ObjectCache + else + Google::Protobuf::LegacyObjectCache end end + + public + OBJECT_CACHE = cache_implementation.new end end \ No newline at end of file diff --git a/ruby/lib/google/protobuf/ffi/repeated_field.rb b/ruby/lib/google/protobuf/ffi/repeated_field.rb index 542e57434400..5fe1c39bb107 100644 --- a/ruby/lib/google/protobuf/ffi/repeated_field.rb +++ b/ruby/lib/google/protobuf/ffi/repeated_field.rb @@ -478,7 +478,7 @@ def initialize(type, type_class: nil, initial_values: nil, name: nil, arena: nil # Should always be the last expression of the initializer to avoid # leaking references to this object before construction is complete. - ObjectCache.add(@array, self) + OBJECT_CACHE.try_add(@array.address, self) end # @param field [FieldDescriptor] Descriptor of the field where the RepeatedField will be assigned From 72d6ee86bb5ccedda8aafb860f7efa839464c065 Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Mon, 10 Jul 2023 18:51:33 -0400 Subject: [PATCH 46/54] Allow fallback to native CRuby implementation when FFI doesn't load. --- ruby/lib/google/protobuf.rb | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ruby/lib/google/protobuf.rb b/ruby/lib/google/protobuf.rb index 4c3e8565cef9..f2beb462b290 100644 --- a/ruby/lib/google/protobuf.rb +++ b/ruby/lib/google/protobuf.rb @@ -68,9 +68,10 @@ def self.decode_json(klass, json, options = {}) require 'google/protobuf_ffi' :FFI rescue LoadError - warn "Caught exception `#{$!.message}` while loading FFI implementation of google/protobuf." - raise $! - warn "Falling back to native implementation." + if $DEBUG + warn "Caught exception `#{$!.message}` while loading FFI implementation of google/protobuf." + warn "Falling back to native implementation." + end require 'google/protobuf_native' :NATIVE end From a3eb31777f70f90e535df53f73e82e629d6f030b Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Wed, 12 Jul 2023 11:07:50 -0400 Subject: [PATCH 47/54] Introduce a new test that the expected implementation is used. Make tests executable from bazel and rake. --- ruby/BUILD.bazel | 58 +++++----- ruby/Rakefile | 16 +-- ruby/ext/google/protobuf_c/BUILD.bazel | 102 ++++++++++++++++-- ruby/lib/google/BUILD.bazel | 42 ++++++-- ruby/lib/google/protobuf.rb | 25 +++-- .../google/protobuf/ffi/descriptor_pool.rb | 2 +- ruby/lib/google/protobuf/ffi/ffi.rb | 99 +++++++++-------- ruby/lib/google/protobuf_ffi.rb | 2 +- ruby/tests/BUILD.bazel | 9 ++ ruby/tests/backend.rb | 11 ++ 10 files changed, 249 insertions(+), 117 deletions(-) create mode 100644 ruby/tests/backend.rb diff --git a/ruby/BUILD.bazel b/ruby/BUILD.bazel index fe215943ece7..f3e950dcf55c 100755 --- a/ruby/BUILD.bazel +++ b/ruby/BUILD.bazel @@ -27,23 +27,23 @@ string_flag( config_setting( name = "ffi_enabled", flag_values = { - "ffi": "enabled" + ":ffi": "enabled", }, ) config_setting( name = "ffi_disabled", flag_values = { - "ffi": "disabled" + ":ffi": "disabled", }, ) ruby_library( name = "protobuf", - deps = ["//ruby/lib/google:protobuf_lib"], visibility = [ "//visibility:public", ], + deps = ["//ruby/lib/google:protobuf_lib"], ) # Note: these can be greatly simplified using inline_sh_binary in Bazel 6, @@ -52,12 +52,12 @@ ruby_library( genrule( name = "jruby_release", srcs = [ - "//ruby/lib/google:copy_jar", - "//ruby/lib/google:dist_files", - "//:well_known_ruby_protos", - "google-protobuf.gemspec", + "//ruby/lib/google:copy_jar", + "//ruby/lib/google:dist_files", + "//:well_known_ruby_protos", + "google-protobuf.gemspec", ], - outs = ["google-protobuf-"+PROTOBUF_RUBY_VERSION+"-java.gem"], + outs = ["google-protobuf-" + PROTOBUF_RUBY_VERSION + "-java.gem"], cmd = """ set -eux mkdir tmp @@ -84,14 +84,14 @@ genrule( genrule( name = "ruby_release", srcs = [ - "@utf8_range//:utf8_range_srcs", - "@utf8_range//:LICENSE", - "//:well_known_ruby_protos", - "//ruby/ext/google/protobuf_c:dist_files", - "//ruby/lib/google:dist_files", - "google-protobuf.gemspec", + "@utf8_range//:utf8_range_srcs", + "@utf8_range//:LICENSE", + "//:well_known_ruby_protos", + "//ruby/ext/google/protobuf_c:dist_files", + "//ruby/lib/google:dist_files", + "google-protobuf.gemspec", ], - outs = ["google-protobuf-"+PROTOBUF_RUBY_VERSION+".gem"], + outs = ["google-protobuf-" + PROTOBUF_RUBY_VERSION + ".gem"], cmd = """ set -eux mkdir tmp @@ -127,7 +127,6 @@ filegroup( tags = ["manual"], ) - ################################################################################ # Tests ################################################################################ @@ -136,34 +135,37 @@ filegroup( internal_ruby_proto_library( name = "test_ruby_protos", srcs = ["//ruby/tests:test_protos"], - deps = ["//:well_known_ruby_protos"], - includes = [".", "src", "ruby/tests"], + includes = [ + ".", + "ruby/tests", + "src", + ], visibility = [ "//ruby:__subpackages__", ], + deps = ["//:well_known_ruby_protos"], ) - conformance_test( name = "conformance_test", failure_list = "//conformance:failure_list_ruby.txt", - testee = "//conformance:conformance_ruby", - text_format_failure_list = "//conformance:text_format_failure_list_ruby.txt", target_compatible_with = select({ "@rules_ruby//ruby/runtime:config_ruby": [], "//conditions:default": ["@platforms//:incompatible"], }), + testee = "//conformance:conformance_ruby", + text_format_failure_list = "//conformance:text_format_failure_list_ruby.txt", ) conformance_test( name = "conformance_test_jruby", failure_list = "//conformance:failure_list_jruby.txt", - testee = "//conformance:conformance_ruby", - text_format_failure_list = "//conformance:text_format_failure_list_jruby.txt", target_compatible_with = select({ "@rules_ruby//ruby/runtime:config_jruby": [], "//conditions:default": ["@platforms//:incompatible"], }), + testee = "//conformance:conformance_ruby", + text_format_failure_list = "//conformance:text_format_failure_list_jruby.txt", ) ################################################################################ @@ -173,15 +175,15 @@ conformance_test( pkg_files( name = "dist_files", srcs = [ - "//ruby/ext/google/protobuf_c:dist_files", - "//ruby/lib/google:dist_files", - "//ruby/src/main/java:dist_files", - "//ruby/tests:dist_files", ".gitignore", "BUILD.bazel", "Gemfile", - "Rakefile", "README.md", + "Rakefile", + "//ruby/ext/google/protobuf_c:dist_files", + "//ruby/lib/google:dist_files", + "//ruby/src/main/java:dist_files", + "//ruby/tests:dist_files", ], strip_prefix = strip_prefix.from_root(""), visibility = ["//pkg:__pkg__"], diff --git a/ruby/Rakefile b/ruby/Rakefile index 5a8fd5e3c1a0..58dfc978ca08 100644 --- a/ruby/Rakefile +++ b/ruby/Rakefile @@ -81,6 +81,8 @@ if USE_FFI # in order to create subtasks for each source file, so the files from third_party # must be copied into place before the task is defined. ffi_compiler_config_block = Proc.new do |c| + c.ext_dir = File.dirname('ext/google/protobuf_c') + c.source_dirs = [c.ext_dir] c.cflags << common_c_flags common_excluded_source.each { | file | c.exclude << "/#{file}.c" } # Ruby UPB was already compiled with different flags. @@ -88,14 +90,6 @@ if USE_FFI c.exclude << "/range2-sse.c" c.exclude << "/naive.c" c.exclude << "/ruby-upb.c" - # ffi-upb:default's only dependency is the platform-specific path of the - # shared library. Ignore the shared library (or figure out how to pass the - # right flags to build one shared lib from another?) but explicitly add the - # object files that make it up as ldflags so that their symbols are - # available. - # Rake::Task[Rake::Task[:"ffi-upb:default"].prereqs.first].prereqs.each do |prereq| - # c.ldflags << prereq - # end if RbConfig::CONFIG['target_os'] =~ /darwin|linux/ c.cflags << "-Wall -Wsign-compare -Wno-declaration-after-statement" end @@ -104,7 +98,7 @@ if USE_FFI if File.exist? 'ext/google/protobuf_c/utf8_range.h' desc "compiler tasks" namespace "ffi-compiler" do - FFI::Compiler::CompileTask.new('ext/google/protobuf_c', &ffi_compiler_config_block) + FFI::Compiler::CompileTask.new('ext/google/protobuf_c_ffi', &ffi_compiler_config_block) protobuf_c_task = Rake::Task[:"ffi-compiler:default"] protobuf_c_shared_lib_task = Rake::Task[protobuf_c_task.prereqs.first] Rake::Task[Rake::Task[:"ffi-upb:default"].prereqs.first].prereqs.each do |dependency| @@ -118,7 +112,7 @@ if USE_FFI Rake::Task[:copy_third_party].invoke desc "compiler tasks" namespace "ffi-compiler" do - FFI::Compiler::CompileTask.new('ext/google/protobuf_c', &ffi_compiler_config_block) + FFI::Compiler::CompileTask.new('ext/google/protobuf_c_ffi', &ffi_compiler_config_block) protobuf_c_task = Rake::Task[:"ffi-compiler:default"] Rake::Task[:"ffi-upb:default"].prereqs.each do |dependency| protobuf_c_task.prereqs.prepend dependency @@ -202,7 +196,7 @@ else 'x86_64-linux', 'x86-linux', 'x86_64-darwin', 'arm64-darwin', ] - end unless USE_FFI + end task 'gem:java' do sh "rm Gemfile.lock" diff --git a/ruby/ext/google/protobuf_c/BUILD.bazel b/ruby/ext/google/protobuf_c/BUILD.bazel index 1a6b963fad81..673a71599ab8 100644 --- a/ruby/ext/google/protobuf_c/BUILD.bazel +++ b/ruby/ext/google/protobuf_c/BUILD.bazel @@ -6,31 +6,115 @@ package(default_visibility = ["//ruby:__subpackages__"]) cc_library( name = "protobuf_c", - srcs = glob([ - "*.h", - "*.c", - ]), + srcs = [ + "convert.c", + "convert.h", + "defs.c", + "defs.h", + "map.c", + "map.h", + "message.c", + "message.h", + "protobuf.c", + "protobuf.h", + "repeated_field.c", + "repeated_field.h", + "ruby-upb.c", + "ruby-upb.h", + "shared_convert.c", + "shared_convert.h", + "shared_message.c", + "shared_message.h", + "wrap_memcpy.c", + ], + linkstatic = True, + target_compatible_with = select({ + "@rules_ruby//ruby/runtime:config_jruby": ["@platforms//:incompatible"], + "//conditions:default": [], + }), deps = [ "@rules_ruby//ruby/runtime:headers", - "@utf8_range//:utf8_range", + "@utf8_range", + ], + alwayslink = True, +) + +# Needs to be compiled with UPB_BUILD_API in order to expose functions called +# via FFI directly by Ruby. +cc_library( + name = "upb_api", + srcs = [ + "ruby-upb.c", + ], + hdrs = [ + "ruby-upb.h", + ], + copts = ["-fvisibility=hidden"], + linkstatic = False, + local_defines = [ + "UPB_BUILD_API", ], target_compatible_with = select({ - "@rules_ruby//ruby/runtime:config_jruby": ["@platforms//:incompatible"], + "//ruby:ffi_disabled": ["@platforms//:incompatible"], "//conditions:default": [], }), + deps = [ + "@utf8_range", + ], +) + +cc_library( + name = "protobuf_c_ffi", + srcs = [ + "glue.c", + "shared_convert.c", + "shared_convert.h", + "shared_message.c", + "shared_message.h", + ], + copts = [ + "-std=gnu99", + "-O3", + "-Wall", + "-Wsign-compare", + "-Wno-declaration-after-statement", + ], linkstatic = True, - alwayslink = True, + local_defines = [ + "NDEBUG", + ], + target_compatible_with = select({ + "//ruby:ffi_disabled": ["@platforms//:incompatible"], + "//conditions:default": [], + }), + deps = [":upb_api"], + alwayslink = 1, ) apple_binary( - name = "bundle", + name = "ffi_bundle", binary_type = "loadable_bundle", linkopts = [ "-undefined,dynamic_lookup", "-multiply_defined,suppress", ], + minimum_os_version = "10.11", platform_type = "macos", + tags = ["manual"], + deps = [ + ":protobuf_c_ffi", + ], +) + +apple_binary( + name = "bundle", + binary_type = "loadable_bundle", + linkopts = [ + "-undefined,dynamic_lookup", + "-multiply_defined,suppress", + ], minimum_os_version = "10.11", + platform_type = "macos", tags = ["manual"], deps = [ ":protobuf_c", @@ -65,8 +149,8 @@ genrule( staleness_test( name = "test_amalgamation_staleness", outs = [ - "ruby-upb.h", "ruby-upb.c", + "ruby-upb.h", ], generated_pattern = "generated-in/%s", ) diff --git a/ruby/lib/google/BUILD.bazel b/ruby/lib/google/BUILD.bazel index 16a34eefa649..f076fb65384a 100644 --- a/ruby/lib/google/BUILD.bazel +++ b/ruby/lib/google/BUILD.bazel @@ -8,9 +8,16 @@ config_setting( cc_binary( name = "protobuf_c.so", + linkshared = 1, + tags = ["manual"], deps = ["//ruby/ext/google/protobuf_c"], +) + +cc_binary( + name = "protobuf_c_ffi.so", linkshared = 1, tags = ["manual"], + deps = ["//ruby/ext/google/protobuf_c:protobuf_c_ffi"], ) # Move the bundle to the location expected by our Ruby files. @@ -22,13 +29,25 @@ genrule( tags = ["manual"], ) +# Move the bundle to the location expected by our Ruby files. +genrule( + name = "copy_ffi_bundle", + srcs = ["//ruby/ext/google/protobuf_c:ffi_bundle"], + outs = ["libprotobuf_c_ffi.bundle"], + cmd = "cp $< $@", + tags = ["manual"], + visibility = [ + "//ruby:__subpackages__", + ], +) + java_binary( name = "protobuf_java_bin", + create_executable = False, + deploy_env = ["@rules_ruby//ruby/runtime:jruby_binary"], runtime_deps = [ - "//ruby/src/main/java:protobuf_java" + "//ruby/src/main/java:protobuf_java", ], - deploy_env = ["@rules_ruby//ruby/runtime:jruby_binary"], - create_executable = False, ) # Move the jar to the location expected by our Ruby files. @@ -46,6 +65,16 @@ ruby_library( srcs = glob([ "**/*.rb", ]), + data = select({ + "@rules_ruby//ruby/runtime:config_jruby": ["protobuf_java.jar"], + "@platforms//os:osx": ["protobuf_c.bundle"], + "//conditions:default": ["protobuf_c.so"], + }) + select({ + "//ruby:ffi_enabled": ["//ruby/lib/google:libprotobuf_c_ffi.bundle"], + "//conditions:default": [], + }), + includes = ["ruby/lib"], + visibility = ["//ruby:__pkg__"], deps = ["//:well_known_ruby_protos"] + select({ "//ruby:ffi_enabled": [ "@protobuf_bundle//:ffi", @@ -53,13 +82,6 @@ ruby_library( ], "//conditions:default": [], }), - includes = ["ruby/lib"], - data = select({ - "@rules_ruby//ruby/runtime:config_jruby": ["protobuf_java.jar"], - "@platforms//os:osx": ["protobuf_c.bundle"], - "//conditions:default": ["protobuf_c.so"], - }), - visibility = ["//ruby:__pkg__"], ) pkg_files( diff --git a/ruby/lib/google/protobuf.rb b/ruby/lib/google/protobuf.rb index f2beb462b290..9bbb1b898b6a 100644 --- a/ruby/lib/google/protobuf.rb +++ b/ruby/lib/google/protobuf.rb @@ -40,12 +40,25 @@ class Error < StandardError; end class ParseError < Error; end class TypeError < ::TypeError; end - begin - require 'ffi' - PREFER_FFI = true - rescue LoadError - PREFER_FFI = false - end + PREFER_FFI = case ENV['PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION'] + when nil, "" + # When unspecified, PREFER_FFI is autodetected based on + # available gems. + begin + require 'ffi' + require 'ffi-compiler/loader' + true + rescue LoadError + false + end + when /^ffi$/i + true + when /^native$/i + false + else + warn "Unexpected value `#{ENV['PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION']}` for environment variable `PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION`. Should be either \"FFI\", \"NATIVE\". Omit for autodetection." + false + end def self.encode(msg, options = {}) msg.to_proto(options) diff --git a/ruby/lib/google/protobuf/ffi/descriptor_pool.rb b/ruby/lib/google/protobuf/ffi/descriptor_pool.rb index 356ad15549a8..74e6c857c2bc 100644 --- a/ruby/lib/google/protobuf/ffi/descriptor_pool.rb +++ b/ruby/lib/google/protobuf/ffi/descriptor_pool.rb @@ -35,7 +35,7 @@ class DescriptorPool attr_accessor :descriptor_class_by_def def initialize - @descriptor_pool = ::FFI::AutoPointer.new(Google::Protobuf::FFI.create_DescriptorPool, Google::Protobuf::FFI.method(:free_DescriptorPool)) + @descriptor_pool = ::FFI::AutoPointer.new(Google::Protobuf::FFI.create_descriptor_pool, Google::Protobuf::FFI.method(:free_descriptor_pool)) @descriptor_class_by_def = {} # Should always be the last expression of the initializer to avoid diff --git a/ruby/lib/google/protobuf/ffi/ffi.rb b/ruby/lib/google/protobuf/ffi/ffi.rb index afa1a3f78fe9..8b7471995bf4 100644 --- a/ruby/lib/google/protobuf/ffi/ffi.rb +++ b/ruby/lib/google/protobuf/ffi/ffi.rb @@ -28,9 +28,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -require 'ffi' -require 'ffi-compiler/loader' - module Google module Protobuf class FFI @@ -38,11 +35,30 @@ class FFI # Workaround for Bazel's use of symlinks + JRuby's __FILE__ and `caller` # that resolves them. if ENV['BAZEL'] == 'true' - ffi_lib ::FFI::Compiler::Loader.find 'protobuf_c', ENV['PWD'] + ffi_lib ::FFI::Compiler::Loader.find 'protobuf_c_ffi', ENV['PWD'] else - ffi_lib ::FFI::Compiler::Loader.find 'protobuf_c' + ffi_lib ::FFI::Compiler::Loader.find 'protobuf_c_ffi' end + ## Map + Upb_Map_Begin = -1 + + ## Encoding Status + Upb_Status_MaxMessage = 127 + Upb_Encode_Deterministic = 1 + Upb_Encode_SkipUnknown = 2 + + ## JSON Encoding options + # When set, emits 0/default values. TODO(haberman): proto3 only? + Upb_JsonEncode_EmitDefaults = 1 + # When set, use normal (snake_case) field names instead of JSON (camelCase) names. + Upb_JsonEncode_UseProtoNames = 2 + # When set, emits enums as their integer values instead of as their names. + Upb_JsonEncode_FormatEnumsAsIntegers = 4 + + ## JSON Decoding options + Upb_JsonDecode_IgnoreUnknown = 1 + Arena = Google::Protobuf::Internal::Arena MessageDef = Google::Protobuf::Descriptor EnumDef = Google::Protobuf::EnumDescriptor @@ -108,31 +124,6 @@ class FFI :repeated ) - class StringView < ::FFI::Struct - layout :data, :pointer, - :size, :size_t - end - - class MessageValue < ::FFI::Union - layout :bool_val, :bool, - :float_val, :float, - :double_val, :double, - :int32_val, :int32_t, - :int64_val, :int64_t, - :uint32_val, :uint32_t, - :uint64_val,:uint64_t, - :map_val, :pointer, - :msg_val, :pointer, - :array_val,:pointer, - :str_val, StringView - end - - class MutableMessageValue < ::FFI::Union - layout :map, :Map, - :msg, :Message, - :array, :Array - end - Syntax = enum( :Proto2, 2, :Proto3 @@ -184,6 +175,11 @@ class MutableMessageValue < ::FFI::Union :MissingRequired, ) + class StringView < ::FFI::Struct + layout :data, :pointer, + :size, :size_t + end + class MiniTable < ::FFI::Struct layout :subs, :pointer, :fields, :pointer, @@ -199,25 +195,6 @@ class MiniTable < ::FFI::Struct # _upb_FastTable_Entry fasttable[]; end - ## Map - Upb_Map_Begin = -1 - - ## Encoding Status - Upb_Status_MaxMessage = 127 - Upb_Encode_Deterministic = 1 - Upb_Encode_SkipUnknown = 2 - - ## JSON Encoding options - # When set, emits 0/default values. TODO(haberman): proto3 only? - Upb_JsonEncode_EmitDefaults = 1 - # When set, use normal (snake_case) field names instead of JSON (camelCase) names. - Upb_JsonEncode_UseProtoNames = 2 - # When set, emits enums as their integer values instead of as their names. - Upb_JsonEncode_FormatEnumsAsIntegers = 4 - - ## JSON Decoding options - Upb_JsonDecode_IgnoreUnknown = 1 - class Status < ::FFI::Struct layout :ok, :bool, :msg, [:char, Upb_Status_MaxMessage] @@ -228,6 +205,26 @@ def initialize end end + class MessageValue < ::FFI::Union + layout :bool_val, :bool, + :float_val, :float, + :double_val, :double, + :int32_val, :int32_t, + :int64_val, :int64_t, + :uint32_val, :uint32_t, + :uint64_val,:uint64_t, + :map_val, :pointer, + :msg_val, :pointer, + :array_val,:pointer, + :str_val, StringView + end + + class MutableMessageValue < ::FFI::Union + layout :map, :Map, + :msg, :Message, + :array, :Array + end + # Arena attach_function :create_arena, :Arena_create, [], Arena attach_function :fuse_arena, :upb_Arena_Fuse, [Arena, Arena], :bool @@ -246,8 +243,8 @@ def initialize # DefPool attach_function :add_serialized_file, :API_PENDING_upb_DefPool_AddFile, [:DefPool, :FileDescriptorProto, Status.by_ref], :FileDef - attach_function :free_DescriptorPool, :API_PENDING_upb_DefPool_Free, [:DefPool], :void - attach_function :create_DescriptorPool, :API_PENDING_upb_DefPool_New, [], :DefPool + attach_function :free_descriptor_pool, :API_PENDING_upb_DefPool_Free, [:DefPool], :void + attach_function :create_descriptor_pool,:API_PENDING_upb_DefPool_New, [], :DefPool attach_function :lookup_enum, :API_PENDING_upb_DefPool_FindEnumByName, [:DefPool, :string], EnumDef attach_function :lookup_msg, :API_PENDING_upb_DefPool_FindMessageByName, [:DefPool, :string], MessageDef diff --git a/ruby/lib/google/protobuf_ffi.rb b/ruby/lib/google/protobuf_ffi.rb index 99f440befff1..a8fd7a379adc 100644 --- a/ruby/lib/google/protobuf_ffi.rb +++ b/ruby/lib/google/protobuf_ffi.rb @@ -28,7 +28,7 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -require 'ffi' +require 'ffi-compiler/loader' require 'google/protobuf/ffi/internal/type_safety' require 'google/protobuf/ffi/internal/pointer_helper' require 'google/protobuf/ffi/internal/arena' diff --git a/ruby/tests/BUILD.bazel b/ruby/tests/BUILD.bazel index c072cc713647..01427ef9ce80 100644 --- a/ruby/tests/BUILD.bazel +++ b/ruby/tests/BUILD.bazel @@ -14,6 +14,15 @@ filegroup( ], ) +ruby_test( + name = "backend", + srcs = ["backend.rb"], + deps = [ + "//ruby:protobuf", + "@protobuf_bundle//:test-unit", + ], +) + ruby_test( name = "basic", srcs = ["basic.rb"], diff --git a/ruby/tests/backend.rb b/ruby/tests/backend.rb new file mode 100644 index 000000000000..3fe366333d2e --- /dev/null +++ b/ruby/tests/backend.rb @@ -0,0 +1,11 @@ +require 'google/protobuf' +require 'test/unit' + +# Verifies that the implementation of Protobuf is the expected (preferred) one. +# See protobuf.rb for the logic that defines PREFER_FFI. +class BackendTest < Test::Unit::TestCase + EXPECTED_IMPLEMENTATION = Google::Protobuf::PREFER_FFI ? :FFI : :NATIVE + def test_ffi_implementation + assert_equal EXPECTED_IMPLEMENTATION, Google::Protobuf::IMPLEMENTATION + end +end From 2c8d0ef6cd820ea9925d183a57e760f2d6c9fefa Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Fri, 14 Jul 2023 10:54:16 -0400 Subject: [PATCH 48/54] Use the FFI-fix branch of rules_ruby. --- protobuf_deps.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/protobuf_deps.bzl b/protobuf_deps.bzl index a1bf349ba916..e3cd4e1195e4 100644 --- a/protobuf_deps.bzl +++ b/protobuf_deps.bzl @@ -110,8 +110,8 @@ def protobuf_deps(): _github_archive( name = "rules_ruby", repo = "https://github.com/protocolbuffers/rules_ruby", - commit = "5cf6ff74161d7f985b9bf86bb3c5fb16cef6337b", - sha256 = "c88dd69eb50fcfd7fbc5d7db79adc6631ef0e1d80b3c94efe33ac5ee3ccc37f7", + commit = "b7f3e9756f3c45527be27bc38840d5a1ba690436", + sha256 = "347927fd8de6132099fcdc58e8f7eab7bde4eb2fd424546b9cd4f1c6f8f8bad8", ) if not native.existing_rule("rules_jvm_external"): From 2a1cf7c71566c03ed3c81857f249728c934ac310 Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Fri, 14 Jul 2023 11:15:28 -0400 Subject: [PATCH 49/54] Use backwards compatible range syntax. --- ruby/lib/google/protobuf/ffi/enum_descriptor.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ruby/lib/google/protobuf/ffi/enum_descriptor.rb b/ruby/lib/google/protobuf/ffi/enum_descriptor.rb index b31c4bbc066d..4cd3c2c64093 100644 --- a/ruby/lib/google/protobuf/ffi/enum_descriptor.rb +++ b/ruby/lib/google/protobuf/ffi/enum_descriptor.rb @@ -155,7 +155,7 @@ def self.resolve(name) self.each do |name, value| if name[0] < 'A' || name[0] > 'Z' if name[0] >= 'a' and name[0] <= 'z' - name = name[0].upcase + name[1..] # auto capitalize + name = name[0].upcase + name[1..-1] # auto capitalize else warn( "Enum value '#{name}' does not start with an uppercase letter " + From e28e256bd60a6b807f0b679014367f4875d9d3ca Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Fri, 14 Jul 2023 23:52:30 -0400 Subject: [PATCH 50/54] Move most calls to `attach_function` out of `ffi.rb`. Allows reordering of `require` statements in `protobuf_ffi.b` so that failed attempts load `ffi/ffi.rb` can be recovered in `protobuf.rb` without having to undo constant definitions when falling back from FFI to native implementations. --- ruby/lib/google/protobuf/ffi/descriptor.rb | 13 ++ .../google/protobuf/ffi/descriptor_pool.rb | 10 ++ .../google/protobuf/ffi/enum_descriptor.rb | 12 ++ ruby/lib/google/protobuf/ffi/ffi.rb | 119 ------------------ .../google/protobuf/ffi/field_descriptor.rb | 24 ++++ .../google/protobuf/ffi/file_descriptor.rb | 6 + .../lib/google/protobuf/ffi/internal/arena.rb | 10 ++ ruby/lib/google/protobuf/ffi/map.rb | 15 +++ ruby/lib/google/protobuf/ffi/message.rb | 20 ++- .../google/protobuf/ffi/oneof_descriptor.rb | 15 +++ .../lib/google/protobuf/ffi/repeated_field.rb | 10 ++ ruby/lib/google/protobuf_ffi.rb | 4 +- 12 files changed, 136 insertions(+), 122 deletions(-) diff --git a/ruby/lib/google/protobuf/ffi/descriptor.rb b/ruby/lib/google/protobuf/ffi/descriptor.rb index 31a4596ded7b..a9883f619a24 100644 --- a/ruby/lib/google/protobuf/ffi/descriptor.rb +++ b/ruby/lib/google/protobuf/ffi/descriptor.rb @@ -160,5 +160,18 @@ def pool @descriptor_pool end end + + class FFI + # MessageDef + attach_function :new_message_from_def, :upb_Message_New, [Descriptor, Internal::Arena], :Message + attach_function :field_count, :API_PENDING_upb_MessageDef_FieldCount, [Descriptor], :int + attach_function :get_message_file_def, :API_PENDING_upb_MessageDef_File, [:pointer], :FileDef + attach_function :get_message_fullname, :API_PENDING_upb_MessageDef_FullName, [Descriptor], :string + attach_function :get_mini_table, :API_PENDING_upb_MessageDef_MiniTable, [Descriptor], MiniTable.ptr + attach_function :oneof_count, :API_PENDING_upb_MessageDef_OneofCount, [Descriptor], :int + attach_function :get_well_known_type, :API_PENDING_upb_MessageDef_WellKnownType, [Descriptor], WellKnown + attach_function :message_def_syntax, :API_PENDING_upb_MessageDef_Syntax, [Descriptor], Syntax + attach_function :find_msg_def_by_name, :API_PENDING_upb_MessageDef_FindByNameWithSize, [Descriptor, :string, :size_t, :FieldDefPointer, :OneofDefPointer], :bool + end end end diff --git a/ruby/lib/google/protobuf/ffi/descriptor_pool.rb b/ruby/lib/google/protobuf/ffi/descriptor_pool.rb index 74e6c857c2bc..c7152c8ba592 100644 --- a/ruby/lib/google/protobuf/ffi/descriptor_pool.rb +++ b/ruby/lib/google/protobuf/ffi/descriptor_pool.rb @@ -30,6 +30,16 @@ module Google module Protobuf + class FFI + # DefPool + attach_function :add_serialized_file, :API_PENDING_upb_DefPool_AddFile, [:DefPool, :FileDescriptorProto, Status.by_ref], :FileDef + attach_function :free_descriptor_pool, :API_PENDING_upb_DefPool_Free, [:DefPool], :void + attach_function :create_descriptor_pool,:API_PENDING_upb_DefPool_New, [], :DefPool + attach_function :lookup_enum, :API_PENDING_upb_DefPool_FindEnumByName, [:DefPool, :string], EnumDescriptor + attach_function :lookup_msg, :API_PENDING_upb_DefPool_FindMessageByName, [:DefPool, :string], Descriptor + # FileDescriptorProto + attach_function :parse, :FileDescriptorProto_parse, [:binary_string, :size_t], :FileDescriptorProto + end class DescriptorPool attr :descriptor_pool attr_accessor :descriptor_class_by_def diff --git a/ruby/lib/google/protobuf/ffi/enum_descriptor.rb b/ruby/lib/google/protobuf/ffi/enum_descriptor.rb index 4cd3c2c64093..0ced5bfae8f0 100644 --- a/ruby/lib/google/protobuf/ffi/enum_descriptor.rb +++ b/ruby/lib/google/protobuf/ffi/enum_descriptor.rb @@ -168,5 +168,17 @@ def self.resolve(name) dynamic_module end end + + class FFI + # EnumDescriptor + attach_function :get_enum_file_descriptor, :API_PENDING_upb_EnumDef_File, [EnumDescriptor], :FileDef + attach_function :enum_value_by_name, :API_PENDING_upb_EnumDef_FindValueByNameWithSize,[EnumDescriptor, :string, :size_t], :EnumValueDef + attach_function :enum_value_by_number, :API_PENDING_upb_EnumDef_FindValueByNumber, [EnumDescriptor, :int], :EnumValueDef + attach_function :get_enum_fullname, :API_PENDING_upb_EnumDef_FullName, [EnumDescriptor], :string + attach_function :enum_value_by_index, :API_PENDING_upb_EnumDef_Value, [EnumDescriptor, :int], :EnumValueDef + attach_function :enum_value_count, :API_PENDING_upb_EnumDef_ValueCount, [EnumDescriptor], :int + attach_function :enum_name, :API_PENDING_upb_EnumValueDef_Name, [:EnumValueDef], :string + attach_function :enum_number, :API_PENDING_upb_EnumValueDef_Number, [:EnumValueDef], :int + end end end diff --git a/ruby/lib/google/protobuf/ffi/ffi.rb b/ruby/lib/google/protobuf/ffi/ffi.rb index 8b7471995bf4..69c57b955ab2 100644 --- a/ruby/lib/google/protobuf/ffi/ffi.rb +++ b/ruby/lib/google/protobuf/ffi/ffi.rb @@ -59,12 +59,6 @@ class FFI ## JSON Decoding options Upb_JsonDecode_IgnoreUnknown = 1 - Arena = Google::Protobuf::Internal::Arena - MessageDef = Google::Protobuf::Descriptor - EnumDef = Google::Protobuf::EnumDescriptor - FieldDef = Google::Protobuf::FieldDescriptor - OneofDef = Google::Protobuf::OneofDescriptor - typedef :pointer, :Array typedef :pointer, :DefPool typedef :pointer, :EnumValueDef @@ -225,119 +219,6 @@ class MutableMessageValue < ::FFI::Union :array, :Array end - # Arena - attach_function :create_arena, :Arena_create, [], Arena - attach_function :fuse_arena, :upb_Arena_Fuse, [Arena, Arena], :bool - # Argument takes a :pointer rather than a typed Arena here due to - # implementation details of FFI::AutoPointer. - attach_function :free_arena, :upb_Arena_Free, [:pointer], :void - attach_function :arena_malloc, :upb_Arena_Malloc, [Arena, :size_t], :pointer - - # Array - attach_function :append_array, :upb_Array_Append, [:Array, MessageValue.by_value, Arena], :bool - attach_function :get_msgval_at,:upb_Array_Get, [:Array, :size_t], MessageValue.by_value - attach_function :create_array, :upb_Array_New, [Arena, CType], :Array - attach_function :array_resize, :upb_Array_Resize, [:Array, :size_t, Arena], :bool - attach_function :array_set, :upb_Array_Set, [:Array, :size_t, MessageValue.by_value], :void - attach_function :array_size, :upb_Array_Size, [:Array], :size_t - - # DefPool - attach_function :add_serialized_file, :API_PENDING_upb_DefPool_AddFile, [:DefPool, :FileDescriptorProto, Status.by_ref], :FileDef - attach_function :free_descriptor_pool, :API_PENDING_upb_DefPool_Free, [:DefPool], :void - attach_function :create_descriptor_pool,:API_PENDING_upb_DefPool_New, [], :DefPool - attach_function :lookup_enum, :API_PENDING_upb_DefPool_FindEnumByName, [:DefPool, :string], EnumDef - attach_function :lookup_msg, :API_PENDING_upb_DefPool_FindMessageByName, [:DefPool, :string], MessageDef - - # EnumDescriptor - attach_function :get_enum_file_descriptor, :API_PENDING_upb_EnumDef_File, [EnumDef], :FileDef - attach_function :enum_value_by_name, :API_PENDING_upb_EnumDef_FindValueByNameWithSize,[EnumDef, :string, :size_t], :EnumValueDef - attach_function :enum_value_by_number, :API_PENDING_upb_EnumDef_FindValueByNumber, [EnumDef, :int], :EnumValueDef - attach_function :get_enum_fullname, :API_PENDING_upb_EnumDef_FullName, [EnumDef], :string - attach_function :enum_value_by_index, :API_PENDING_upb_EnumDef_Value, [EnumDef, :int], :EnumValueDef - attach_function :enum_value_count, :API_PENDING_upb_EnumDef_ValueCount, [EnumDef], :int - attach_function :enum_name, :API_PENDING_upb_EnumValueDef_Name, [:EnumValueDef], :string - attach_function :enum_number, :API_PENDING_upb_EnumValueDef_Number, [:EnumValueDef], :int - - # FileDescriptor - attach_function :file_def_name, :API_PENDING_upb_FileDef_Name, [:FileDef], :string - attach_function :file_def_syntax, :API_PENDING_upb_FileDef_Syntax, [:FileDef], Syntax - attach_function :file_def_pool, :API_PENDING_upb_FileDef_Pool, [:FileDef], :DefPool - - # FileDescriptorProto - attach_function :parse, :FileDescriptorProto_parse, [:binary_string, :size_t], :FileDescriptorProto - - # FieldDescriptor - attach_function :get_containing_message_def, :API_PENDING_upb_FieldDef_ContainingType, [FieldDescriptor], MessageDef - attach_function :get_c_type, :API_PENDING_upb_FieldDef_CType, [FieldDescriptor], CType - attach_function :get_default, :API_PENDING_upb_FieldDef_Default, [FieldDescriptor], MessageValue.by_value - attach_function :get_subtype_as_enum, :API_PENDING_upb_FieldDef_EnumSubDef, [FieldDescriptor], EnumDef - attach_function :get_has_presence, :API_PENDING_upb_FieldDef_HasPresence, [FieldDescriptor], :bool - attach_function :is_map, :API_PENDING_upb_FieldDef_IsMap, [FieldDescriptor], :bool - attach_function :is_repeated, :API_PENDING_upb_FieldDef_IsRepeated, [FieldDescriptor], :bool - attach_function :is_sub_message, :API_PENDING_upb_FieldDef_IsSubMessage, [FieldDescriptor], :bool - attach_function :get_json_name, :API_PENDING_upb_FieldDef_JsonName, [FieldDescriptor], :string - attach_function :get_label, :API_PENDING_upb_FieldDef_Label, [FieldDescriptor], Label - attach_function :get_subtype_as_message, :API_PENDING_upb_FieldDef_MessageSubDef, [FieldDescriptor], MessageDef - attach_function :get_full_name, :API_PENDING_upb_FieldDef_Name, [FieldDescriptor], :string - attach_function :get_number, :API_PENDING_upb_FieldDef_Number, [FieldDescriptor], :uint32_t - attach_function :real_containing_oneof, :API_PENDING_upb_FieldDef_RealContainingOneof,[FieldDescriptor], OneofDef - attach_function :get_type, :API_PENDING_upb_FieldDef_Type, [FieldDescriptor], FieldType - attach_function :file_def_by_raw_field_def, :API_PENDING_upb_FieldDef_File, [:pointer], :FileDef - # Map - attach_function :map_clear, :upb_Map_Clear, [:Map], :void - attach_function :map_delete, :upb_Map_Delete, [:Map, MessageValue.by_value, MessageValue.by_ref], :bool - attach_function :map_get, :upb_Map_Get, [:Map, MessageValue.by_value, MessageValue.by_ref], :bool - attach_function :create_map, :upb_Map_New, [Arena, CType, CType], :Map - attach_function :map_size, :upb_Map_Size, [:Map], :size_t - attach_function :map_set, :upb_Map_Set, [:Map, MessageValue.by_value, MessageValue.by_value, Arena], :bool - - # MapIterator - attach_function :map_next, :API_PENDING_upb_MapIterator_Next, [:Map, :pointer], :bool - attach_function :map_done, :API_PENDING_upb_MapIterator_Done, [:Map, :size_t], :bool - attach_function :map_key, :API_PENDING_upb_MapIterator_Key, [:Map, :size_t], MessageValue.by_value - attach_function :map_value, :API_PENDING_upb_MapIterator_Value, [:Map, :size_t], MessageValue.by_value - - # MessageDef - attach_function :new_message_from_def, :upb_Message_New, [MessageDef, Arena], :Message - attach_function :get_field_by_index, :API_PENDING_upb_MessageDef_Field, [MessageDef, :int], FieldDescriptor - attach_function :field_count, :API_PENDING_upb_MessageDef_FieldCount, [MessageDef], :int - attach_function :get_message_file_def, :API_PENDING_upb_MessageDef_File, [:pointer], :FileDef - attach_function :get_field_by_name, :API_PENDING_upb_MessageDef_FindFieldByNameWithSize, [MessageDef, :string, :size_t], FieldDescriptor - attach_function :get_field_by_number, :API_PENDING_upb_MessageDef_FindFieldByNumber, [MessageDef, :uint32_t], FieldDescriptor - attach_function :get_oneof_by_name, :API_PENDING_upb_MessageDef_FindOneofByNameWithSize, [MessageDef, :string, :size_t], OneofDef - attach_function :get_message_fullname, :API_PENDING_upb_MessageDef_FullName, [MessageDef], :string - attach_function :get_mini_table, :API_PENDING_upb_MessageDef_MiniTable, [MessageDef], MiniTable.ptr - attach_function :get_oneof_by_index, :API_PENDING_upb_MessageDef_Oneof, [MessageDef, :int], OneofDef - attach_function :oneof_count, :API_PENDING_upb_MessageDef_OneofCount, [MessageDef], :int - attach_function :get_well_known_type, :API_PENDING_upb_MessageDef_WellKnownType, [MessageDef], WellKnown - attach_function :message_def_syntax, :API_PENDING_upb_MessageDef_Syntax, [MessageDef], Syntax - attach_function :find_msg_def_by_name, :API_PENDING_upb_MessageDef_FindByNameWithSize, [MessageDef, :string, :size_t, :FieldDefPointer, :OneofDefPointer], :bool - - # Message - attach_function :clear_message_field, :API_PENDING_upb_Message_ClearFieldByDef, [:Message, FieldDescriptor], :void - attach_function :get_message_value, :API_PENDING_upb_Message_GetFieldByDef, [:Message, FieldDescriptor], MessageValue.by_value - attach_function :get_message_has, :API_PENDING_upb_Message_HasFieldByDef, [:Message, FieldDescriptor], :bool - attach_function :set_message_field, :API_PENDING_upb_Message_SetFieldByDef, [:Message, FieldDescriptor, MessageValue.by_value, Arena], :bool - attach_function :encode_message, :API_PENDING_upb_Encode, [:Message, MiniTable.by_ref, :size_t, Arena, :pointer, :pointer], EncodeStatus - attach_function :json_decode_message, :API_PENDING_upb_JsonDecode, [:binary_string, :size_t, :Message, MessageDef, :DefPool, :int, Arena, Status.by_ref], :bool - attach_function :json_encode_message, :API_PENDING_upb_JsonEncode, [:Message, MessageDef, :DefPool, :int, :binary_string, :size_t, Status.by_ref], :size_t - attach_function :decode_message, :upb_Decode, [:binary_string, :size_t, :Message, MiniTable.by_ref, :ExtensionRegistry, :int, Arena], DecodeStatus - attach_function :get_mutable_message, :API_PENDING_upb_Message_Mutable, [:Message, FieldDescriptor, Arena], MutableMessageValue.by_value - attach_function :get_message_which_oneof, :API_PENDING_upb_Message_WhichOneof, [:Message, OneofDef], FieldDescriptor - attach_function :message_discard_unknown, :API_PENDING_upb_Message_DiscardUnknown, [:Message, MessageDef, :int], :bool - - # MessageValue - attach_function :message_value_equal, :shared_Msgval_IsEqual, [MessageValue.by_value, MessageValue.by_value, CType, MessageDef], :bool - attach_function :message_value_hash, :shared_Msgval_GetHash, [MessageValue.by_value, CType, MessageDef, :uint64_t], :uint64_t - - # OneofDescriptor - attach_function :get_oneof_name, :API_PENDING_upb_OneofDef_Name, [OneofDef], :string - attach_function :get_oneof_field_count, :API_PENDING_upb_OneofDef_FieldCount, [OneofDef], :int - attach_function :get_oneof_field_by_index, :API_PENDING_upb_OneofDef_Field, [OneofDef, :int], FieldDescriptor - attach_function :get_oneof_containing_type,:API_PENDING_upb_OneofDef_ContainingType,[:pointer], MessageDef - - # RepeatableField - # Status attach_function :clear, :upb_Status_Clear, [Status.by_ref], :void attach_function :error_message, :upb_Status_ErrorMessage, [Status.by_ref], :string diff --git a/ruby/lib/google/protobuf/ffi/field_descriptor.rb b/ruby/lib/google/protobuf/ffi/field_descriptor.rb index 1424b8d4f7e7..a799de33b657 100644 --- a/ruby/lib/google/protobuf/ffi/field_descriptor.rb +++ b/ruby/lib/google/protobuf/ffi/field_descriptor.rb @@ -304,5 +304,29 @@ def c_type @c_type ||= Google::Protobuf::FFI.get_c_type(self) end end + + class FFI + # MessageDef + attach_function :get_field_by_index, :API_PENDING_upb_MessageDef_Field, [Descriptor, :int], FieldDescriptor + attach_function :get_field_by_name, :API_PENDING_upb_MessageDef_FindFieldByNameWithSize,[Descriptor, :string, :size_t], FieldDescriptor + attach_function :get_field_by_number, :API_PENDING_upb_MessageDef_FindFieldByNumber, [Descriptor, :uint32_t], FieldDescriptor + + # FieldDescriptor + attach_function :get_containing_message_def, :API_PENDING_upb_FieldDef_ContainingType, [FieldDescriptor], Descriptor + attach_function :get_c_type, :API_PENDING_upb_FieldDef_CType, [FieldDescriptor], CType + attach_function :get_default, :API_PENDING_upb_FieldDef_Default, [FieldDescriptor], MessageValue.by_value + attach_function :get_subtype_as_enum, :API_PENDING_upb_FieldDef_EnumSubDef, [FieldDescriptor], EnumDescriptor + attach_function :get_has_presence, :API_PENDING_upb_FieldDef_HasPresence, [FieldDescriptor], :bool + attach_function :is_map, :API_PENDING_upb_FieldDef_IsMap, [FieldDescriptor], :bool + attach_function :is_repeated, :API_PENDING_upb_FieldDef_IsRepeated, [FieldDescriptor], :bool + attach_function :is_sub_message, :API_PENDING_upb_FieldDef_IsSubMessage, [FieldDescriptor], :bool + attach_function :get_json_name, :API_PENDING_upb_FieldDef_JsonName, [FieldDescriptor], :string + attach_function :get_label, :API_PENDING_upb_FieldDef_Label, [FieldDescriptor], Label + attach_function :get_subtype_as_message, :API_PENDING_upb_FieldDef_MessageSubDef, [FieldDescriptor], Descriptor + attach_function :get_full_name, :API_PENDING_upb_FieldDef_Name, [FieldDescriptor], :string + attach_function :get_number, :API_PENDING_upb_FieldDef_Number, [FieldDescriptor], :uint32_t + attach_function :get_type, :API_PENDING_upb_FieldDef_Type, [FieldDescriptor], FieldType + attach_function :file_def_by_raw_field_def, :API_PENDING_upb_FieldDef_File, [:pointer], :FileDef + end end end diff --git a/ruby/lib/google/protobuf/ffi/file_descriptor.rb b/ruby/lib/google/protobuf/ffi/file_descriptor.rb index d19838f5e470..9c14f775b5fb 100644 --- a/ruby/lib/google/protobuf/ffi/file_descriptor.rb +++ b/ruby/lib/google/protobuf/ffi/file_descriptor.rb @@ -30,6 +30,12 @@ module Google module Protobuf + class FFI + # FileDescriptor + attach_function :file_def_name, :API_PENDING_upb_FileDef_Name, [:FileDef], :string + attach_function :file_def_syntax, :API_PENDING_upb_FileDef_Syntax, [:FileDef], Syntax + attach_function :file_def_pool, :API_PENDING_upb_FileDef_Pool, [:FileDef], :DefPool + end class FileDescriptor attr :descriptor_pool, :file_def diff --git a/ruby/lib/google/protobuf/ffi/internal/arena.rb b/ruby/lib/google/protobuf/ffi/internal/arena.rb index 946176e2e367..757a84fc9093 100644 --- a/ruby/lib/google/protobuf/ffi/internal/arena.rb +++ b/ruby/lib/google/protobuf/ffi/internal/arena.rb @@ -69,5 +69,15 @@ def fuse(other_arena) end end end + + class FFI + # Arena + attach_function :create_arena, :Arena_create, [], Internal::Arena + attach_function :fuse_arena, :upb_Arena_Fuse, [Internal::Arena, Internal::Arena], :bool + # Argument takes a :pointer rather than a typed Arena here due to + # implementation details of FFI::AutoPointer. + attach_function :free_arena, :upb_Arena_Free, [:pointer], :void + attach_function :arena_malloc, :upb_Arena_Malloc, [Internal::Arena, :size_t], :pointer + end end end diff --git a/ruby/lib/google/protobuf/ffi/map.rb b/ruby/lib/google/protobuf/ffi/map.rb index 51c87387a6bf..a9f896be289c 100644 --- a/ruby/lib/google/protobuf/ffi/map.rb +++ b/ruby/lib/google/protobuf/ffi/map.rb @@ -30,6 +30,21 @@ module Google module Protobuf + class FFI + # Map + attach_function :map_clear, :upb_Map_Clear, [:Map], :void + attach_function :map_delete, :upb_Map_Delete, [:Map, MessageValue.by_value, MessageValue.by_ref], :bool + attach_function :map_get, :upb_Map_Get, [:Map, MessageValue.by_value, MessageValue.by_ref], :bool + attach_function :create_map, :upb_Map_New, [Internal::Arena, CType, CType], :Map + attach_function :map_size, :upb_Map_Size, [:Map], :size_t + attach_function :map_set, :upb_Map_Set, [:Map, MessageValue.by_value, MessageValue.by_value, Internal::Arena], :bool + + # MapIterator + attach_function :map_next, :API_PENDING_upb_MapIterator_Next, [:Map, :pointer], :bool + attach_function :map_done, :API_PENDING_upb_MapIterator_Done, [:Map, :size_t], :bool + attach_function :map_key, :API_PENDING_upb_MapIterator_Key, [:Map, :size_t], MessageValue.by_value + attach_function :map_value, :API_PENDING_upb_MapIterator_Value, [:Map, :size_t], MessageValue.by_value + end class Map include Enumerable ## diff --git a/ruby/lib/google/protobuf/ffi/message.rb b/ruby/lib/google/protobuf/ffi/message.rb index 15083352cbc1..57e827b69aef 100644 --- a/ruby/lib/google/protobuf/ffi/message.rb +++ b/ruby/lib/google/protobuf/ffi/message.rb @@ -29,10 +29,28 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# Decorates Descriptor with the `build_messsage_class` method that defines +# Decorates Descriptor with the `build_message_class` method that defines # Message classes. module Google module Protobuf + class FFI + # Message + attach_function :clear_message_field, :API_PENDING_upb_Message_ClearFieldByDef, [:Message, FieldDescriptor], :void + attach_function :get_message_value, :API_PENDING_upb_Message_GetFieldByDef, [:Message, FieldDescriptor], MessageValue.by_value + attach_function :get_message_has, :API_PENDING_upb_Message_HasFieldByDef, [:Message, FieldDescriptor], :bool + attach_function :set_message_field, :API_PENDING_upb_Message_SetFieldByDef, [:Message, FieldDescriptor, MessageValue.by_value, Internal::Arena], :bool + attach_function :encode_message, :API_PENDING_upb_Encode, [:Message, MiniTable.by_ref, :size_t, Internal::Arena, :pointer, :pointer], EncodeStatus + attach_function :json_decode_message, :API_PENDING_upb_JsonDecode, [:binary_string, :size_t, :Message, Descriptor, :DefPool, :int, Internal::Arena, Status.by_ref], :bool + attach_function :json_encode_message, :API_PENDING_upb_JsonEncode, [:Message, Descriptor, :DefPool, :int, :binary_string, :size_t, Status.by_ref], :size_t + attach_function :decode_message, :upb_Decode, [:binary_string, :size_t, :Message, MiniTable.by_ref, :ExtensionRegistry, :int, Internal::Arena], DecodeStatus + attach_function :get_mutable_message, :API_PENDING_upb_Message_Mutable, [:Message, FieldDescriptor, Internal::Arena], MutableMessageValue.by_value + attach_function :get_message_which_oneof, :API_PENDING_upb_Message_WhichOneof, [:Message, OneofDescriptor], FieldDescriptor + attach_function :message_discard_unknown, :API_PENDING_upb_Message_DiscardUnknown, [:Message, Descriptor, :int], :bool + # MessageValue + attach_function :message_value_equal, :shared_Msgval_IsEqual, [MessageValue.by_value, MessageValue.by_value, CType, Descriptor], :bool + attach_function :message_value_hash, :shared_Msgval_GetHash, [MessageValue.by_value, CType, Descriptor, :uint64_t], :uint64_t + end + class Descriptor def build_message_class descriptor = self diff --git a/ruby/lib/google/protobuf/ffi/oneof_descriptor.rb b/ruby/lib/google/protobuf/ffi/oneof_descriptor.rb index 5a622f98b05c..44eacf296442 100644 --- a/ruby/lib/google/protobuf/ffi/oneof_descriptor.rb +++ b/ruby/lib/google/protobuf/ffi/oneof_descriptor.rb @@ -92,5 +92,20 @@ def self.private_constructor(oneof_def, descriptor_pool) instance end end + + class FFI + # MessageDef + attach_function :get_oneof_by_name, :API_PENDING_upb_MessageDef_FindOneofByNameWithSize, [Descriptor, :string, :size_t], OneofDescriptor + attach_function :get_oneof_by_index, :API_PENDING_upb_MessageDef_Oneof, [Descriptor, :int], OneofDescriptor + + # OneofDescriptor + attach_function :get_oneof_name, :API_PENDING_upb_OneofDef_Name, [OneofDescriptor], :string + attach_function :get_oneof_field_count, :API_PENDING_upb_OneofDef_FieldCount, [OneofDescriptor], :int + attach_function :get_oneof_field_by_index, :API_PENDING_upb_OneofDef_Field, [OneofDescriptor, :int], FieldDescriptor + attach_function :get_oneof_containing_type,:API_PENDING_upb_OneofDef_ContainingType,[:pointer], Descriptor + + # FieldDescriptor + attach_function :real_containing_oneof, :API_PENDING_upb_FieldDef_RealContainingOneof,[FieldDescriptor], OneofDescriptor + end end end diff --git a/ruby/lib/google/protobuf/ffi/repeated_field.rb b/ruby/lib/google/protobuf/ffi/repeated_field.rb index 5fe1c39bb107..0b5a67d00532 100644 --- a/ruby/lib/google/protobuf/ffi/repeated_field.rb +++ b/ruby/lib/google/protobuf/ffi/repeated_field.rb @@ -47,6 +47,16 @@ # module Google module Protobuf + class FFI + # Array + attach_function :append_array, :upb_Array_Append, [:Array, MessageValue.by_value, Internal::Arena], :bool + attach_function :get_msgval_at,:upb_Array_Get, [:Array, :size_t], MessageValue.by_value + attach_function :create_array, :upb_Array_New, [Internal::Arena, CType], :Array + attach_function :array_resize, :upb_Array_Resize, [:Array, :size_t, Internal::Arena], :bool + attach_function :array_set, :upb_Array_Set, [:Array, :size_t, MessageValue.by_value], :void + attach_function :array_size, :upb_Array_Size, [:Array], :size_t + end + class RepeatedField extend Forwardable # NOTE: using delegators rather than method_missing to make the diff --git a/ruby/lib/google/protobuf_ffi.rb b/ruby/lib/google/protobuf_ffi.rb index a8fd7a379adc..22219c179d39 100644 --- a/ruby/lib/google/protobuf_ffi.rb +++ b/ruby/lib/google/protobuf_ffi.rb @@ -29,21 +29,21 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. require 'ffi-compiler/loader' +require 'google/protobuf/ffi/ffi' require 'google/protobuf/ffi/internal/type_safety' require 'google/protobuf/ffi/internal/pointer_helper' require 'google/protobuf/ffi/internal/arena' require 'google/protobuf/ffi/internal/convert' require 'google/protobuf/ffi/descriptor' -require 'google/protobuf/ffi/message' require 'google/protobuf/ffi/enum_descriptor' require 'google/protobuf/ffi/field_descriptor' require 'google/protobuf/ffi/oneof_descriptor' -require 'google/protobuf/ffi/ffi' require 'google/protobuf/ffi/descriptor_pool' require 'google/protobuf/ffi/file_descriptor' require 'google/protobuf/ffi/map' require 'google/protobuf/ffi/object_cache' require 'google/protobuf/ffi/repeated_field' +require 'google/protobuf/ffi/message' require 'google/protobuf/descriptor_dsl' module Google From 55f28843ee8bdb74eaa426c1371e998e9cdc6e49 Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Fri, 14 Jul 2023 23:53:38 -0400 Subject: [PATCH 51/54] Breakout conformance tests by platform and whether FFI is enabled. --- conformance/BUILD.bazel | 6 ++- conformance/failure_list_jruby_ffi.txt | 2 + ruby/BUILD.bazel | 58 +++++++++++++++++++++++++- 3 files changed, 62 insertions(+), 4 deletions(-) create mode 100644 conformance/failure_list_jruby_ffi.txt diff --git a/conformance/BUILD.bazel b/conformance/BUILD.bazel index dde52022816a..c4955042cbce 100644 --- a/conformance/BUILD.bazel +++ b/conformance/BUILD.bazel @@ -24,6 +24,7 @@ exports_files([ "failure_list_python_cpp.txt", "failure_list_ruby.txt", "failure_list_jruby.txt", + "failure_list_jruby_ffi.txt", "text_format_failure_list_cpp.txt", "text_format_failure_list_csharp.txt", "text_format_failure_list_java.txt", @@ -34,6 +35,7 @@ exports_files([ "text_format_failure_list_python_cpp.txt", "text_format_failure_list_ruby.txt", "text_format_failure_list_jruby.txt", + "text_format_failure_list_jruby_ffi.txt", ]) cc_proto_library( @@ -326,12 +328,12 @@ ruby_binary( name = "conformance_ruby", testonly = True, srcs = ["conformance_ruby.rb"], + visibility = ["//ruby:__subpackages__"], deps = [ ":conformance_ruby_proto", "//:test_messages_proto2_ruby_proto", - "//:test_messages_proto3_ruby_proto", + "//:test_messages_proto3_ruby_proto", ], - visibility = ["//ruby:__subpackages__"], ) ################################################################################ diff --git a/conformance/failure_list_jruby_ffi.txt b/conformance/failure_list_jruby_ffi.txt new file mode 100644 index 000000000000..6a487cb293a9 --- /dev/null +++ b/conformance/failure_list_jruby_ffi.txt @@ -0,0 +1,2 @@ +Recommended.Proto3.JsonInput.IgnoreUnknownEnumStringValueInMapValue.ProtobufOutput +Recommended.Proto3.JsonInput.IgnoreUnknownEnumStringValueInRepeatedField.ProtobufOutput \ No newline at end of file diff --git a/ruby/BUILD.bazel b/ruby/BUILD.bazel index f3e950dcf55c..f6c4fbf602bd 100755 --- a/ruby/BUILD.bazel +++ b/ruby/BUILD.bazel @@ -38,6 +38,38 @@ config_setting( }, ) +selects.config_setting_group( + name = "jruby_ffi", + match_all = [ + ":ffi_enabled", + "@rules_ruby//ruby/runtime:config_jruby", + ], +) + +selects.config_setting_group( + name = "jruby_native", + match_all = [ + ":ffi_disabled", + "@rules_ruby//ruby/runtime:config_jruby", + ], +) + +selects.config_setting_group( + name = "ruby_ffi", + match_all = [ + ":ffi_enabled", + "@rules_ruby//ruby/runtime:config_ruby", + ], +) + +selects.config_setting_group( + name = "ruby_native", + match_all = [ + ":ffi_disabled", + "@rules_ruby//ruby/runtime:config_ruby", + ], +) + ruby_library( name = "protobuf", visibility = [ @@ -150,7 +182,18 @@ conformance_test( name = "conformance_test", failure_list = "//conformance:failure_list_ruby.txt", target_compatible_with = select({ - "@rules_ruby//ruby/runtime:config_ruby": [], + ":ruby_native": [], + "//conditions:default": ["@platforms//:incompatible"], + }), + testee = "//conformance:conformance_ruby", + text_format_failure_list = "//conformance:text_format_failure_list_ruby.txt", +) + +conformance_test( + name = "conformance_test_ffi", + failure_list = "//conformance:failure_list_ruby.txt", + target_compatible_with = select({ + ":ruby_ffi": [], "//conditions:default": ["@platforms//:incompatible"], }), testee = "//conformance:conformance_ruby", @@ -161,7 +204,18 @@ conformance_test( name = "conformance_test_jruby", failure_list = "//conformance:failure_list_jruby.txt", target_compatible_with = select({ - "@rules_ruby//ruby/runtime:config_jruby": [], + ":jruby_native": [], + "//conditions:default": ["@platforms//:incompatible"], + }), + testee = "//conformance:conformance_ruby", + text_format_failure_list = "//conformance:text_format_failure_list_jruby.txt", +) + +conformance_test( + name = "conformance_test_jruby_ffi", + failure_list = "//conformance:failure_list_jruby_ffi.txt", + target_compatible_with = select({ + ":jruby_ffi": [], "//conditions:default": ["@platforms//:incompatible"], }), testee = "//conformance:conformance_ruby", From 472b65331c57dfd3a5b2514c49d5639caa1ff772 Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Sun, 16 Jul 2023 17:27:11 -0400 Subject: [PATCH 52/54] Remove workaround for access to UPI API functions. --- ruby/ext/google/protobuf_c/glue.c | 222 ------------------ ruby/lib/google/protobuf/ffi/descriptor.rb | 16 +- .../google/protobuf/ffi/descriptor_pool.rb | 10 +- .../google/protobuf/ffi/enum_descriptor.rb | 16 +- .../google/protobuf/ffi/field_descriptor.rb | 36 +-- .../google/protobuf/ffi/file_descriptor.rb | 6 +- ruby/lib/google/protobuf/ffi/map.rb | 8 +- ruby/lib/google/protobuf/ffi/message.rb | 20 +- .../google/protobuf/ffi/oneof_descriptor.rb | 14 +- 9 files changed, 63 insertions(+), 285 deletions(-) diff --git a/ruby/ext/google/protobuf_c/glue.c b/ruby/ext/google/protobuf_c/glue.c index f1005b6bc994..16d53130d151 100644 --- a/ruby/ext/google/protobuf_c/glue.c +++ b/ruby/ext/google/protobuf_c/glue.c @@ -44,225 +44,3 @@ google_protobuf_FileDescriptorProto* FileDescriptorProto_parse (const char* seri serialized_file_proto, length, arena); } - -bool API_PENDING_upb_MapIterator_Next(const upb_Map* map, size_t* iter) { - return upb_MapIterator_Next(map, iter); -} -bool API_PENDING_upb_MapIterator_Done(const upb_Map* map, size_t iter) { - return upb_MapIterator_Done(map, iter); -} -upb_MessageValue API_PENDING_upb_MapIterator_Key(const upb_Map* map, size_t iter) { - return upb_MapIterator_Key(map, iter); -} -upb_MessageValue API_PENDING_upb_MapIterator_Value(const upb_Map* map, size_t iter) { - return upb_MapIterator_Value(map, iter); -} -upb_EncodeStatus API_PENDING_upb_Encode(const void* msg, const upb_MiniTable* l, - int options, upb_Arena* arena, char** buf, - size_t* size) { - return upb_Encode(msg, l, options, arena, buf, size); -} - -void API_PENDING_upb_DefPool_Free(upb_DefPool* s) { - return upb_DefPool_Free(s); -} - -upb_DefPool* API_PENDING_upb_DefPool_New(void) { - return upb_DefPool_New(); -} - -const upb_MessageDef* API_PENDING_upb_DefPool_FindMessageByName(const upb_DefPool* s, - const char* sym) { - return upb_DefPool_FindMessageByName(s, sym); -} - -const upb_EnumDef* API_PENDING_upb_DefPool_FindEnumByName(const upb_DefPool* s, - const char* sym) { - return upb_DefPool_FindEnumByName(s, sym); -} -const upb_FileDef* API_PENDING_upb_DefPool_AddFile(upb_DefPool* s, - void * - file_proto, - upb_Status* status) { - return upb_DefPool_AddFile(s, file_proto, status); -} -const upb_FileDef* API_PENDING_upb_EnumDef_File(const upb_EnumDef* e) { - return upb_EnumDef_File(e); -} -const upb_EnumValueDef* API_PENDING_upb_EnumDef_FindValueByNameWithSize( - const upb_EnumDef* e, const char* name, size_t size) { - return upb_EnumDef_FindValueByNameWithSize(e, name, size); -} -const upb_EnumValueDef* API_PENDING_upb_EnumDef_FindValueByNumber(const upb_EnumDef* e, - int32_t num) { - return upb_EnumDef_FindValueByNumber(e, num); -} -const char* API_PENDING_upb_EnumDef_FullName(const upb_EnumDef* e) { - return upb_EnumDef_FullName(e); -} -const upb_EnumValueDef* API_PENDING_upb_EnumDef_Value(const upb_EnumDef* e, int i) { - return upb_EnumDef_Value(e, i); -} -int API_PENDING_upb_EnumDef_ValueCount(const upb_EnumDef* e) { - return upb_EnumDef_ValueCount(e); -} -const char* API_PENDING_upb_EnumValueDef_Name(const upb_EnumValueDef* v) { - return upb_EnumValueDef_Name(v); -} -int32_t API_PENDING_upb_EnumValueDef_Number(const upb_EnumValueDef* v) { - return upb_EnumValueDef_Number(v); -} -const upb_MessageDef* API_PENDING_upb_FieldDef_ContainingType(const upb_FieldDef* f) { - return upb_FieldDef_ContainingType(f); -} -upb_CType API_PENDING_upb_FieldDef_CType(const upb_FieldDef* f) { - return upb_FieldDef_CType(f); -} -upb_MessageValue API_PENDING_upb_FieldDef_Default(const upb_FieldDef* f) { - return upb_FieldDef_Default(f); -} -const upb_EnumDef* API_PENDING_upb_FieldDef_EnumSubDef(const upb_FieldDef* f) { - return upb_FieldDef_EnumSubDef(f); -} -const upb_FileDef* API_PENDING_upb_FieldDef_File(const upb_FieldDef* f) { - return upb_FieldDef_File(f); -} -bool API_PENDING_upb_FieldDef_HasPresence(const upb_FieldDef* f) { - return upb_FieldDef_HasPresence(f); -} -bool API_PENDING_upb_FieldDef_IsMap(const upb_FieldDef* f) { - return upb_FieldDef_IsMap(f); -} -bool API_PENDING_upb_FieldDef_IsRepeated(const upb_FieldDef* f) { - return upb_FieldDef_IsRepeated(f); -} -bool API_PENDING_upb_FieldDef_IsSubMessage(const upb_FieldDef* f) { - return upb_FieldDef_IsSubMessage(f); -} -const char* API_PENDING_upb_FieldDef_JsonName(const upb_FieldDef* f) { - return upb_FieldDef_JsonName(f); -} -upb_Label API_PENDING_upb_FieldDef_Label(const upb_FieldDef* f) { - return upb_FieldDef_Label(f); -} -const upb_MessageDef* API_PENDING_upb_FieldDef_MessageSubDef(const upb_FieldDef* f) { - return upb_FieldDef_MessageSubDef(f); -} -const char* API_PENDING_upb_FieldDef_Name(const upb_FieldDef* f) { - return upb_FieldDef_Name(f); -} -uint32_t API_PENDING_upb_FieldDef_Number(const upb_FieldDef* f) { - return upb_FieldDef_Number(f); -} -const upb_OneofDef* API_PENDING_upb_FieldDef_RealContainingOneof(const upb_FieldDef* f) { - return upb_FieldDef_RealContainingOneof(f); -} -upb_FieldType API_PENDING_upb_FieldDef_Type(const upb_FieldDef* f) { - return upb_FieldDef_Type(f); -} -const char* API_PENDING_upb_FileDef_Name(const upb_FileDef* f) { - return upb_FileDef_Name(f); -} -const upb_DefPool* API_PENDING_upb_FileDef_Pool(const upb_FileDef* f) { - return upb_FileDef_Pool(f); -} -upb_Syntax API_PENDING_upb_FileDef_Syntax(const upb_FileDef* f) { - return upb_FileDef_Syntax(f); -} -const upb_FieldDef* API_PENDING_upb_MessageDef_Field(const upb_MessageDef* m, int i) { - return upb_MessageDef_Field(m, i); -} -int API_PENDING_upb_MessageDef_FieldCount(const upb_MessageDef* m) { - return upb_MessageDef_FieldCount(m); -} -const upb_FileDef* API_PENDING_upb_MessageDef_File(const upb_MessageDef* m) { - return upb_MessageDef_File(m); -} -bool API_PENDING_upb_MessageDef_FindByNameWithSize(const upb_MessageDef* m, - const char* name, size_t size, - const upb_FieldDef** f, - const upb_OneofDef** o) { - return upb_MessageDef_FindByNameWithSize(m, name, size, f, o); -} -const upb_FieldDef* API_PENDING_upb_MessageDef_FindFieldByNameWithSize( - const upb_MessageDef* m, const char* name, size_t size) { - return upb_MessageDef_FindFieldByNameWithSize(m, name, size); -} -const upb_FieldDef* API_PENDING_upb_MessageDef_FindFieldByNumber(const upb_MessageDef* m, - uint32_t i) { - return upb_MessageDef_FindFieldByNumber(m, i); -} -const upb_OneofDef* API_PENDING_upb_MessageDef_FindOneofByNameWithSize( - const upb_MessageDef* m, const char* name, size_t size) { - return upb_MessageDef_FindOneofByNameWithSize(m, name, size); -} -const char* API_PENDING_upb_MessageDef_FullName(const upb_MessageDef* m) { - - return upb_MessageDef_FullName(m); -} -const upb_MiniTable* API_PENDING_upb_MessageDef_MiniTable(const upb_MessageDef* m) { - return upb_MessageDef_MiniTable(m); -} -const upb_OneofDef* API_PENDING_upb_MessageDef_Oneof(const upb_MessageDef* m, int i) { - return upb_MessageDef_Oneof(m, i); -} -int API_PENDING_upb_MessageDef_OneofCount(const upb_MessageDef* m) { - return upb_MessageDef_OneofCount(m); -} -upb_Syntax API_PENDING_upb_MessageDef_Syntax(const upb_MessageDef* m) { - return upb_MessageDef_Syntax(m); -} -upb_WellKnown API_PENDING_upb_MessageDef_WellKnownType(const upb_MessageDef* m) { - return upb_MessageDef_WellKnownType(m); -} -const upb_MessageDef* API_PENDING_upb_OneofDef_ContainingType(const upb_OneofDef* o) { - return upb_OneofDef_ContainingType(o); -} -const upb_FieldDef* API_PENDING_upb_OneofDef_Field(const upb_OneofDef* o, int i) { - return upb_OneofDef_Field(o, i); -} -int API_PENDING_upb_OneofDef_FieldCount(const upb_OneofDef* o) { - return upb_OneofDef_FieldCount(o); -} -const char* API_PENDING_upb_OneofDef_Name(const upb_OneofDef* o) { - return upb_OneofDef_Name(o); -} -bool API_PENDING_upb_JsonDecode(const char* buf, size_t size, upb_Message* msg, - const upb_MessageDef* m, const upb_DefPool* symtab, - int options, upb_Arena* arena, upb_Status* status) { - return upb_JsonDecode(buf, size, msg, m, symtab, options, arena, status); -} -upb_MutableMessageValue API_PENDING_upb_Message_Mutable(upb_Message* msg, - const upb_FieldDef* f, - upb_Arena* a) { - return upb_Message_Mutable(msg, f, a); -} -const upb_FieldDef* API_PENDING_upb_Message_WhichOneof(const upb_Message* msg, - const upb_OneofDef* o) { - return upb_Message_WhichOneof(msg, o); -} -void API_PENDING_upb_Message_ClearFieldByDef(upb_Message* msg, const upb_FieldDef* f) { - return upb_Message_ClearFieldByDef(msg, f); -} - -bool API_PENDING_upb_Message_HasFieldByDef(const upb_Message* msg, const upb_FieldDef* f) { - return upb_Message_HasFieldByDef(msg, f); -} - -upb_MessageValue API_PENDING_upb_Message_GetFieldByDef(const upb_Message* msg, - const upb_FieldDef* f) { - return upb_Message_GetFieldByDef(msg, f); -} -bool API_PENDING_upb_Message_SetFieldByDef(upb_Message* msg, const upb_FieldDef* f, - upb_MessageValue val, upb_Arena* a) { - return upb_Message_SetFieldByDef(msg, f, val, a); -} -bool API_PENDING_upb_Message_DiscardUnknown(upb_Message* msg, const upb_MessageDef* m, - int maxdepth) { - return upb_Message_DiscardUnknown(msg, m, maxdepth); -} -size_t API_PENDING_upb_JsonEncode(const upb_Message* msg, const upb_MessageDef* m, - const upb_DefPool* ext_pool, int options, char* buf, - size_t size, upb_Status* status) { - return upb_JsonEncode(msg, m, ext_pool, options, buf, size, status); -} diff --git a/ruby/lib/google/protobuf/ffi/descriptor.rb b/ruby/lib/google/protobuf/ffi/descriptor.rb index a9883f619a24..ad8a98f54a1d 100644 --- a/ruby/lib/google/protobuf/ffi/descriptor.rb +++ b/ruby/lib/google/protobuf/ffi/descriptor.rb @@ -164,14 +164,14 @@ def pool class FFI # MessageDef attach_function :new_message_from_def, :upb_Message_New, [Descriptor, Internal::Arena], :Message - attach_function :field_count, :API_PENDING_upb_MessageDef_FieldCount, [Descriptor], :int - attach_function :get_message_file_def, :API_PENDING_upb_MessageDef_File, [:pointer], :FileDef - attach_function :get_message_fullname, :API_PENDING_upb_MessageDef_FullName, [Descriptor], :string - attach_function :get_mini_table, :API_PENDING_upb_MessageDef_MiniTable, [Descriptor], MiniTable.ptr - attach_function :oneof_count, :API_PENDING_upb_MessageDef_OneofCount, [Descriptor], :int - attach_function :get_well_known_type, :API_PENDING_upb_MessageDef_WellKnownType, [Descriptor], WellKnown - attach_function :message_def_syntax, :API_PENDING_upb_MessageDef_Syntax, [Descriptor], Syntax - attach_function :find_msg_def_by_name, :API_PENDING_upb_MessageDef_FindByNameWithSize, [Descriptor, :string, :size_t, :FieldDefPointer, :OneofDefPointer], :bool + attach_function :field_count, :upb_MessageDef_FieldCount, [Descriptor], :int + attach_function :get_message_file_def, :upb_MessageDef_File, [:pointer], :FileDef + attach_function :get_message_fullname, :upb_MessageDef_FullName, [Descriptor], :string + attach_function :get_mini_table, :upb_MessageDef_MiniTable, [Descriptor], MiniTable.ptr + attach_function :oneof_count, :upb_MessageDef_OneofCount, [Descriptor], :int + attach_function :get_well_known_type, :upb_MessageDef_WellKnownType, [Descriptor], WellKnown + attach_function :message_def_syntax, :upb_MessageDef_Syntax, [Descriptor], Syntax + attach_function :find_msg_def_by_name, :upb_MessageDef_FindByNameWithSize, [Descriptor, :string, :size_t, :FieldDefPointer, :OneofDefPointer], :bool end end end diff --git a/ruby/lib/google/protobuf/ffi/descriptor_pool.rb b/ruby/lib/google/protobuf/ffi/descriptor_pool.rb index c7152c8ba592..b09cb8fecaa2 100644 --- a/ruby/lib/google/protobuf/ffi/descriptor_pool.rb +++ b/ruby/lib/google/protobuf/ffi/descriptor_pool.rb @@ -32,11 +32,11 @@ module Google module Protobuf class FFI # DefPool - attach_function :add_serialized_file, :API_PENDING_upb_DefPool_AddFile, [:DefPool, :FileDescriptorProto, Status.by_ref], :FileDef - attach_function :free_descriptor_pool, :API_PENDING_upb_DefPool_Free, [:DefPool], :void - attach_function :create_descriptor_pool,:API_PENDING_upb_DefPool_New, [], :DefPool - attach_function :lookup_enum, :API_PENDING_upb_DefPool_FindEnumByName, [:DefPool, :string], EnumDescriptor - attach_function :lookup_msg, :API_PENDING_upb_DefPool_FindMessageByName, [:DefPool, :string], Descriptor + attach_function :add_serialized_file, :upb_DefPool_AddFile, [:DefPool, :FileDescriptorProto, Status.by_ref], :FileDef + attach_function :free_descriptor_pool, :upb_DefPool_Free, [:DefPool], :void + attach_function :create_descriptor_pool,:upb_DefPool_New, [], :DefPool + attach_function :lookup_enum, :upb_DefPool_FindEnumByName, [:DefPool, :string], EnumDescriptor + attach_function :lookup_msg, :upb_DefPool_FindMessageByName, [:DefPool, :string], Descriptor # FileDescriptorProto attach_function :parse, :FileDescriptorProto_parse, [:binary_string, :size_t], :FileDescriptorProto end diff --git a/ruby/lib/google/protobuf/ffi/enum_descriptor.rb b/ruby/lib/google/protobuf/ffi/enum_descriptor.rb index 0ced5bfae8f0..37b846df14f0 100644 --- a/ruby/lib/google/protobuf/ffi/enum_descriptor.rb +++ b/ruby/lib/google/protobuf/ffi/enum_descriptor.rb @@ -171,14 +171,14 @@ def self.resolve(name) class FFI # EnumDescriptor - attach_function :get_enum_file_descriptor, :API_PENDING_upb_EnumDef_File, [EnumDescriptor], :FileDef - attach_function :enum_value_by_name, :API_PENDING_upb_EnumDef_FindValueByNameWithSize,[EnumDescriptor, :string, :size_t], :EnumValueDef - attach_function :enum_value_by_number, :API_PENDING_upb_EnumDef_FindValueByNumber, [EnumDescriptor, :int], :EnumValueDef - attach_function :get_enum_fullname, :API_PENDING_upb_EnumDef_FullName, [EnumDescriptor], :string - attach_function :enum_value_by_index, :API_PENDING_upb_EnumDef_Value, [EnumDescriptor, :int], :EnumValueDef - attach_function :enum_value_count, :API_PENDING_upb_EnumDef_ValueCount, [EnumDescriptor], :int - attach_function :enum_name, :API_PENDING_upb_EnumValueDef_Name, [:EnumValueDef], :string - attach_function :enum_number, :API_PENDING_upb_EnumValueDef_Number, [:EnumValueDef], :int + attach_function :get_enum_file_descriptor, :upb_EnumDef_File, [EnumDescriptor], :FileDef + attach_function :enum_value_by_name, :upb_EnumDef_FindValueByNameWithSize,[EnumDescriptor, :string, :size_t], :EnumValueDef + attach_function :enum_value_by_number, :upb_EnumDef_FindValueByNumber, [EnumDescriptor, :int], :EnumValueDef + attach_function :get_enum_fullname, :upb_EnumDef_FullName, [EnumDescriptor], :string + attach_function :enum_value_by_index, :upb_EnumDef_Value, [EnumDescriptor, :int], :EnumValueDef + attach_function :enum_value_count, :upb_EnumDef_ValueCount, [EnumDescriptor], :int + attach_function :enum_name, :upb_EnumValueDef_Name, [:EnumValueDef], :string + attach_function :enum_number, :upb_EnumValueDef_Number, [:EnumValueDef], :int end end end diff --git a/ruby/lib/google/protobuf/ffi/field_descriptor.rb b/ruby/lib/google/protobuf/ffi/field_descriptor.rb index a799de33b657..28b5da04d5bd 100644 --- a/ruby/lib/google/protobuf/ffi/field_descriptor.rb +++ b/ruby/lib/google/protobuf/ffi/field_descriptor.rb @@ -307,26 +307,26 @@ def c_type class FFI # MessageDef - attach_function :get_field_by_index, :API_PENDING_upb_MessageDef_Field, [Descriptor, :int], FieldDescriptor - attach_function :get_field_by_name, :API_PENDING_upb_MessageDef_FindFieldByNameWithSize,[Descriptor, :string, :size_t], FieldDescriptor - attach_function :get_field_by_number, :API_PENDING_upb_MessageDef_FindFieldByNumber, [Descriptor, :uint32_t], FieldDescriptor + attach_function :get_field_by_index, :upb_MessageDef_Field, [Descriptor, :int], FieldDescriptor + attach_function :get_field_by_name, :upb_MessageDef_FindFieldByNameWithSize,[Descriptor, :string, :size_t], FieldDescriptor + attach_function :get_field_by_number, :upb_MessageDef_FindFieldByNumber, [Descriptor, :uint32_t], FieldDescriptor # FieldDescriptor - attach_function :get_containing_message_def, :API_PENDING_upb_FieldDef_ContainingType, [FieldDescriptor], Descriptor - attach_function :get_c_type, :API_PENDING_upb_FieldDef_CType, [FieldDescriptor], CType - attach_function :get_default, :API_PENDING_upb_FieldDef_Default, [FieldDescriptor], MessageValue.by_value - attach_function :get_subtype_as_enum, :API_PENDING_upb_FieldDef_EnumSubDef, [FieldDescriptor], EnumDescriptor - attach_function :get_has_presence, :API_PENDING_upb_FieldDef_HasPresence, [FieldDescriptor], :bool - attach_function :is_map, :API_PENDING_upb_FieldDef_IsMap, [FieldDescriptor], :bool - attach_function :is_repeated, :API_PENDING_upb_FieldDef_IsRepeated, [FieldDescriptor], :bool - attach_function :is_sub_message, :API_PENDING_upb_FieldDef_IsSubMessage, [FieldDescriptor], :bool - attach_function :get_json_name, :API_PENDING_upb_FieldDef_JsonName, [FieldDescriptor], :string - attach_function :get_label, :API_PENDING_upb_FieldDef_Label, [FieldDescriptor], Label - attach_function :get_subtype_as_message, :API_PENDING_upb_FieldDef_MessageSubDef, [FieldDescriptor], Descriptor - attach_function :get_full_name, :API_PENDING_upb_FieldDef_Name, [FieldDescriptor], :string - attach_function :get_number, :API_PENDING_upb_FieldDef_Number, [FieldDescriptor], :uint32_t - attach_function :get_type, :API_PENDING_upb_FieldDef_Type, [FieldDescriptor], FieldType - attach_function :file_def_by_raw_field_def, :API_PENDING_upb_FieldDef_File, [:pointer], :FileDef + attach_function :get_containing_message_def, :upb_FieldDef_ContainingType, [FieldDescriptor], Descriptor + attach_function :get_c_type, :upb_FieldDef_CType, [FieldDescriptor], CType + attach_function :get_default, :upb_FieldDef_Default, [FieldDescriptor], MessageValue.by_value + attach_function :get_subtype_as_enum, :upb_FieldDef_EnumSubDef, [FieldDescriptor], EnumDescriptor + attach_function :get_has_presence, :upb_FieldDef_HasPresence, [FieldDescriptor], :bool + attach_function :is_map, :upb_FieldDef_IsMap, [FieldDescriptor], :bool + attach_function :is_repeated, :upb_FieldDef_IsRepeated, [FieldDescriptor], :bool + attach_function :is_sub_message, :upb_FieldDef_IsSubMessage, [FieldDescriptor], :bool + attach_function :get_json_name, :upb_FieldDef_JsonName, [FieldDescriptor], :string + attach_function :get_label, :upb_FieldDef_Label, [FieldDescriptor], Label + attach_function :get_subtype_as_message, :upb_FieldDef_MessageSubDef, [FieldDescriptor], Descriptor + attach_function :get_full_name, :upb_FieldDef_Name, [FieldDescriptor], :string + attach_function :get_number, :upb_FieldDef_Number, [FieldDescriptor], :uint32_t + attach_function :get_type, :upb_FieldDef_Type, [FieldDescriptor], FieldType + attach_function :file_def_by_raw_field_def, :upb_FieldDef_File, [:pointer], :FileDef end end end diff --git a/ruby/lib/google/protobuf/ffi/file_descriptor.rb b/ruby/lib/google/protobuf/ffi/file_descriptor.rb index 9c14f775b5fb..05968a36e288 100644 --- a/ruby/lib/google/protobuf/ffi/file_descriptor.rb +++ b/ruby/lib/google/protobuf/ffi/file_descriptor.rb @@ -32,9 +32,9 @@ module Google module Protobuf class FFI # FileDescriptor - attach_function :file_def_name, :API_PENDING_upb_FileDef_Name, [:FileDef], :string - attach_function :file_def_syntax, :API_PENDING_upb_FileDef_Syntax, [:FileDef], Syntax - attach_function :file_def_pool, :API_PENDING_upb_FileDef_Pool, [:FileDef], :DefPool + attach_function :file_def_name, :upb_FileDef_Name, [:FileDef], :string + attach_function :file_def_syntax, :upb_FileDef_Syntax, [:FileDef], Syntax + attach_function :file_def_pool, :upb_FileDef_Pool, [:FileDef], :DefPool end class FileDescriptor attr :descriptor_pool, :file_def diff --git a/ruby/lib/google/protobuf/ffi/map.rb b/ruby/lib/google/protobuf/ffi/map.rb index a9f896be289c..db1be80a0f98 100644 --- a/ruby/lib/google/protobuf/ffi/map.rb +++ b/ruby/lib/google/protobuf/ffi/map.rb @@ -40,10 +40,10 @@ class FFI attach_function :map_set, :upb_Map_Set, [:Map, MessageValue.by_value, MessageValue.by_value, Internal::Arena], :bool # MapIterator - attach_function :map_next, :API_PENDING_upb_MapIterator_Next, [:Map, :pointer], :bool - attach_function :map_done, :API_PENDING_upb_MapIterator_Done, [:Map, :size_t], :bool - attach_function :map_key, :API_PENDING_upb_MapIterator_Key, [:Map, :size_t], MessageValue.by_value - attach_function :map_value, :API_PENDING_upb_MapIterator_Value, [:Map, :size_t], MessageValue.by_value + attach_function :map_next, :upb_MapIterator_Next, [:Map, :pointer], :bool + attach_function :map_done, :upb_MapIterator_Done, [:Map, :size_t], :bool + attach_function :map_key, :upb_MapIterator_Key, [:Map, :size_t], MessageValue.by_value + attach_function :map_value, :upb_MapIterator_Value, [:Map, :size_t], MessageValue.by_value end class Map include Enumerable diff --git a/ruby/lib/google/protobuf/ffi/message.rb b/ruby/lib/google/protobuf/ffi/message.rb index 57e827b69aef..9ec5bc114e0b 100644 --- a/ruby/lib/google/protobuf/ffi/message.rb +++ b/ruby/lib/google/protobuf/ffi/message.rb @@ -35,17 +35,17 @@ module Google module Protobuf class FFI # Message - attach_function :clear_message_field, :API_PENDING_upb_Message_ClearFieldByDef, [:Message, FieldDescriptor], :void - attach_function :get_message_value, :API_PENDING_upb_Message_GetFieldByDef, [:Message, FieldDescriptor], MessageValue.by_value - attach_function :get_message_has, :API_PENDING_upb_Message_HasFieldByDef, [:Message, FieldDescriptor], :bool - attach_function :set_message_field, :API_PENDING_upb_Message_SetFieldByDef, [:Message, FieldDescriptor, MessageValue.by_value, Internal::Arena], :bool - attach_function :encode_message, :API_PENDING_upb_Encode, [:Message, MiniTable.by_ref, :size_t, Internal::Arena, :pointer, :pointer], EncodeStatus - attach_function :json_decode_message, :API_PENDING_upb_JsonDecode, [:binary_string, :size_t, :Message, Descriptor, :DefPool, :int, Internal::Arena, Status.by_ref], :bool - attach_function :json_encode_message, :API_PENDING_upb_JsonEncode, [:Message, Descriptor, :DefPool, :int, :binary_string, :size_t, Status.by_ref], :size_t + attach_function :clear_message_field, :upb_Message_ClearFieldByDef, [:Message, FieldDescriptor], :void + attach_function :get_message_value, :upb_Message_GetFieldByDef, [:Message, FieldDescriptor], MessageValue.by_value + attach_function :get_message_has, :upb_Message_HasFieldByDef, [:Message, FieldDescriptor], :bool + attach_function :set_message_field, :upb_Message_SetFieldByDef, [:Message, FieldDescriptor, MessageValue.by_value, Internal::Arena], :bool + attach_function :encode_message, :upb_Encode, [:Message, MiniTable.by_ref, :size_t, Internal::Arena, :pointer, :pointer], EncodeStatus + attach_function :json_decode_message, :upb_JsonDecode, [:binary_string, :size_t, :Message, Descriptor, :DefPool, :int, Internal::Arena, Status.by_ref], :bool + attach_function :json_encode_message, :upb_JsonEncode, [:Message, Descriptor, :DefPool, :int, :binary_string, :size_t, Status.by_ref], :size_t attach_function :decode_message, :upb_Decode, [:binary_string, :size_t, :Message, MiniTable.by_ref, :ExtensionRegistry, :int, Internal::Arena], DecodeStatus - attach_function :get_mutable_message, :API_PENDING_upb_Message_Mutable, [:Message, FieldDescriptor, Internal::Arena], MutableMessageValue.by_value - attach_function :get_message_which_oneof, :API_PENDING_upb_Message_WhichOneof, [:Message, OneofDescriptor], FieldDescriptor - attach_function :message_discard_unknown, :API_PENDING_upb_Message_DiscardUnknown, [:Message, Descriptor, :int], :bool + attach_function :get_mutable_message, :upb_Message_Mutable, [:Message, FieldDescriptor, Internal::Arena], MutableMessageValue.by_value + attach_function :get_message_which_oneof, :upb_Message_WhichOneof, [:Message, OneofDescriptor], FieldDescriptor + attach_function :message_discard_unknown, :upb_Message_DiscardUnknown, [:Message, Descriptor, :int], :bool # MessageValue attach_function :message_value_equal, :shared_Msgval_IsEqual, [MessageValue.by_value, MessageValue.by_value, CType, Descriptor], :bool attach_function :message_value_hash, :shared_Msgval_GetHash, [MessageValue.by_value, CType, Descriptor, :uint64_t], :uint64_t diff --git a/ruby/lib/google/protobuf/ffi/oneof_descriptor.rb b/ruby/lib/google/protobuf/ffi/oneof_descriptor.rb index 44eacf296442..c863022cb120 100644 --- a/ruby/lib/google/protobuf/ffi/oneof_descriptor.rb +++ b/ruby/lib/google/protobuf/ffi/oneof_descriptor.rb @@ -95,17 +95,17 @@ def self.private_constructor(oneof_def, descriptor_pool) class FFI # MessageDef - attach_function :get_oneof_by_name, :API_PENDING_upb_MessageDef_FindOneofByNameWithSize, [Descriptor, :string, :size_t], OneofDescriptor - attach_function :get_oneof_by_index, :API_PENDING_upb_MessageDef_Oneof, [Descriptor, :int], OneofDescriptor + attach_function :get_oneof_by_name, :upb_MessageDef_FindOneofByNameWithSize, [Descriptor, :string, :size_t], OneofDescriptor + attach_function :get_oneof_by_index, :upb_MessageDef_Oneof, [Descriptor, :int], OneofDescriptor # OneofDescriptor - attach_function :get_oneof_name, :API_PENDING_upb_OneofDef_Name, [OneofDescriptor], :string - attach_function :get_oneof_field_count, :API_PENDING_upb_OneofDef_FieldCount, [OneofDescriptor], :int - attach_function :get_oneof_field_by_index, :API_PENDING_upb_OneofDef_Field, [OneofDescriptor, :int], FieldDescriptor - attach_function :get_oneof_containing_type,:API_PENDING_upb_OneofDef_ContainingType,[:pointer], Descriptor + attach_function :get_oneof_name, :upb_OneofDef_Name, [OneofDescriptor], :string + attach_function :get_oneof_field_count, :upb_OneofDef_FieldCount, [OneofDescriptor], :int + attach_function :get_oneof_field_by_index, :upb_OneofDef_Field, [OneofDescriptor, :int], FieldDescriptor + attach_function :get_oneof_containing_type,:upb_OneofDef_ContainingType,[:pointer], Descriptor # FieldDescriptor - attach_function :real_containing_oneof, :API_PENDING_upb_FieldDef_RealContainingOneof,[FieldDescriptor], OneofDescriptor + attach_function :real_containing_oneof, :upb_FieldDef_RealContainingOneof,[FieldDescriptor], OneofDescriptor end end end From af40b0490903ba6bc1431a862816a744bf7f59d1 Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Mon, 17 Jul 2023 09:52:45 -0400 Subject: [PATCH 53/54] Default JRuby to autodetecting FFI but make CRuby request it explicitly. --- ruby/lib/google/protobuf.rb | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/ruby/lib/google/protobuf.rb b/ruby/lib/google/protobuf.rb index 9bbb1b898b6a..bde2a0d8cade 100644 --- a/ruby/lib/google/protobuf.rb +++ b/ruby/lib/google/protobuf.rb @@ -42,13 +42,17 @@ class TypeError < ::TypeError; end PREFER_FFI = case ENV['PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION'] when nil, "" - # When unspecified, PREFER_FFI is autodetected based on - # available gems. - begin - require 'ffi' - require 'ffi-compiler/loader' - true - rescue LoadError + if RUBY_PLATFORM == "java" + # When unspecified, PREFER_FFI is autodetected based on + # available gems for JRuby. For CRuby, users must opt-in. + begin + require 'ffi' + require 'ffi-compiler/loader' + true + rescue LoadError + false + end + else false end when /^ffi$/i From 839a3d77e35488ffe32f25d53978f7fa1d6d291d Mon Sep 17 00:00:00 2001 From: Jason Lunn Date: Tue, 18 Jul 2023 10:04:48 -0400 Subject: [PATCH 54/54] Make all interpreters opt-in for FFI. Rename `backend.rb` to `implementation.rb`. --- ruby/BUILD.bazel | 6 ++++++ ruby/Rakefile | 12 ++++++++---- ruby/lib/google/protobuf.rb | 26 +++++-------------------- ruby/tests/BUILD.bazel | 4 ++-- ruby/tests/backend.rb | 11 ----------- ruby/tests/implementation.rb | 37 ++++++++++++++++++++++++++++++++++++ 6 files changed, 58 insertions(+), 38 deletions(-) delete mode 100644 ruby/tests/backend.rb create mode 100644 ruby/tests/implementation.rb diff --git a/ruby/BUILD.bazel b/ruby/BUILD.bazel index f6c4fbf602bd..bf908645ab5f 100755 --- a/ruby/BUILD.bazel +++ b/ruby/BUILD.bazel @@ -191,6 +191,9 @@ conformance_test( conformance_test( name = "conformance_test_ffi", + env = { + "PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION": "ffi", + }, failure_list = "//conformance:failure_list_ruby.txt", target_compatible_with = select({ ":ruby_ffi": [], @@ -213,6 +216,9 @@ conformance_test( conformance_test( name = "conformance_test_jruby_ffi", + env = { + "PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION": "ffi", + }, failure_list = "//conformance:failure_list_jruby_ffi.txt", target_compatible_with = select({ ":jruby_ffi": [], diff --git a/ruby/Rakefile b/ruby/Rakefile index 58dfc978ca08..f030f061f522 100644 --- a/ruby/Rakefile +++ b/ruby/Rakefile @@ -2,12 +2,16 @@ require "rubygems" require "rubygems/package_task" require "rake/extensiontask" unless RUBY_PLATFORM == "java" require "rake/testtask" + +# FFI and FFI-Compiler are developer dependencies everywhere but only +# dependencies for JRuby right now. begin require 'ffi' require "ffi-compiler/compile_task" - USE_FFI = true + FFI_AVAILABLE = true rescue LoadError - USE_FFI = false + warn "LoadError in Rakefile while loading FFI: #{$!}. FFI-related tasks unavailable. To enable, install `ffi` and `ffi-compiler` gems." + FFI_AVAILABLE = false end spec = Gem::Specification.load("google-protobuf.gemspec") @@ -54,7 +58,7 @@ end genproto_output = [] -if USE_FFI +if FFI_AVAILABLE common_excluded_source = [ :convert, :defs, :map, :message, :protobuf, :repeated_field, :wrap_memcpy ] @@ -233,7 +237,7 @@ else end end -task :compile => ["ffi-compiler:default"] if USE_FFI +task :compile => ["ffi-compiler:default"] if FFI_AVAILABLE task :genproto => genproto_output diff --git a/ruby/lib/google/protobuf.rb b/ruby/lib/google/protobuf.rb index bde2a0d8cade..ca3e33b58970 100644 --- a/ruby/lib/google/protobuf.rb +++ b/ruby/lib/google/protobuf.rb @@ -41,26 +41,12 @@ class ParseError < Error; end class TypeError < ::TypeError; end PREFER_FFI = case ENV['PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION'] - when nil, "" - if RUBY_PLATFORM == "java" - # When unspecified, PREFER_FFI is autodetected based on - # available gems for JRuby. For CRuby, users must opt-in. - begin - require 'ffi' - require 'ffi-compiler/loader' - true - rescue LoadError - false - end - else - false - end + when nil, "", /^native$/i + false when /^ffi$/i true - when /^native$/i - false else - warn "Unexpected value `#{ENV['PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION']}` for environment variable `PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION`. Should be either \"FFI\", \"NATIVE\". Omit for autodetection." + warn "Unexpected value `#{ENV['PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION']}` for environment variable `PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION`. Should be either \"FFI\", \"NATIVE\"." false end @@ -85,10 +71,8 @@ def self.decode_json(klass, json, options = {}) require 'google/protobuf_ffi' :FFI rescue LoadError - if $DEBUG - warn "Caught exception `#{$!.message}` while loading FFI implementation of google/protobuf." - warn "Falling back to native implementation." - end + warn "Caught exception `#{$!.message}` while loading FFI implementation of google/protobuf." + warn "Falling back to native implementation." require 'google/protobuf_native' :NATIVE end diff --git a/ruby/tests/BUILD.bazel b/ruby/tests/BUILD.bazel index 01427ef9ce80..57566207c0d9 100644 --- a/ruby/tests/BUILD.bazel +++ b/ruby/tests/BUILD.bazel @@ -15,8 +15,8 @@ filegroup( ) ruby_test( - name = "backend", - srcs = ["backend.rb"], + name = "implementation", + srcs = ["implementation.rb"], deps = [ "//ruby:protobuf", "@protobuf_bundle//:test-unit", diff --git a/ruby/tests/backend.rb b/ruby/tests/backend.rb deleted file mode 100644 index 3fe366333d2e..000000000000 --- a/ruby/tests/backend.rb +++ /dev/null @@ -1,11 +0,0 @@ -require 'google/protobuf' -require 'test/unit' - -# Verifies that the implementation of Protobuf is the expected (preferred) one. -# See protobuf.rb for the logic that defines PREFER_FFI. -class BackendTest < Test::Unit::TestCase - EXPECTED_IMPLEMENTATION = Google::Protobuf::PREFER_FFI ? :FFI : :NATIVE - def test_ffi_implementation - assert_equal EXPECTED_IMPLEMENTATION, Google::Protobuf::IMPLEMENTATION - end -end diff --git a/ruby/tests/implementation.rb b/ruby/tests/implementation.rb new file mode 100644 index 000000000000..56241fc42b44 --- /dev/null +++ b/ruby/tests/implementation.rb @@ -0,0 +1,37 @@ +require 'google/protobuf' +require 'test/unit' + +class BackendTest < Test::Unit::TestCase + # Verifies the implementation of Protobuf is the preferred one. + # See protobuf.rb for the logic that defines PREFER_FFI. + def test_prefer_ffi_aligns_with_implementation + expected = Google::Protobuf::PREFER_FFI ? :FFI : :NATIVE + assert_equal expected, Google::Protobuf::IMPLEMENTATION + end + + def test_prefer_ffi + unless ENV['PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION'] =~ /ffi/i + omit"FFI implementation requires environment variable PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION=FFI to activate." + end + assert_equal true, Google::Protobuf::PREFER_FFI + end + def test_ffi_implementation + unless ENV['PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION'] =~ /ffi/i + omit "FFI implementation requires environment variable PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION=FFI to activate." + end + assert_equal :FFI, Google::Protobuf::IMPLEMENTATION + end + + def test_prefer_native + if ENV.include?('PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION') and ENV['PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION'] !~ /native/i + omit"Native implementation requires omitting environment variable PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION or setting it to `NATIVE` to activate." + end + assert_equal false, Google::Protobuf::PREFER_FFI + end + def test_native_implementation + if ENV.include?('PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION') and ENV['PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION'] !~ /native/i + omit"Native implementation requires omitting environment variable PROTOCOL_BUFFERS_RUBY_IMPLEMENTATION or setting it to `NATIVE` to activate." + end + assert_equal :NATIVE, Google::Protobuf::IMPLEMENTATION + end +end