Introduce bytes type (#1543)

This type is mapped to the native bytestring type in the various
languages, i.e.

- Kotlin: `ByteArray`
- Python: `bytes`
- Ruby: `String` with `Encoding::BINARY`
- Swift: `Data`

Also add tests exercising this new type.

Fixes #1299.
Fixes #1541.

CHANGELOG.md

@@ -30,6 +30,8 @@
 - Traits can be exposed as a UniFFI `interface` by using a `[Trait]` attribute in the UDL.
  See [the documentation](https://mozilla.github.io/uniffi-rs/udl/interfaces.html#exposing-traits-as-interfaces).
 
+- The `bytes` primitive type was added, it represents an array of bytes. It maps to `ByteArray` in Kotlin, `bytes` in Python, `String` with `Encoding::BINARY` in Ruby and `Data` in Swift.
+
 ## v0.23.0 (backend crates: v0.23.0) - (_2023-01-27_)
 
 [All changes in v0.23.0](https://github.com/mozilla/uniffi-rs/compare/v0.22.0...v0.23.0).

docs/manual/src/internals/lifting_and_lowering.md

@@ -59,6 +59,7 @@ Calling this function from foreign language code involves the following steps:
 | `f64`/`double` | `double` |
 | `boolean` | `int8_t`, either `0` or `1` |
 | `string` | `RustBuffer` struct pointing to utf8 bytes |
+| `bytes` | Same as `sequence<u8>` |
 | `timestamp` | `RustBuffer` struct pointing to a i64 representing seconds and a u32 representing nanoseconds |
 | `duration` | `RustBuffer` struct pointing to a u64 representing seconds and a u32 representing nanoseconds |
 | `T?` | `RustBuffer` struct pointing to serialized bytes |

docs/manual/src/swift/overview.md

@@ -4,7 +4,7 @@ UniFFI ships with production-quality support for generating Swift bindings.
 Concepts from the UDL file map into Swift as follows:
 
 * Primitive datatypes map to their obvious Swift counterpart, e.g. `u32` becomes `UInt32`,
- `string` becomes `String`, etc.
+ `string` becomes `String`, `bytes` becomes `Data`, etc.
 * An object interface declared as `interface T` is represented as a Swift `protocol TProtocol`
  and a concrete Swift `class T` that conforms to it. Having the protocol declared explicitly
  can be useful for mocking instances of the class in unittests.

docs/manual/src/udl/builtin_types.md

@@ -9,6 +9,7 @@ The following built-in types can be passed as arguments/returned by Rust methods
 | `f32` | `float` | |
 | `f64` | `double` | |
 | `String` | `string` | |
+| `Vec<u8>` | `bytes` | Different from `sequence<u8>` only in foreign type mappings |
 | `SystemTime` | `timestamp` | Precision may be lost when converting to Python and Swift types |
 | `Duration ` | `duration` | Precision may be lost when converting to Python and Swift types |
 | `&T` | `[ByRef] T` | This works for `&str` and `&[T]` |

fixtures/coverall/src/coverall.udl

@@ -14,6 +14,8 @@ namespace coverall {
 dictionary SimpleDict {
  string text;
  string? maybe_text;
+ bytes some_bytes;
+ bytes? maybe_some_bytes;
  boolean a_bool;
  boolean? maybe_a_bool;
  u8 unsigned8;
@@ -131,6 +133,9 @@ interface Coveralls {
 
  /// Returns all repairs made.
  sequence<Repair> get_repairs();
+
+ /// Reverses the bytes.
+ bytes reverse(bytes value);
 };
 
 // coveralls keep track of their repairs (an interface in a dict)

fixtures/coverall/src/lib.rs

@@ -48,6 +48,8 @@ pub enum ComplexError {
 pub struct SimpleDict {
  text: String,
  maybe_text: Option<String>,
+ some_bytes: Vec<u8>,
+ maybe_some_bytes: Option<Vec<u8>>,
  a_bool: bool,
  maybe_a_bool: Option<bool>,
  unsigned8: u8,
@@ -85,6 +87,8 @@ fn create_some_dict() -> SimpleDict {
  SimpleDict {
  text: "text".to_string(),
  maybe_text: Some("maybe_text".to_string()),
+ some_bytes: b"some_bytes".to_vec(),
+ maybe_some_bytes: Some(b"maybe_some_bytes".to_vec()),
  a_bool: true,
  maybe_a_bool: Some(false),
  unsigned8: 1,
@@ -109,6 +113,7 @@ fn create_some_dict() -> SimpleDict {
 fn create_none_dict() -> SimpleDict {
  SimpleDict {
  text: "text".to_string(),
+ some_bytes: b"some_bytes".to_vec(),
  a_bool: true,
  unsigned8: 1,
  unsigned16: 3,
@@ -277,6 +282,11 @@ impl Coveralls {
  let repairs = self.repairs.lock().unwrap();
  repairs.clone()
  }
+
+ fn reverse(&self, mut value: Vec<u8>) -> Vec<u8> {
+ value.reverse();
+ value
+ }
 }
 
 impl Drop for Coveralls {

fixtures/coverall/tests/bindings/test_coverall.kts

@@ -12,10 +12,12 @@ import uniffi.coverall.*
 // Test some_dict().
 // N.B. we need to `use` here to clean up the contained `Coveralls` reference.
 createSomeDict().use { d ->
- assert(d.text == "text");
- assert(d.maybeText == "maybe_text");
- assert(d.aBool);
- assert(d.maybeABool == false);
+ assert(d.text == "text")
+ assert(d.maybeText == "maybe_text")
+ assert(d.someBytes.contentEquals("some_bytes".toByteArray(Charsets.UTF_8)))
+ assert(d.maybeSomeBytes.contentEquals("maybe_some_bytes".toByteArray(Charsets.UTF_8)))
+ assert(d.aBool)
+ assert(d.maybeABool == false)
  assert(d.unsigned8 == 1.toUByte())
  assert(d.maybeUnsigned8 == 2.toUByte())
  assert(d.unsigned16 == 3.toUShort())
@@ -39,6 +41,36 @@ createSomeDict().use { d ->
  assert(d.coveralls!!.getName() == "some_dict")
 }
 
+createNoneDict().use { d ->
+ assert(d.text == "text")
+ assert(d.maybeText == null)
+ assert(d.someBytes.contentEquals("some_bytes".toByteArray(Charsets.UTF_8)))
+ assert(d.maybeSomeBytes == null)
+ assert(d.aBool)
+ assert(d.maybeABool == null)
+ assert(d.unsigned8 == 1.toUByte())
+ assert(d.maybeUnsigned8 == null)
+ assert(d.unsigned16 == 3.toUShort())
+ assert(d.maybeUnsigned16 == null)
+ assert(d.unsigned64 == 18446744073709551615UL)
+ assert(d.maybeUnsigned64 == null)
+ assert(d.signed8 == 8.toByte())
+ assert(d.maybeSigned8 == null)
+ assert(d.signed64 == 9223372036854775807L)
+ assert(d.maybeSigned64 == null)
+
+ // floats should be "close enough".
+ fun Float.almostEquals(other: Float) = Math.abs(this - other) < 0.000001
+ fun Double.almostEquals(other: Double) = Math.abs(this - other) < 0.000001
+
+ assert(d.float32.almostEquals(1.2345F))
+ assert(d.maybeFloat32 == null)
+ assert(d.float64.almostEquals(0.0))
+ assert(d.maybeFloat64 == null)
+
+ assert(d.coveralls == null)
+}
+
 
 // Test arcs.
 
@@ -216,3 +248,8 @@ d = DictWithDefaults(name = "this", category = "that", integer = 42UL)
 assert(d.name == "this")
 assert(d.category == "that")
 assert(d.integer == 42UL)
+
+// Test bytes
+Coveralls("test_bytes").use { coveralls ->
+ assert(coveralls.reverse("123".toByteArray(Charsets.UTF_8)).toString(Charsets.UTF_8) == "321")
+}

fixtures/coverall/tests/bindings/test_coverall.py

@@ -16,6 +16,8 @@ def test_some_dict(self):
  d = create_some_dict()
  self.assertEqual(d.text, "text")
  self.assertEqual(d.maybe_text, "maybe_text")
+ self.assertEqual(d.some_bytes, b"some_bytes")
+ self.assertEqual(d.maybe_some_bytes, b"maybe_some_bytes")
  self.assertTrue(d.a_bool)
  self.assertFalse(d.maybe_a_bool)
  self.assertEqual(d.unsigned8, 1)
@@ -42,6 +44,8 @@ def test_none_dict(self):
  d = create_none_dict()
  self.assertEqual(d.text, "text")
  self.assertIsNone(d.maybe_text)
+ self.assertEqual(d.some_bytes, b"some_bytes")
+ self.assertIsNone(d.maybe_some_bytes)
  self.assertTrue(d.a_bool)
  self.assertIsNone(d.maybe_a_bool)
  self.assertEqual(d.unsigned8, 1)
@@ -214,6 +218,10 @@ def test_dict_with_non_string_keys(self):
  dict3 = coveralls.get_dict3(key=31, value=42)
  assert dict3[31] == 42
 
+ def test_bytes(self):
+ coveralls = Coveralls("test_bytes")
+ self.assertEqual(coveralls.reverse(b"123"), b"321")
+
 class TraitsTest(unittest.TestCase):
  def test_simple(self):
  traits = get_traits()

fixtures/coverall/tests/bindings/test_coverall.rb

@@ -12,7 +12,14 @@ class TestCoverall < Test::Unit::TestCase
  def test_some_dict
  d = Coverall.create_some_dict
  assert_equal(d.text, 'text')
+ assert_equal(d.text.encoding, Encoding::UTF_8)
  assert_equal(d.maybe_text, 'maybe_text')
+ assert_equal(d.maybe_text.encoding, Encoding::UTF_8)
+ assert_equal(d.some_bytes, 'some_bytes')
+ assert_equal(d.some_bytes.encoding, Encoding::BINARY)
+ assert_equal(d.maybe_some_bytes, 'maybe_some_bytes')
+ assert_equal(d.maybe_some_bytes.encoding, Encoding::BINARY)
+ assert_true(d.a_bool)
  assert_true(d.a_bool)
  assert_false(d.maybe_a_bool)
  assert_equal(d.unsigned8, 1)
@@ -38,7 +45,11 @@ def test_some_dict
  def test_none_dict
  d = Coverall.create_none_dict
  assert_equal(d.text, 'text')
+ assert_equal(d.text.encoding, Encoding::UTF_8)
  assert_nil(d.maybe_text)
+ assert_equal(d.some_bytes, 'some_bytes')
+ assert_equal(d.some_bytes.encoding, Encoding::BINARY)
+ assert_nil(d.maybe_some_bytes)
  assert_true(d.a_bool)
  assert_nil(d.maybe_a_bool)
  assert_equal(d.unsigned8, 1)
@@ -234,5 +245,10 @@ def test_bad_objects
  end
  end
 
+ def test_bytes
+ coveralls = Coverall::Coveralls.new "test_bytes"
+ assert_equal coveralls.reverse("123"), "321"
+ assert_equal coveralls.reverse("123").encoding, Encoding::BINARY
+ end
 
 end

fixtures/coverall/tests/bindings/test_coverall.swift

@@ -26,6 +26,8 @@ do {
  let d = createSomeDict()
  assert(d.text == "text")
  assert(d.maybeText == "maybe_text")
+ assert(d.someBytes == Data("some_bytes".utf8))
+ assert(d.maybeSomeBytes == Data("maybe_some_bytes".utf8))
  assert(d.aBool)
  assert(d.maybeABool == false);
  assert(d.unsigned8 == 1)
@@ -45,6 +47,32 @@ do {
  assert(d.coveralls!.getName() == "some_dict")
 }
 
+// Test none_dict().
+do {
+ let d = createNoneDict()
+ assert(d.text == "text")
+ assert(d.maybeText == nil)
+ assert(d.someBytes == Data("some_bytes".utf8))
+ assert(d.maybeSomeBytes == nil)
+ assert(d.aBool)
+ assert(d.maybeABool == nil);
+ assert(d.unsigned8 == 1)
+ assert(d.maybeUnsigned8 == nil)
+ assert(d.unsigned16 == 3)
+ assert(d.maybeUnsigned16 == nil)
+ assert(d.unsigned64 == 18446744073709551615)
+ assert(d.maybeUnsigned64 == nil)
+ assert(d.signed8 == 8)
+ assert(d.maybeSigned8 == nil)
+ assert(d.signed64 == 9223372036854775807)
+ assert(d.maybeSigned64 == nil)
+ assert(d.float32.almostEquals(1.2345))
+ assert(d.maybeFloat32 == nil)
+ assert(d.float64.almostEquals(0.0))
+ assert(d.maybeFloat64 == nil)
+ assert(d.coveralls == nil)
+}
+
 // Test arcs.
 do {
  let coveralls = Coveralls(name: "test_arcs")
@@ -202,3 +230,9 @@ do {
  coveralls.addRepair(repair: Repair(when: Date.init(), patch: Patch(color: Color.blue)))
  assert(coveralls.getRepairs().count == 2)
 }
+
+// Test bytes
+do {
+ let coveralls = Coveralls(name: "test_bytes")
+ assert(coveralls.reverse(value: Data("123".utf8)) == Data("321".utf8))
+}

uniffi_bindgen/src/bindings/kotlin/gen_kotlin/mod.rs

@@ -243,6 +243,7 @@ impl KotlinCodeOracle {
  Type::Float64 => Box::new(primitives::Float64CodeType),
  Type::Boolean => Box::new(primitives::BooleanCodeType),
  Type::String => Box::new(primitives::StringCodeType),
+ Type::Bytes => Box::new(primitives::BytesCodeType),
 
  Type::Timestamp => Box::new(miscellany::TimestampCodeType),
  Type::Duration => Box::new(miscellany::DurationCodeType),

uniffi_bindgen/src/bindings/kotlin/gen_kotlin/primitives.rs

@@ -67,6 +67,7 @@ macro_rules! impl_code_type_for_primitive {
 
 impl_code_type_for_primitive!(BooleanCodeType, "Boolean");
 impl_code_type_for_primitive!(StringCodeType, "String");
+impl_code_type_for_primitive!(BytesCodeType, "ByteArray");
 impl_code_type_for_primitive!(Int8CodeType, "Byte");
 impl_code_type_for_primitive!(Int16CodeType, "Short");
 impl_code_type_for_primitive!(Int32CodeType, "Int");

uniffi_bindgen/src/bindings/kotlin/templates/ByteArrayHelper.kt

@@ -0,0 +1,15 @@
+public object FfiConverterByteArray: FfiConverterRustBuffer<ByteArray> {
+ override fun read(buf: ByteBuffer): ByteArray {
+ val len = buf.getInt()
+ val byteArr = ByteArray(len)
+ buf.get(byteArr)
+ return byteArr
+ }
+ override fun allocationSize(value: ByteArray): Int {
+ return 4 + value.size
+ }
+ override fun write(value: ByteArray, buf: ByteBuffer) {
+ buf.putInt(value.size)
+ buf.put(value)
+ }
+}

uniffi_bindgen/src/bindings/kotlin/templates/Types.kt

@@ -53,6 +53,9 @@
 {%- when Type::String %}
 {%- include "StringHelper.kt" %}
 
+{%- when Type::Bytes %}
+{%- include "ByteArrayHelper.kt" %}
+
 {%- when Type::Enum(name) %}
 {% include "EnumTemplate.kt" %}
 

uniffi_bindgen/src/bindings/python/gen_python/mod.rs

@@ -289,6 +289,7 @@ impl PythonCodeOracle {
  Type::Float64 => Box::new(primitives::Float64CodeType),
  Type::Boolean => Box::new(primitives::BooleanCodeType),
  Type::String => Box::new(primitives::StringCodeType),
+ Type::Bytes => Box::new(primitives::BytesCodeType),
 
  Type::Timestamp => Box::new(miscellany::TimestampCodeType),
  Type::Duration => Box::new(miscellany::DurationCodeType),

uniffi_bindgen/src/bindings/python/gen_python/primitives.rs

@@ -57,6 +57,7 @@ macro_rules! impl_code_type_for_primitive {
 
 impl_code_type_for_primitive!(BooleanCodeType, "Bool", "bool({})");
 impl_code_type_for_primitive!(StringCodeType, "String", "{}");
+impl_code_type_for_primitive!(BytesCodeType, "Bytes", "{}");
 impl_code_type_for_primitive!(Int8CodeType, "Int8", "int({})");
 impl_code_type_for_primitive!(Int16CodeType, "Int16", "int({})");
 impl_code_type_for_primitive!(Int32CodeType, "Int32", "int({})");

uniffi_bindgen/src/bindings/python/templates/BytesHelper.py

@@ -0,0 +1,12 @@
+class FfiConverterBytes(FfiConverterRustBuffer):
+ @staticmethod
+ def read(buf):
+ size = buf.readI32()
+ if size < 0:
+ raise InternalError("Unexpected negative byte string length")
+ return buf.read(size)
+
+ @staticmethod
+ def write(value, buf):
+ buf.writeI32(len(value))
+ buf.write(value)

uniffi_bindgen/src/bindings/python/templates/Types.py

@@ -52,6 +52,9 @@
 {%- when Type::String %}
 {%- include "StringHelper.py" %}
 
+{%- when Type::Bytes %}
+{%- include "BytesHelper.py" %}
+
 {%- when Type::Enum(name) %}
 {%- include "EnumTemplate.py" %}
 

uniffi_bindgen/src/bindings/ruby/gen_ruby/mod.rs

@@ -175,7 +175,7 @@ mod filters {
  Type::Object { .. } | Type::Enum(_) | Type::Error(_) | Type::Record(_) => {
  nm.to_string()
  }
- Type::String => format!("{nm}.to_s"),
+ Type::String | Type::Bytes => format!("{nm}.to_s"),
  Type::Timestamp | Type::Duration => nm.to_string(),
  Type::CallbackInterface(_) => panic!("No support for coercing callback interfaces yet"),
  Type::Optional(t) => format!("({nm} ? {} : nil)", coerce_rb(nm, t)?),
@@ -219,6 +219,7 @@ mod filters {
  | Type::Float64 => nm.to_string(),
  Type::Boolean => format!("({nm} ? 1 : 0)"),
  Type::String => format!("RustBuffer.allocFromString({nm})"),
+ Type::Bytes => format!("RustBuffer.allocFromBytes({nm})"),
  Type::Object { name, .. } => format!("({}._uniffi_lower {nm})", class_name_rb(name)?),
  Type::CallbackInterface(_) => panic!("No support for lowering callback interfaces yet"),
  Type::Error(_) => panic!("No support for lowering errors, yet"),
@@ -252,6 +253,7 @@ mod filters {
  Type::Float32 | Type::Float64 => format!("{nm}.to_f"),
  Type::Boolean => format!("1 == {nm}"),
  Type::String => format!("{nm}.consumeIntoString"),
+ Type::Bytes => format!("{nm}.consumeIntoBytes"),
  Type::Object { name, .. } => format!("{}._uniffi_allocate({nm})", class_name_rb(name)?),
  Type::CallbackInterface(_) => panic!("No support for lifting callback interfaces, yet"),
  Type::Error(_) => panic!("No support for lowering errors, yet"),

uniffi_bindgen/src/bindings/ruby/templates/RustBufferBuilder.rb

@@ -105,6 +105,14 @@ def write_String(v)
  write v
  end
 
+ {% when Type::Bytes -%}
+
+ def write_Bytes(v)
+ v = v.to_s
+ pack_into 4, 'l>', v.bytes.size
+ write v
+ end
+
  {% when Type::Timestamp -%}
  # The Timestamp type.
  ONE_SECOND_IN_NANOSECONDS = 10**9