From 1dd2f1e75df4d8573f1434325d5cfb2293fa4823 Mon Sep 17 00:00:00 2001 From: mustiikhalil <26250654+mustiikhalil@users.noreply.github.com> Date: Tue, 24 Dec 2024 11:09:24 +0100 Subject: [PATCH] Offical Swift port for FlexBuffers This is the offical port for FlexBuffers within swift, and it introcudes a Common Module where code is shared between flatbuffers and flexbuffers. Writing most supported values like maps, vectors, nil and scalars into a flexbuffer buffer. And includes tests to verify that its similar to cpp --- Package.swift | 13 +- swift/Sources/Common/Constants.swift | 103 +++ .../Int+extension.swift | 4 +- swift/Sources/Common/functions.swift | 29 + swift/Sources/FlatBuffers/ByteBuffer.swift | 1 + swift/Sources/FlatBuffers/Constants.swift | 109 +-- .../FlatBuffers/FlatBufferBuilder.swift | 22 +- swift/Sources/FlatBuffers/Table.swift | 1 + swift/Sources/FlexBuffers/ByteBuffer.swift | 464 ++++++++++ .../Sources/FlexBuffers/FlexBufferType.swift | 75 ++ .../Sources/FlexBuffers/Utils/BitWidth.swift | 48 + .../Sources/FlexBuffers/Utils/Constants.swift | 49 ++ swift/Sources/FlexBuffers/Utils/Value.swift | 132 +++ .../Sources/FlexBuffers/Utils/functions.swift | 90 ++ .../Writer/FlexBuffersWriter.swift | 826 ++++++++++++++++++ .../FlexBuffers/_InternalByteBuffer.swift | 221 +++++ tests/swift/tests/Package.swift | 9 +- .../FlatBuffersTests.swift | 1 + .../FlexBuffersWriterTests.swift | 233 +++++ .../FlexBuffers.Test.SwiftTests/Mocks.swift | 19 + 20 files changed, 2345 insertions(+), 104 deletions(-) create mode 100644 swift/Sources/Common/Constants.swift rename swift/Sources/{FlatBuffers => Common}/Int+extension.swift (94%) create mode 100644 swift/Sources/Common/functions.swift create mode 100644 swift/Sources/FlexBuffers/ByteBuffer.swift create mode 100644 swift/Sources/FlexBuffers/FlexBufferType.swift create mode 100644 swift/Sources/FlexBuffers/Utils/BitWidth.swift create mode 100644 swift/Sources/FlexBuffers/Utils/Constants.swift create mode 100644 swift/Sources/FlexBuffers/Utils/Value.swift create mode 100644 swift/Sources/FlexBuffers/Utils/functions.swift create mode 100644 swift/Sources/FlexBuffers/Writer/FlexBuffersWriter.swift create mode 100644 swift/Sources/FlexBuffers/_InternalByteBuffer.swift create mode 100644 tests/swift/tests/Tests/FlexBuffers.Test.SwiftTests/FlexBuffersWriterTests.swift create mode 100644 tests/swift/tests/Tests/FlexBuffers.Test.SwiftTests/Mocks.swift diff --git a/Package.swift b/Package.swift index ad6f0e866c2..1729b59acce 100644 --- a/Package.swift +++ b/Package.swift @@ -27,10 +27,21 @@ let package = Package( .library( name: "FlatBuffers", targets: ["FlatBuffers"]), + .library( + name: "FlexBuffers", + targets: ["FlexBuffers"]), ], targets: [ .target( name: "FlatBuffers", + dependencies: ["Common"], + path: "swift/Sources/FlatBuffers"), + .target( + name: "FlexBuffers", + dependencies: ["Common"], + path: "swift/Sources/FlexBuffers"), + .target( + name: "Common", dependencies: [], - path: "swift/Sources"), + path: "swift/Sources/Common"), ]) diff --git a/swift/Sources/Common/Constants.swift b/swift/Sources/Common/Constants.swift new file mode 100644 index 00000000000..19c21ebae48 --- /dev/null +++ b/swift/Sources/Common/Constants.swift @@ -0,0 +1,103 @@ +/* + * Copyright 2024 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Foundation + +/// A boolean to see if the system is littleEndian +public let isLitteEndian: Bool = { + let number: UInt32 = 0x12345678 + return number == number.littleEndian +}() + +/// Constant for the file id length +public let FileIdLength = 4 + +/// Protocol that All Scalars should conform to +/// +/// Scalar is used to conform all the numbers that can be represented in a FlatBuffer. It's used to write/read from the buffer. +public protocol Scalar: Equatable { + associatedtype NumericValue + var convertedEndian: NumericValue { get } +} + +extension Scalar where Self: FixedWidthInteger { + /// Converts the value from BigEndian to LittleEndian + /// + /// Converts values to little endian on machines that work with BigEndian, however this is NOT TESTED yet. + public var convertedEndian: NumericValue { + self as! Self.NumericValue + } +} + +extension Double: Scalar { + public typealias NumericValue = UInt64 + + public var convertedEndian: UInt64 { + bitPattern.littleEndian + } +} + +extension Float32: Scalar { + public typealias NumericValue = UInt32 + + public var convertedEndian: UInt32 { + bitPattern.littleEndian + } +} + +extension Bool: Scalar { + public var convertedEndian: UInt8 { + self == true ? 1 : 0 + } + + public typealias NumericValue = UInt8 +} + +extension Int: Scalar { + public typealias NumericValue = Int +} + +extension Int8: Scalar { + public typealias NumericValue = Int8 +} + +extension Int16: Scalar { + public typealias NumericValue = Int16 +} + +extension Int32: Scalar { + public typealias NumericValue = Int32 +} + +extension Int64: Scalar { + public typealias NumericValue = Int64 +} + +extension UInt8: Scalar { + public typealias NumericValue = UInt8 +} + +extension UInt16: Scalar { + public typealias NumericValue = UInt16 +} + +extension UInt32: Scalar { + public typealias NumericValue = UInt32 +} + +extension UInt64: Scalar { + public typealias NumericValue = UInt64 +} diff --git a/swift/Sources/FlatBuffers/Int+extension.swift b/swift/Sources/Common/Int+extension.swift similarity index 94% rename from swift/Sources/FlatBuffers/Int+extension.swift rename to swift/Sources/Common/Int+extension.swift index 62b5cd5cd1a..29bca2a7a04 100644 --- a/swift/Sources/FlatBuffers/Int+extension.swift +++ b/swift/Sources/Common/Int+extension.swift @@ -22,9 +22,9 @@ extension Int { /// /// This is used since the UnsafeMutableRawPointer will face issues when writing/reading /// if the buffer alignment exceeds that actual size of the buffer - var convertToPowerofTwo: Int { + public var convertToPowerofTwo: Int { guard self > 0 else { return 1 } - var n = UOffset(self) + var n = UInt32(self) #if arch(arm) || arch(i386) let max = UInt32(Int.max) diff --git a/swift/Sources/Common/functions.swift b/swift/Sources/Common/functions.swift new file mode 100644 index 00000000000..e6ba11d777f --- /dev/null +++ b/swift/Sources/Common/functions.swift @@ -0,0 +1,29 @@ +/* + * Copyright 2024 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Foundation + +/// Gets the padding for the current element +/// - Parameters: +/// - bufSize: Current size of the buffer + the offset of the object to be written +/// - elementSize: Element size +@inline(__always) +public func padding( + bufSize: UInt, + elementSize: UInt) -> UInt +{ + ((~bufSize) &+ 1) & (elementSize &- 1) +} diff --git a/swift/Sources/FlatBuffers/ByteBuffer.swift b/swift/Sources/FlatBuffers/ByteBuffer.swift index 31485794e45..c76c78f1022 100644 --- a/swift/Sources/FlatBuffers/ByteBuffer.swift +++ b/swift/Sources/FlatBuffers/ByteBuffer.swift @@ -14,6 +14,7 @@ * limitations under the License. */ +import Common import Foundation /// `ByteBuffer` is the interface that stores the data for a `Flatbuffers` object diff --git a/swift/Sources/FlatBuffers/Constants.swift b/swift/Sources/FlatBuffers/Constants.swift index e24fc05b1f3..ed4dfc2008a 100644 --- a/swift/Sources/FlatBuffers/Constants.swift +++ b/swift/Sources/FlatBuffers/Constants.swift @@ -14,15 +14,9 @@ * limitations under the License. */ +@_exported import Common import Foundation -/// A boolean to see if the system is littleEndian -let isLitteEndian: Bool = { - let number: UInt32 = 0x12345678 - return number == number.littleEndian -}() -/// Constant for the file id length -let FileIdLength = 4 /// Type aliases public typealias Byte = UInt8 public typealias UOffset = UInt32 @@ -35,80 +29,31 @@ public let FlatBufferMaxSize = UInt32 /// Protocol that All Scalars should conform to /// /// Scalar is used to conform all the numbers that can be represented in a FlatBuffer. It's used to write/read from the buffer. -public protocol Scalar: Equatable { - associatedtype NumericValue - var convertedEndian: NumericValue { get } -} - -extension Scalar where Self: Verifiable {} - -extension Scalar where Self: FixedWidthInteger { - /// Converts the value from BigEndian to LittleEndian - /// - /// Converts values to little endian on machines that work with BigEndian, however this is NOT TESTED yet. - public var convertedEndian: NumericValue { - self as! Self.NumericValue - } -} - -extension Double: Scalar, Verifiable { - public typealias NumericValue = UInt64 - - public var convertedEndian: UInt64 { - bitPattern.littleEndian - } -} - -extension Float32: Scalar, Verifiable { - public typealias NumericValue = UInt32 - - public var convertedEndian: UInt32 { - bitPattern.littleEndian - } -} - -extension Bool: Scalar, Verifiable { - public var convertedEndian: UInt8 { - self == true ? 1 : 0 - } - - public typealias NumericValue = UInt8 -} - -extension Int: Scalar, Verifiable { - public typealias NumericValue = Int -} - -extension Int8: Scalar, Verifiable { - public typealias NumericValue = Int8 -} - -extension Int16: Scalar, Verifiable { - public typealias NumericValue = Int16 -} - -extension Int32: Scalar, Verifiable { - public typealias NumericValue = Int32 -} - -extension Int64: Scalar, Verifiable { - public typealias NumericValue = Int64 -} - -extension UInt8: Scalar, Verifiable { - public typealias NumericValue = UInt8 -} - -extension UInt16: Scalar, Verifiable { - public typealias NumericValue = UInt16 -} - -extension UInt32: Scalar, Verifiable { - public typealias NumericValue = UInt32 -} - -extension UInt64: Scalar, Verifiable { - public typealias NumericValue = UInt64 -} + +extension Scalar where Self: FixedWidthInteger {} + +extension Double: Verifiable {} + +extension Float32: Verifiable {} + +extension Bool: Verifiable {} + +extension Int: Verifiable {} + +extension Int8: Verifiable {} + +extension Int16: Verifiable {} + +extension Int32: Verifiable {} + +extension Int64: Verifiable {} + +extension UInt8: Verifiable {} + +extension UInt16: Verifiable {} + +extension UInt32: Verifiable {} + +extension UInt64: Verifiable {} public func FlatBuffersVersion_25_2_10() {} diff --git a/swift/Sources/FlatBuffers/FlatBufferBuilder.swift b/swift/Sources/FlatBuffers/FlatBufferBuilder.swift index c96f24cde90..9c06ca97f14 100644 --- a/swift/Sources/FlatBuffers/FlatBufferBuilder.swift +++ b/swift/Sources/FlatBuffers/FlatBufferBuilder.swift @@ -14,6 +14,7 @@ * limitations under the License. */ +import Common import Foundation /// ``FlatBufferBuilder`` builds a `FlatBuffer` through manipulating its internal state. @@ -343,19 +344,6 @@ public struct FlatBufferBuilder { } } - /// Gets the padding for the current element - /// - Parameters: - /// - bufSize: Current size of the buffer + the offset of the object to be written - /// - elementSize: Element size - @inline(__always) - @usableFromInline - mutating internal func padding( - bufSize: UInt32, - elementSize: UInt32) -> UInt32 - { - ((~bufSize) &+ 1) & (elementSize &- 1) - } - /// Prealigns the buffer before writting a new object into the buffer /// - Parameters: /// - len:Length of the object @@ -364,11 +352,9 @@ public struct FlatBufferBuilder { @usableFromInline mutating internal func preAlign(len: Int, alignment: Int) { minAlignment(size: alignment) - _bb.fill( - padding: Int( - padding( - bufSize: _bb.size &+ UOffset(len), - elementSize: UOffset(alignment)))) + _bb.fill(padding: numericCast(padding( + bufSize: numericCast(_bb.size) &+ numericCast(len), + elementSize: numericCast(alignment)))) } /// Prealigns the buffer before writting a new object into the buffer diff --git a/swift/Sources/FlatBuffers/Table.swift b/swift/Sources/FlatBuffers/Table.swift index b3be4d1737e..91d79982396 100644 --- a/swift/Sources/FlatBuffers/Table.swift +++ b/swift/Sources/FlatBuffers/Table.swift @@ -14,6 +14,7 @@ * limitations under the License. */ +import Common import Foundation /// `Table` is a Flatbuffers object that can read, diff --git a/swift/Sources/FlexBuffers/ByteBuffer.swift b/swift/Sources/FlexBuffers/ByteBuffer.swift new file mode 100644 index 00000000000..a7824a44b49 --- /dev/null +++ b/swift/Sources/FlexBuffers/ByteBuffer.swift @@ -0,0 +1,464 @@ +/* + * Copyright 2024 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Foundation + +/// `ByteBuffer` is the interface that stores the data for a `Flatbuffers` object +/// it allows users to write and read data directly from memory thus the use of its +/// functions should be used +@frozen +public struct ByteBuffer { + + /// Storage is a container that would hold the memory pointer to solve the issue of + /// deallocating the memory that was held by (memory: UnsafeMutableRawPointer) + @usableFromInline + final class Storage { + @usableFromInline + enum Blob { + #if !os(WASI) + case data(Data) + case bytes(ContiguousBytes) + #endif + + case byteBuffer(_InternalByteBuffer) + case array([UInt8]) + case pointer(UnsafeMutableRawPointer) + } + + /// This storage doesn't own the memory, therefore, we won't deallocate on deinit. + private let isOwned: Bool + /// Retained blob of data that requires the storage to retain a pointer to. + @usableFromInline + var retainedBlob: Blob + /// Capacity of UInt8 the buffer can hold + var capacity: Int + + @usableFromInline + init(count: Int) { + let memory = UnsafeMutableRawPointer.allocate( + byteCount: count, + alignment: MemoryLayout.alignment) + capacity = count + retainedBlob = .pointer(memory) + isOwned = true + } + + @usableFromInline + init(blob: Blob, capacity count: Int) { + capacity = count + retainedBlob = blob + isOwned = false + } + + deinit { + guard isOwned else { return } + switch retainedBlob { + case .pointer(let unsafeMutableRawPointer): + unsafeMutableRawPointer.deallocate() + default: break + } + } + + @usableFromInline + func copy(from ptr: UnsafeRawPointer, count: Int) { + assert( + isOwned, + "copy should NOT be called on a buffer that is built by assumingMemoryBound") + withUnsafeRawPointer { + $0.copyMemory(from: ptr, byteCount: count) + } + } + + @usableFromInline + func initialize(for size: Int) { + assert( + isOwned, + "initalize should NOT be called on a buffer that is built by assumingMemoryBound") + withUnsafeRawPointer { + memset($0, 0, size) + } + } + + @discardableResult + @inline(__always) + func withUnsafeBytes( + _ body: (UnsafeRawBufferPointer) throws + -> T) rethrows -> T + { + switch retainedBlob { + case .byteBuffer(let byteBuffer): + return try byteBuffer.withUnsafeBytes(body) + #if !os(WASI) + case .data(let data): + return try data.withUnsafeBytes(body) + case .bytes(let contiguousBytes): + return try contiguousBytes.withUnsafeBytes(body) + #endif + case .array(let array): + return try array.withUnsafeBytes(body) + case .pointer(let ptr): + return try body(UnsafeRawBufferPointer(start: ptr, count: capacity)) + } + } + + @discardableResult + @inline(__always) + func withUnsafeRawPointer( + _ body: (UnsafeMutableRawPointer) throws + -> T) rethrows -> T + { + switch retainedBlob { + case .byteBuffer(let byteBuffer): + return try byteBuffer.withUnsafeRawPointer(body) + #if !os(WASI) + case .data(let data): + return try data + .withUnsafeBytes { + try body(UnsafeMutableRawPointer(mutating: $0.baseAddress!)) + } + case .bytes(let contiguousBytes): + return try contiguousBytes + .withUnsafeBytes { + try body(UnsafeMutableRawPointer(mutating: $0.baseAddress!)) + } + #endif + case .array(let array): + return try array + .withUnsafeBytes { + try body(UnsafeMutableRawPointer(mutating: $0.baseAddress!)) + } + case .pointer(let ptr): + return try body(ptr) + } + } + + @discardableResult + @inline(__always) + func readWithUnsafeRawPointer( + position: Int, + _ body: (UnsafeRawPointer) throws -> T) rethrows -> T + { + switch retainedBlob { + case .byteBuffer(let byteBuffer): + return try byteBuffer.readWithUnsafeRawPointer(position: position, body) + #if !os(WASI) + case .data(let data): + return try data.withUnsafeBytes { + try body($0.baseAddress!.advanced(by: position)) + } + case .bytes(let contiguousBytes): + return try contiguousBytes.withUnsafeBytes { + try body($0.baseAddress!.advanced(by: position)) + } + #endif + case .array(let array): + return try array.withUnsafeBytes { + try body($0.baseAddress!.advanced(by: position)) + } + case .pointer(let ptr): + return try body(ptr.advanced(by: position)) + } + } + } + + @usableFromInline var _storage: Storage + + /// The size of the elements written to the buffer + their paddings + private var _readerIndex: Int = 0 + /// Reader is the position of the current Writer Index (capacity - size) + public var reader: Int { _storage.capacity &- _readerIndex } + /// Current size of the buffer + public var size: Int { _readerIndex } + /// Current capacity for the buffer + public var capacity: Int { _storage.capacity } + + /// Constructor that creates a Flatbuffer object from an InternalByteBuffer + /// - Parameter + /// - bytes: Array of UInt8 + @inline(__always) + init(byteBuffer: _InternalByteBuffer) { + _storage = Storage( + blob: .byteBuffer(byteBuffer), + capacity: byteBuffer.capacity) + _readerIndex = byteBuffer.writerIndex + } + + /// Constructor that creates a Flatbuffer from unsafe memory region by copying + /// the underlying data to a new pointer + /// + /// - Parameters: + /// - copyingMemoryBound: The unsafe memory region + /// - capacity: The size of the given memory region + @inline(__always) + public init( + copyingMemoryBound memory: UnsafeRawPointer, + capacity: Int) + { + _storage = Storage(count: capacity) + _storage.copy(from: memory, count: capacity) + _readerIndex = _storage.capacity + } + + /// Constructor that creates a Flatbuffer object from a UInt8 + /// - Parameter + /// - bytes: Array of UInt8 + @inline(__always) + public init(bytes: [UInt8]) { + _storage = Storage(blob: .array(bytes), capacity: bytes.count) + _readerIndex = _storage.capacity + } + + #if !os(WASI) + /// Constructor that creates a Flatbuffer from the Swift Data type object + /// - Parameter + /// - data: Swift data Object + @inline(__always) + public init(data: Data) { + _storage = Storage(blob: .data(data), capacity: data.count) + _readerIndex = _storage.capacity + } + + /// Constructor that creates a Flatbuffer object from a ContiguousBytes + /// - Parameters: + /// - contiguousBytes: Binary stripe to use as the buffer + /// - count: amount of readable bytes + @inline(__always) + public init( + contiguousBytes: Bytes, + count: Int) + { + _storage = Storage(blob: .bytes(contiguousBytes), capacity: count) + _readerIndex = _storage.capacity + } + #endif + + /// Constructor that creates a Flatbuffer from unsafe memory region without copying + /// **NOTE** Needs a call to `memory.deallocate()` later on to free the memory + /// + /// - Parameters: + /// - assumingMemoryBound: The unsafe memory region + /// - capacity: The size of the given memory region + @inline(__always) + public init( + assumingMemoryBound memory: UnsafeMutableRawPointer, + capacity: Int) + { + _storage = Storage( + blob: .pointer(memory), + capacity: capacity) + _readerIndex = _storage.capacity + } + + /// Creates a copy of the existing flatbuffer, by copying it to a different memory. + /// - Parameters: + /// - memory: Current memory of the buffer + /// - count: count of bytes + /// - removeBytes: Removes a number of bytes from the current size + @inline(__always) + init( + blob: Storage.Blob, + count: Int, + removing removeBytes: Int) + { + _storage = Storage(blob: blob, capacity: count) + _readerIndex = removeBytes + } + + /// Write stores an object into the buffer directly or indirectly. + /// + /// Direct: ignores the capacity of buffer which would mean we are referring to the direct point in memory + /// indirect: takes into respect the current capacity of the buffer (capacity - index), writing to the buffer from the end + /// - Parameters: + /// - value: Value that needs to be written to the buffer + /// - index: index to write to + /// - direct: Should take into consideration the capacity of the buffer + @inline(__always) + func write(value: T, index: Int, direct: Bool = false) { + var index = index + if !direct { + index = _storage.capacity &- index + } + assert(index < _storage.capacity, "Write index is out of writing bound") + assert(index >= 0, "Writer index should be above zero") + _ = withUnsafePointer(to: value) { ptr in + _storage.withUnsafeRawPointer { + memcpy( + $0.advanced(by: index), + ptr, + MemoryLayout.size) + } + } + } + + /// Reads an object from the buffer + /// - Parameters: + /// - def: Type of the object + /// - position: the index of the object in the buffer + @inline(__always) + public func read(def: T.Type, position: Int) -> T { + _storage.readWithUnsafeRawPointer(position: position) { + $0.bindMemory(to: T.self, capacity: 1) + .pointee + } + } + + /// Reads a slice from the memory assuming a type of T + /// - Parameters: + /// - index: index of the object to be read from the buffer + /// - count: count of bytes in memory + @inline(__always) + public func readSlice( + index: Int, + count: Int) -> [T] + { + assert( + index + count <= _storage.capacity, + "Reading out of bounds is illegal") + + return _storage.readWithUnsafeRawPointer(position: index) { + let buf = UnsafeBufferPointer( + start: $0.bindMemory(to: T.self, capacity: count), + count: count) + return Array(buf) + } + } + + /// Provides a pointer towards the underlying primitive types + /// - Parameters: + /// - index: index of the object to be read from the buffer + /// - count: count of bytes in memory + @discardableResult + @inline(__always) + public func withUnsafePointerToSlice( + index: Int, + count: Int, + body: (UnsafeRawBufferPointer) throws -> T) rethrows -> T + { + assert( + index + count <= _storage.capacity, + "Reading out of bounds is illegal") + return try _storage.readWithUnsafeRawPointer(position: index) { + try body(UnsafeRawBufferPointer(start: $0, count: count)) + } + } + + #if !os(WASI) + /// Reads a string from the buffer and encodes it to a swift string + /// - Parameters: + /// - index: index of the string in the buffer + /// - count: length of the string + /// - type: Encoding of the string + @inline(__always) + public func readString( + at index: Int, + count: Int, + type: String.Encoding = .utf8) -> String? + { + assert( + index + count <= _storage.capacity, + "Reading out of bounds is illegal") + return _storage.readWithUnsafeRawPointer(position: index) { + let buf = UnsafeBufferPointer( + start: $0.bindMemory(to: UInt8.self, capacity: count), + count: count) + return String( + bytes: buf, + encoding: type) + } + } + #else + /// Reads a string from the buffer and encodes it to a swift string + /// - Parameters: + /// - index: index of the string in the buffer + /// - count: length of the string + @inline(__always) + public func readString( + at index: Int, + count: Int) -> String? + { + assert( + index + count <= _storage.capacity, + "Reading out of bounds is illegal") + return _storage.readWithUnsafeRawPointer(position: index) { + String(cString: $0.bindMemory(to: UInt8.self, capacity: count)) + } + } + #endif + + /// Creates a new Flatbuffer object that's duplicated from the current one + /// - Parameter removeBytes: the amount of bytes to remove from the current Size + @inline(__always) + public func duplicate(removing removeBytes: Int = 0) -> ByteBuffer { + assert(removeBytes > 0, "Can NOT remove negative bytes") + assert( + removeBytes < _storage.capacity, + "Can NOT remove more bytes than the ones allocated") + return ByteBuffer( + blob: _storage.retainedBlob, + count: _storage.capacity, + removing: _readerIndex &- removeBytes) + } + + /// SkipPrefix Skips the first 4 bytes in case one of the following + /// functions are called `getPrefixedSizeCheckedRoot` & `getPrefixedSizeRoot` + /// which allows us to skip the first 4 bytes instead of recreating the buffer + @discardableResult + @usableFromInline + @inline(__always) + mutating func skipPrefix() -> Int32 { + _readerIndex = _readerIndex &- MemoryLayout.size + return read(def: Int32.self, position: 0) + } + + @discardableResult + @inline(__always) + public func withUnsafeBytes( + body: (UnsafeRawBufferPointer) throws + -> T) rethrows -> T + { + try _storage.withUnsafeBytes(body) + } + + @discardableResult + @inline(__always) + func withUnsafeMutableRawPointer( + body: (UnsafeMutableRawPointer) throws + -> T) rethrows -> T + { + try _storage.withUnsafeRawPointer(body) + } + + @discardableResult + @inline(__always) + func readWithUnsafeRawPointer( + position: Int, + _ body: (UnsafeRawPointer) throws -> T) rethrows -> T + { + try _storage.readWithUnsafeRawPointer(position: position, body) + } +} + +extension ByteBuffer: CustomDebugStringConvertible { + + public var debugDescription: String { + """ + buffer located at: \(_storage.retainedBlob), + with capacity of \(_storage.capacity), + { writtenSize: \(_readerIndex), readerSize: \(reader), + size: \(size) } + """ + } +} diff --git a/swift/Sources/FlexBuffers/FlexBufferType.swift b/swift/Sources/FlexBuffers/FlexBufferType.swift new file mode 100644 index 00000000000..4f51c5d2871 --- /dev/null +++ b/swift/Sources/FlexBuffers/FlexBufferType.swift @@ -0,0 +1,75 @@ +/* + * Copyright 2024 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Foundation + +public enum FlexBufferType: UInt64 { + case null = 0 + /// Variable width signed integer: `Int8, Int16, Int32, Int64` + case int = 1 + /// Variable width signed integer: `UInt8, UInt16, UInt32, UInt64` + case uint = 2 + /// Variable width floating point: `Float32, Double64` + case float = 3 + case key = 4 + case string = 5 + /// An Int, stored by offset rather than inline. Indirect types can keep the bitwidth of a + /// vector or map small when the inline value would have increased the bitwidth. + case indirectInt = 6 + /// A UInt, stored by offset rather than inline. Indirect types can keep the bitwidth of a + /// vector or map small when the inline value would have increased the bitwidth. + case indirectUInt = 7 + /// A Float, stored by offset rather than inline. Indirect types can keep the bitwidth of a + /// vector or map small when the inline value would have increased the bitwidth. + case indirectFloat = 8 + case map = 9 + /// Untyped + case vector = 10 + /// Typed any sizes (stores no type table). + case vectorInt = 11 + case vectorUInt = 12 + case vectorFloat = 13 + case vectorKey = 14 + @available( + *, + deprecated, + message: "use FBT_VECTOR or FBT_VECTOR_KEY instead.") + case vectorString = 15 + + /// Typed tuples (no type table, no size field). + case vectorInt2 = 16 + case vectorUInt2 = 17 + case vectorFloat2 = 18 + /// Typed triples (no type table, no size field). + case vectorInt3 = 19 + case vectorUInt3 = 20 + case vectorFloat3 = 21 + /// Typed quad (no type table, no size field). + case vectorInt4 = 22 + case vectorUInt4 = 23 + case vectorFloat4 = 24 + case blob = 25 + case bool = 26 + /// To Allow the same type of conversion of type to vector type + case vectorBool = 36 + case max = 37 +} + +extension FlexBufferType: Comparable { + public static func < (lhs: FlexBufferType, rhs: FlexBufferType) -> Bool { + lhs.rawValue < rhs.rawValue + } +} diff --git a/swift/Sources/FlexBuffers/Utils/BitWidth.swift b/swift/Sources/FlexBuffers/Utils/BitWidth.swift new file mode 100644 index 00000000000..08c818e7f62 --- /dev/null +++ b/swift/Sources/FlexBuffers/Utils/BitWidth.swift @@ -0,0 +1,48 @@ +/* + * Copyright 2024 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Foundation + +@usableFromInline +enum BitWidth: UInt64, CaseIterable { + case w8 = 0, w16 = 1, w32 = 2, w64 = 3 +} + +extension BitWidth: Comparable { + @usableFromInline + static func < (lhs: BitWidth, rhs: BitWidth) -> Bool { + lhs.rawValue < rhs.rawValue + } + + @inline(__always) + static func widthB(_ v: Int) -> BitWidth { + switch v { + case 1: return .w8 + case 2: return .w16 + case 4: return .w32 + case 8: return .w64 + default: + assert(false, "We shouldn't reach here") + return .w64 + } + } + + @inline(__always) + static func max(_ lhs: BitWidth, rhs: BitWidth) -> BitWidth { + if lhs.rawValue > rhs.rawValue { return lhs } + return rhs + } +} diff --git a/swift/Sources/FlexBuffers/Utils/Constants.swift b/swift/Sources/FlexBuffers/Utils/Constants.swift new file mode 100644 index 00000000000..c0017913cc6 --- /dev/null +++ b/swift/Sources/FlexBuffers/Utils/Constants.swift @@ -0,0 +1,49 @@ +/* + * Copyright 2024 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@_exported import Common +import Foundation + +extension UInt64 { + static let one: UInt64 = 1 +} + +extension UInt32 { + static let one: UInt32 = 1 +} + +public enum BuilderFlag: UInt8 { + case none = 0 + case shareKeys = 1 + case shareStrings = 2 + case shareKeysAndStrings = 3 + case shareKeyVectors = 4 + case shareAll = 7 +} + +extension BuilderFlag: Comparable { + public static func < (lhs: BuilderFlag, rhs: BuilderFlag) -> Bool { + lhs.rawValue < rhs.rawValue + } +} + +extension UInt: Scalar { + public typealias NumericValue = UInt +} + +extension Int: Scalar { + public typealias NumericValue = Int +} diff --git a/swift/Sources/FlexBuffers/Utils/Value.swift b/swift/Sources/FlexBuffers/Utils/Value.swift new file mode 100644 index 00000000000..ab1ce27da94 --- /dev/null +++ b/swift/Sources/FlexBuffers/Utils/Value.swift @@ -0,0 +1,132 @@ +/* + * Copyright 2024 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Common +import Foundation + +public struct Value: Equatable { + + @usableFromInline + enum Union: Equatable { + case i(Int64) + case u(UInt64) + case f(Double) + } + + var sloc: Union + let type: FlexBufferType + let bitWidth: BitWidth + + private init() { + sloc = .i(0) + type = .null + bitWidth = .w8 + } + + init(bool: Bool) { + sloc = .u(bool ? 1 : 0) + type = .bool + bitWidth = .w8 + } + + init(v: UInt64, type: FlexBufferType, bitWidth: BitWidth) { + sloc = .u(v) + self.type = type + self.bitWidth = bitWidth + } + + init(v: Int64, type: FlexBufferType, bitWidth: BitWidth) { + sloc = .i(v) + self.type = type + self.bitWidth = bitWidth + } + + init(v: Double, type: FlexBufferType, bitWidth: BitWidth) { + sloc = .f(v) + self.type = type + self.bitWidth = bitWidth + } + + init(sloc: Union, type: FlexBufferType, bitWidth: BitWidth) { + self.sloc = sloc + self.type = type + self.bitWidth = bitWidth + } + + @usableFromInline + var i: Int64 { + switch sloc { + case .i(let v): v + default: 0 + } + } + + @usableFromInline + var u: UInt64 { + switch sloc { + case .u(let v): v + default: 0 + } + } + + @usableFromInline + var f: Double { + switch sloc { + case .f(let v): v + default: 0 + } + } + + static let `nil` = Value() +} + +extension Value { + @usableFromInline + @inline(__always) + func elementWidth(size: Int, index: UInt64) -> BitWidth { + if isInline(type) { + return bitWidth + } else { + for byteWidth in stride(from: 1, to: MemoryLayout.size, by: 2) { + let _offsetLoc: UInt64 = numericCast(numericCast(size) &+ padding( + bufSize: numericCast(size), + elementSize: numericCast(byteWidth))) + let offsetLoc = _offsetLoc &+ (index &* numericCast(byteWidth)) + let offset = offsetLoc &- u + + let bitWidth = widthU(offset) + if (UInt32.one << bitWidth.rawValue) == byteWidth { + return bitWidth + } + } + return .w64 + } + } + + @inline(__always) + func storedPackedType(width: BitWidth = .w8) -> UInt8 { + packedType(bitWidth: storedWidth(width: width), type: type) + } + + @inline(__always) + private func storedWidth(width: BitWidth) -> BitWidth { + if isInline(type) { + return max(bitWidth, width) + } else { + return bitWidth + } + } +} diff --git a/swift/Sources/FlexBuffers/Utils/functions.swift b/swift/Sources/FlexBuffers/Utils/functions.swift new file mode 100644 index 00000000000..480cd8d3df3 --- /dev/null +++ b/swift/Sources/FlexBuffers/Utils/functions.swift @@ -0,0 +1,90 @@ +/* + * Copyright 2024 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Foundation + +@inline(__always) +internal func isInline(_ t: FlexBufferType) -> Bool { + return t <= .float || t == .bool +} + +@inline(__always) +private func check(_ v: UInt64, width: UInt64) -> Bool { + (v & ~((.one << width) &- 1)) == 0 +} + +@inline(__always) +internal func widthI(_ v: Int64) -> BitWidth { + let u = unsafeBitCast(v, to: UInt64.self) << 1 + return widthU(v >= 0 ? u : ~u) +} + +@inline(__always) +internal func widthF(_ v: Double) -> BitWidth { + Double(Float(v)) == v ? .w32 : .w64 +} + +@inline(__always) +internal func widthU(_ v: UInt64) -> BitWidth { + if check(v, width: 8) { return .w8 } + if check(v, width: 16) { return .w16 } + if check(v, width: 32) { return .w32 } + return .w64 +} + +@inline(__always) +internal func packedType(bitWidth: BitWidth, type: FlexBufferType) -> UInt8 { + numericCast(bitWidth.rawValue | (type.rawValue << 2)) +} + +@inline(__always) +func getScalarType(type: T.Type) -> FlexBufferType where T: Scalar { + if T.self is (any BinaryFloatingPoint.Type) { + return .float + } + + if T.self is Bool.Type { + return .bool + } + + if T.self is (any UnsignedInteger.Type) { + return .uint + } + + return .int +} + + +@inline(__always) +func toTypedVector(type: FlexBufferType, length: UInt64) -> FlexBufferType { + let type: UInt64 = switch length { + case 0: type.rawValue &- FlexBufferType.int.rawValue &+ FlexBufferType + .vectorInt.rawValue + case 2: type.rawValue &- FlexBufferType.int.rawValue &+ FlexBufferType + .vectorInt2.rawValue + case 3: type.rawValue &- FlexBufferType.int.rawValue &+ FlexBufferType + .vectorInt3.rawValue + case 4: type.rawValue &- FlexBufferType.int.rawValue &+ FlexBufferType + .vectorInt4.rawValue + default: 0 + } + return FlexBufferType(rawValue: type) ?? .null +} + +@inline(__always) +func isTypedVectorType(type: FlexBufferType) -> Bool { + return type >= .int && type <= .string || type == .bool +} diff --git a/swift/Sources/FlexBuffers/Writer/FlexBuffersWriter.swift b/swift/Sources/FlexBuffers/Writer/FlexBuffersWriter.swift new file mode 100644 index 00000000000..e6507d012e3 --- /dev/null +++ b/swift/Sources/FlexBuffers/Writer/FlexBuffersWriter.swift @@ -0,0 +1,826 @@ +/* + * Copyright 2024 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Foundation + +public typealias FlexBuffersWriterBuilder = (inout FlexBuffersWriter) -> Void + +public struct FlexBuffersWriter { + + var capacity: Int { + _bb.capacity + } + + var writerIndex: Int { + get { + _bb.writerIndex + } set { + _bb.writerIndex = newValue + } + } + + private var finished = false + private var hasDuplicatedKeys = false + private var minBitWidth: BitWidth = .w8 + private var _bb: _InternalByteBuffer + private var stack: [Value] = [] + private var keyPool: [Int: UInt] = [:] + private var stringPool: [Int: UInt] = [:] + private var flags: BuilderFlag + + public init(initialSize: Int = 1024, flags: BuilderFlag = .shareKeys) { + _bb = _InternalByteBuffer(initialSize: initialSize) + self.flags = flags + } + + /// Returns the written bytes into the ``ByteBuffer`` + /// + /// Should only be used after ``finish(offset:addPrefix:)`` is called + public var sizedByteArray: [UInt8] { + assert( + finished == true, + "function finish() should be called before accessing data") + return _bb.underlyingBytes + } + + public var sizedByteBuffer: ByteBuffer { + assert( + finished == true, + "function finish() should be called before accessing data") + return _bb.withUnsafeSlicedBytes { + ByteBuffer(copyingMemoryBound: $0.baseAddress!, capacity: $0.count) + } + } + + /// Resets the internal state. Automatically called before building a new flexbuffer. + public mutating func reset() { + _bb.clear() + stack.removeAll(keepingCapacity: true) + finished = false + minBitWidth = .w8 + keyPool.removeAll() + stringPool.removeAll() + } + + // MARK: - Storing root + @inline(__always) + public mutating func finish() { + assert(stack.count == 1) + + // Write root value. + var byteWidth = align(width: stack[0].elementWidth( + size: writerIndex, + index: 0)) + + write(value: stack[0], byteWidth: byteWidth) + var storedType = stack[0].storedPackedType() + // Write root type. + _bb.writeBytes(&storedType, len: 1) + // Write root size. Normally determined by parent, but root has no parent :) + _bb.writeBytes(&byteWidth, len: 1) + + finished = true + } + + // MARK: - Vector + @discardableResult + @inline(__always) + public func startVector() -> Int { + stack.count + } + + @discardableResult + @inline(__always) + public mutating func startVector(key k: String) -> Int { + add(key: k) + return stack.count + } + + @discardableResult + @inline(__always) + public mutating func endVector( + start: Int, + typed: Bool = false, + fixed: Bool = false) -> UInt64 + { + let vec = createVector( + start: start, + count: stack.count - start, + step: 1, + typed: typed, + fixed: fixed, + keys: nil) + stack = Array(stack[..(vector: [T]) -> Int where T: Scalar { + create(vector: vector, fixed: false) + } + + @discardableResult + @inline(__always) + public mutating func create(vector: [T], key: borrowing String) -> Int + where T: Scalar + { + add(key: key) + return create(vector: vector, fixed: false) + } + + @discardableResult + @inline(__always) + public mutating func createFixed(vector: [T]) -> Int where T: Scalar { + assert(vector.count >= 2 && vector.count <= 4) + return create(vector: vector, fixed: true) + } + + @discardableResult + @inline(__always) + public mutating func createFixed( + vector: [T], + key: borrowing String) -> Int where T: Scalar + { + assert(vector.count >= 2 && vector.count <= 4) + add(key: key) + return create(vector: vector, fixed: true) + } + + // MARK: - Map + @discardableResult + @inline(__always) + public func startMap() -> Int { + stack.count + } + + @discardableResult + @inline(__always) + public mutating func startMap(key k: String) -> Int { + add(key: k) + return stack.count + } + + @discardableResult + @inline(__always) + public mutating func endMap(start: Int) -> UInt64 { + let len = sortMapByKeys(start: start) + + let keys = createVector( + start: start, + count: len, + step: 2, + typed: true, + fixed: false) + let vec = createVector( + start: start + 1, + count: len, + step: 2, + typed: false, + fixed: false, + keys: keys) + stack = Array(stack[..( + blob: borrowing T, + length l: Int) -> UInt where T: ContiguousBytes + { + storeBlob(blob, len: l, type: .blob) + } + + @discardableResult + @inline(__always) + public mutating func add( + blob: borrowing T, + key: borrowing String, + length l: Int) -> UInt where T: ContiguousBytes + { + add(key: key) + return storeBlob(blob, len: l, type: .blob) + } + + // MARK: - Reuse Values + @inline(__always) + public func lastValue() -> Value? { + return stack.last + } + + @inline(__always) + public mutating func reuse(value: Value) { + stack.append(value) + } + + @inline(__always) + public mutating func reuse(value: Value, key: borrowing String) { + add(key: key) + reuse(value: value) + } + + // MARK: - Private - + + // MARK: Writing to buffer + + @inline(__always) + private mutating func write(value: Value, byteWidth: Int) { + switch value.type { + case .null: fallthrough + case .int: write(value: value.i, byteWidth: byteWidth) + case .bool: fallthrough + case .uint: write(value: value.u, byteWidth: byteWidth) + case .float: write(double: value.f, byteWidth: byteWidth) + default: + write(offset: value.u, byteWidth: byteWidth) + } + } + + @inline(__always) + private mutating func pushIndirect( + value: T, + type: FlexBufferType, + bitWidth: BitWidth) + { + let byteWidth = align(width: bitWidth) + let iloc = writerIndex + _bb.ensureSpace(size: byteWidth) + + _bb.write(value, len: byteWidth) + stack.append(Value( + sloc: .u(numericCast(iloc)), + type: type, + bitWidth: bitWidth)) + } + + // MARK: Internal Writing Strings + + /// Adds a string to the buffer using swift.utf8 object + /// - Parameter str: String that will be added to the buffer + /// - Parameter len: length of the string + @discardableResult + @inline(__always) + private mutating func write(str: borrowing String, len: Int) -> UInt { + let resetTo = writerIndex + var sloc = str.withCString { + storeBlob(pointer: $0, len: len, trailing: 1, type: .string) + } + + if flags >= .shareKeysAndStrings { + let loc = stringPool[str.hashValue] + if let loc { + writerIndex = resetTo + sloc = loc + assert( + stack.count > 0, + "Attempting to override the location, but stack is empty") + stack[stack.count - 1].sloc = .u(numericCast(sloc)) + } else { + stringPool[str.hashValue] = sloc + } + } + return sloc + } + + // MARK: Write Keys + @discardableResult + @inline(__always) + private mutating func add(key: borrowing String) -> UInt { + add(key: key, len: key.count) + } + + @discardableResult + @inline(__always) + private mutating func add(key: borrowing String, len: Int) -> UInt { + _bb.ensureSpace(size: len) + + var sloc: UInt = numericCast(writerIndex) + key.withCString { + _bb.writeBytes($0, len: len + 1) + } + + if flags > .shareKeys { + let loc = keyPool[key.hashValue] + if let loc { + writerIndex = Int(sloc) + sloc = loc + } else { + keyPool[key.hashValue] = sloc + } + } + stack.append(Value(sloc: .u(numericCast(sloc)), type: .key, bitWidth: .w8)) + return sloc + } + + // MARK: - Storing Blobs + @inline(__always) + private mutating func storeBlob( + _ bytes: T, + len: Int, + type: FlexBufferType) -> UInt where T: ContiguousBytes + { + return bytes.withUnsafeBytes { + storeBlob(pointer: $0.baseAddress!, len: len, type: type) + } + } + + @discardableResult + @usableFromInline + @inline(__always) + mutating func storeBlob( + pointer: borrowing UnsafeRawPointer, + len: Int, + trailing: Int = 0, + type: FlexBufferType) -> UInt + { + _bb.ensureSpace(size: len &+ trailing) + let bitWidth = widthU(numericCast(len)) + + let bytes = align(width: bitWidth) + + var len = len + _bb.writeBytes(&len, len: bytes) + let sloc = writerIndex + + _bb.writeBytes(pointer, len: len &+ trailing) + stack.append(Value( + sloc: .u(numericCast(sloc)), + type: type, + bitWidth: bitWidth)) + return numericCast(sloc) + } + + // MARK: Write Vectors + @discardableResult + @inline(__always) + private mutating func create(vector: [T], fixed: Bool) -> Int + where T: Scalar + { + let length: UInt64 = numericCast(vector.count) + let vectorType = getScalarType(type: T.self) + let byteWidth = MemoryLayout.size + let bitWidth = BitWidth.widthB(byteWidth) + + _bb.ensureSpace(size: vector.count &* Int(bitWidth.rawValue)) + + assert(widthU(length) <= bitWidth) + + align(width: bitWidth) + + if !fixed { + write(value: length, byteWidth: byteWidth) + } + let vloc = _bb.writerIndex + + for i in stride(from: 0, to: vector.count, by: 1) { + write(value: vector[i], byteWidth: byteWidth) + } + + stack.append( + Value( + sloc: .u(numericCast(vloc)), + type: toTypedVector(type: vectorType, length: fixed ? length : 0), + bitWidth: bitWidth)) + return vloc + } + + @inline(__always) + private mutating func createVector( + start: Int, + count: Int, + step: Int, + typed: Bool, + fixed: Bool, + keys: Value? = nil) -> Value + { + assert( + !fixed || typed, + "Typed false and fixed true is a combination not supported currently") + + var bitWidth = BitWidth.max(minBitWidth, rhs: widthU(numericCast(count))) + var prefixElements = 1 + if keys != nil { + /// If this vector is part of a map, we will pre-fix an offset to the keys + /// to this vector. + bitWidth = max(bitWidth, keys!.elementWidth(size: writerIndex, index: 0)) + prefixElements += 2 + } + var vectorType: FlexBufferType = .key + + for i in stride(from: start, to: stack.count, by: step) { + let elemWidth = stack[i].elementWidth( + size: _bb.writerIndex, + index: numericCast(i &- start &+ prefixElements)) + bitWidth = BitWidth.max(bitWidth, rhs: elemWidth) + guard typed else { continue } + if i == start { + vectorType = stack[i].type + } else { + assert( + vectorType == stack[i].type, + """ + If you get this assert you are writing a typed vector + with elements that are not all the same type + """) + } + } + assert( + !typed || isTypedVectorType(type: vectorType), + """ + If you get this assert, your typed types are not one of: + Int / UInt / Float / Key. + """) + + let byteWidth = align(width: bitWidth) + + _bb.ensureSpace( + size: count &* step &* byteWidth) + + if keys != nil { + write(offset: keys!.u, byteWidth: byteWidth) + write(value: UInt64.one << keys!.bitWidth.rawValue, byteWidth: byteWidth) + } + + if !fixed { + write(value: count, byteWidth: byteWidth) + } + + let vloc = _bb.writerIndex + + for i in stride(from: start, to: stack.count, by: step) { + write(value: stack[i], byteWidth: byteWidth) + } + + if !typed { + for i in stride(from: start, to: stack.count, by: step) { + _bb.write(stack[i].storedPackedType(width: bitWidth), len: 1) + } + } + + let type: FlexBufferType = if keys != nil { + .map + } else if typed { + toTypedVector(type: vectorType, length: numericCast(fixed ? count : 0)) + } else { + .vector + } + + return Value(sloc: .u(numericCast(vloc)), type: type, bitWidth: bitWidth) + } + + // MARK: Write Scalar functions + @inline(__always) + private mutating func write(offset: UInt64, byteWidth: Int) { + let offset: UInt64 = numericCast(writerIndex) &- offset + assert(byteWidth == 8 || offset < UInt64.one << (byteWidth * 8)) + _ = withUnsafePointer(to: offset) { + _bb.writeBytes($0, len: byteWidth) + } + } + + @inline(__always) + private mutating func write(value: T, byteWidth: Int) where T: Scalar { + _ = withUnsafePointer(to: value) { + _bb.writeBytes($0, len: byteWidth) + } + } + + @inline(__always) + private mutating func write(double value: Double, byteWidth: Int) { + switch byteWidth { + case 8: write(value: value, byteWidth: byteWidth) + case 4: write(value: Float(value), byteWidth: byteWidth) + default: assert(false, "Should never reach here") + } + } + + // MARK: Misc functions + @discardableResult + @inline(__always) + private mutating func align(width: BitWidth) -> Int { + let bytes: Int = numericCast(UInt32.one << width.rawValue) + _bb.addPadding(bytes: bytes) + return bytes + } + + @inline(__always) + private mutating func sortMapByKeys(start: Int) -> Int { + let len = mapElementCount(start: start) + for index in stride(from: start, to: stack.count, by: 2) { + assert(stack[index].type == .key) + } + + struct TwoValue: Equatable { + let key, value: Value + } + + stack[start...].withUnsafeMutableBytes { buffer in + var ptr = buffer.assumingMemoryBound(to: TwoValue.self) + ptr.sort { a, b in + let aMem = _bb.memory.advanced(by: numericCast(a.key.u)) + .assumingMemoryBound(to: CChar.self) + let bMem = _bb.memory.advanced(by: numericCast(b.key.u)) + .assumingMemoryBound(to: CChar.self) + let comp = strcmp(aMem, bMem) + if (comp == 0) && a != b { hasDuplicatedKeys = true } + return comp < 0 + } + } + return len + } + + @inline(__always) + private func mapElementCount(start: Int) -> Int { + var len = stack.count - start + assert((len & 1) == 0) + len /= 2 + return len + } +} + +// MARK: - Vectors helper functions +extension FlexBuffersWriter { + @discardableResult + @inline(__always) + public mutating func vector( + key: String, + _ closure: @escaping FlexBuffersWriterBuilder) -> UInt64 + { + let start = startVector(key: key) + closure(&self) + return endVector(start: start) + } + + @discardableResult + @inline(__always) + public mutating func vector(_ closure: @escaping FlexBuffersWriterBuilder) + -> UInt64 + { + let start = startVector() + closure(&self) + return endVector(start: start) + } +} + +// MARK: - Maps helper functions +extension FlexBuffersWriter { + @discardableResult + @inline(__always) + public mutating func map( + key: String, + _ closure: @escaping FlexBuffersWriterBuilder) -> UInt64 + { + let start = startMap(key: key) + closure(&self) + return endMap(start: start) + } + + @discardableResult + @inline(__always) + public mutating func map(_ closure: @escaping FlexBuffersWriterBuilder) + -> UInt64 + { + let start = startMap() + closure(&self) + return endMap(start: start) + } +} diff --git a/swift/Sources/FlexBuffers/_InternalByteBuffer.swift b/swift/Sources/FlexBuffers/_InternalByteBuffer.swift new file mode 100644 index 00000000000..f9c1ab41444 --- /dev/null +++ b/swift/Sources/FlexBuffers/_InternalByteBuffer.swift @@ -0,0 +1,221 @@ +/* + * Copyright 2024 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Common +import Foundation + +/// `ByteBuffer` is the interface that stores the data for a `Flatbuffers` object +/// it allows users to write and read data directly from memory thus the use of its +/// functions should be used +@usableFromInline +struct _InternalByteBuffer { + + /// Storage is a container that would hold the memory pointer to solve the issue of + /// deallocating the memory that was held by (memory: UnsafeMutableRawPointer) + @usableFromInline + final class Storage { + // This storage doesn't own the memory, therefore, we won't deallocate on deinit. + private let unowned: Bool + /// pointer to the start of the buffer object in memory + var memory: UnsafeMutableRawPointer + /// Capacity of UInt8 the buffer can hold + var capacity: Int + + @usableFromInline + init(count: Int, alignment: Int) { + memory = UnsafeMutableRawPointer.allocate( + byteCount: count, + alignment: alignment) + capacity = count + unowned = false + } + + @usableFromInline + init(memory: UnsafeMutableRawPointer, capacity: Int, unowned: Bool) { + self.memory = memory + self.capacity = capacity + self.unowned = unowned + } + + deinit { + if !unowned { + memory.deallocate() + } + } + + @usableFromInline + func copy(from ptr: UnsafeRawPointer, count: Int) { + assert( + !unowned, + "copy should NOT be called on a buffer that is built by assumingMemoryBound") + memory.copyMemory(from: ptr, byteCount: count) + } + + @usableFromInline + func initialize(for size: Int) { + assert( + !unowned, + "initalize should NOT be called on a buffer that is built by assumingMemoryBound") + memset(memory, 0, size) + } + + /// Reallocates the buffer incase the object to be written doesnt fit in the current buffer + /// - Parameter size: Size of the current object + @usableFromInline + func reallocate(_ size: Int, writerSize: Int, alignment: Int) { + while capacity <= writerSize &+ size { + capacity = capacity << 1 + } + + /// solution take from Apple-NIO + capacity = capacity.convertToPowerofTwo + + let newData = UnsafeMutableRawPointer.allocate( + byteCount: capacity, + alignment: alignment) + memset(newData, 0, capacity &- writerSize) + memcpy( + newData, + memory, + writerSize) + memory.deallocate() + memory = newData + } + } + + @usableFromInline var _storage: Storage + /// The size of the elements written to the buffer + their paddings + var writerIndex: Int = 0 + /// Alignment of the current memory being written to the buffer + private var alignment = 1 + /// Public Pointer to the buffer object in memory. This should NOT be modified for any reason + public var memory: UnsafeMutableRawPointer { _storage.memory } + /// Current capacity for the buffer + public var capacity: Int { _storage.capacity } + + /// Returns the written bytes into the ``ByteBuffer`` + public var underlyingBytes: [UInt8] { + let start = memory.bindMemory(to: UInt8.self, capacity: writerIndex) + + let ptr = UnsafeBufferPointer(start: start, count: writerIndex) + return Array(ptr) + } + + /// Constructor that creates a Flatbuffer instance with a size + /// - Parameter: + /// - size: Length of the buffer + /// - allowReadingUnalignedBuffers: allow reading from unaligned buffer + init(initialSize size: Int) { + let size = size.convertToPowerofTwo + _storage = Storage(count: size, alignment: alignment) + _storage.initialize(for: size) + } + + /// Clears the current instance of the buffer, replacing it with new memory + @inline(__always) + mutating public func clear() { + writerIndex = 0 + alignment = 1 + _storage.initialize(for: _storage.capacity) + } + + /// Makes sure that buffer has enouch space for each of the objects that will be written into it + /// - Parameter size: size of object + @discardableResult + @inline(__always) + mutating func ensureSpace(size: Int) { + if size &+ writerIndex > _storage.capacity { + _storage.reallocate(size, writerSize: writerIndex, alignment: alignment) + } + } + + @discardableResult + @inline(__always) + mutating func addPadding(bytes: Int) { + writerIndex = writerIndex &+ numericCast(padding( + bufSize: numericCast(writerIndex), + elementSize: numericCast(bytes))) + ensureSpace(size: writerIndex) + } + + mutating func writeBytes(_ ptr: UnsafeRawPointer, len: Int) { + memcpy( + _storage.memory.advanced(by: writerIndex), + ptr, + len) + writerIndex = writerIndex &+ len + } + + mutating func write(_ v: T, len: Int) { + _ = withUnsafePointer(to: v) { + memcpy( + _storage.memory.advanced(by: writerIndex), + $0, + len) + writerIndex = writerIndex &+ len + } + } + + @discardableResult + @inline(__always) + func withUnsafeBytes( + _ body: (UnsafeRawBufferPointer) throws + -> T) rethrows -> T + { + try body(UnsafeRawBufferPointer( + start: _storage.memory, + count: capacity)) + } + + @discardableResult + @inline(__always) + func withUnsafeSlicedBytes( + _ body: (UnsafeRawBufferPointer) throws + -> T) rethrows -> T + { + try body(UnsafeRawBufferPointer( + start: _storage.memory, + count: writerIndex)) + } + + @discardableResult + @inline(__always) + func withUnsafeRawPointer( + _ body: (UnsafeMutableRawPointer) throws + -> T) rethrows -> T + { + try body(_storage.memory) + } + + @discardableResult + @inline(__always) + func readWithUnsafeRawPointer( + position: Int, + _ body: (UnsafeRawPointer) throws -> T) rethrows -> T + { + try body(_storage.memory.advanced(by: position)) + } +} + +extension _InternalByteBuffer: CustomDebugStringConvertible { + + public var debugDescription: String { + """ + buffer located at: \(_storage.memory), with capacity of \(_storage.capacity) + { writerIndex: \(writerIndex) } + """ + } +} diff --git a/tests/swift/tests/Package.swift b/tests/swift/tests/Package.swift index d66380e044c..afbfae559be 100644 --- a/tests/swift/tests/Package.swift +++ b/tests/swift/tests/Package.swift @@ -29,7 +29,9 @@ let package = Package( // Prevent the build system from pulling 2.29.1 to prevent Swift 5.8 build breaks. // The patch update introduced code that uses "switch expression syntax" that wasn't valid until Swift 5.9 [1]. // [1] https://github.com/swiftlang/swift-evolution/blob/main/proposals/0380-if-switch-expressions.md - .package(url: "https://github.com/apple/swift-nio-ssl.git", exact: "2.29.0"), + .package( + url: "https://github.com/apple/swift-nio-ssl.git", + exact: "2.29.0"), ], targets: [ .executableTarget( @@ -43,4 +45,9 @@ let package = Package( .product(name: "FlatBuffers", package: "flatbuffers"), .product(name: "GRPC", package: "grpc-swift"), ]), + .testTarget( + name: "FlexBuffers.Test.SwiftTests", + dependencies: [ + .product(name: "FlexBuffers", package: "flatbuffers"), + ]), ]) diff --git a/tests/swift/tests/Tests/FlatBuffers.Test.SwiftTests/FlatBuffersTests.swift b/tests/swift/tests/Tests/FlatBuffers.Test.SwiftTests/FlatBuffersTests.swift index 13780d53dc4..e2971d274bf 100644 --- a/tests/swift/tests/Tests/FlatBuffers.Test.SwiftTests/FlatBuffersTests.swift +++ b/tests/swift/tests/Tests/FlatBuffers.Test.SwiftTests/FlatBuffersTests.swift @@ -15,6 +15,7 @@ */ import XCTest +@testable import Common @testable import FlatBuffers final class FlatBuffersTests: XCTestCase { diff --git a/tests/swift/tests/Tests/FlexBuffers.Test.SwiftTests/FlexBuffersWriterTests.swift b/tests/swift/tests/Tests/FlexBuffers.Test.SwiftTests/FlexBuffersWriterTests.swift new file mode 100644 index 00000000000..5f086bb50d5 --- /dev/null +++ b/tests/swift/tests/Tests/FlexBuffers.Test.SwiftTests/FlexBuffersWriterTests.swift @@ -0,0 +1,233 @@ +/* + * Copyright 2024 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Common +import FlexBuffers +import XCTest + +final class FlexBuffersWriterTests: XCTestCase { + func testDeallocation() { + let buf: ByteBuffer = { + var fbx = FlexBuffersWriter() + fbx.add(string: "Hello") + fbx.finish() + return fbx.sizedByteBuffer + }() + + buf.withUnsafeBytes { + XCTAssertEqual( + Array($0), + [5, 72, 101, 108, 108, 111, 0, 6, 20, 1]) + } + } + + func testAddingVectorOfScalars() { + var fbx = FlexBuffersWriter() + fbx.vector { + let arr: [Int32] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 20] + $0.create(vector: arr) + } + fbx.finish() + let buf: ByteBuffer = fbx.sizedByteBuffer + + buf.withUnsafeBytes { + // swiftformat:disable all + XCTAssertEqual( + Array($0), + [10, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 4, 0, 0, 0, 5, 0, 0, 0, 6, 0, 0, 0, 7, 0, 0, 0, 8, 0, 0, 0, 9, 0, 0, 0, 20, 0, 0, 0, 1, 41, 46, 2, 40, 1]) + // swiftformat:enable all + } + } + + func testAddingVectorOfUnsignedScalars() { + var fbx = FlexBuffersWriter() + fbx.vector { + let arr: [UInt] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 20] + $0.create(vector: arr) + } + fbx.finish() + let buf: ByteBuffer = fbx.sizedByteBuffer + + buf.withUnsafeBytes { + // swiftformat:disable all + XCTAssertEqual( + Array($0), + [10, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 1, 81, 51, 2, 40, 1]) + // swiftformat:enable all + } + } + + func testAddingVectorOfBools() { + var fbx = FlexBuffersWriter() + fbx.vector { + let arr: [Bool] = [true, false, true, false] + $0.create(vector: arr) + } + fbx.finish() + let buf: ByteBuffer = fbx.sizedByteBuffer + + buf.withUnsafeBytes { + // swiftformat:disable all + XCTAssertEqual( + Array($0), + [4, 1, 0, 1, 0, 1, 5, 144, 2, 40, 1]) + // swiftformat:enable all + } + } + + func testSortingWithinMap() { + var fbx = FlexBuffersWriter() + fbx.map { + $0.add(bool: false, key: "bool2") + $0.add(bool: true, key: "bool1") + } + fbx.finish() + let buf: ByteBuffer = fbx.sizedByteBuffer + buf.withUnsafeBytes { + // swiftformat:disable all + XCTAssertEqual( + Array($0), + [98, 111, 111, 108, 50, 0, 98, 111, 111, 108, 49, 0, 2, 7, 14, 2, 1, 2, 1, 0, 104, 104, 4, 36, 1] + ) + // swiftformat:enable all + } + } + + func testSharingKeyWithinMap() { + var fbx = FlexBuffersWriter(initialSize: 1000, flags: .shareKeysAndStrings) + fbx.map { + $0.add(string: "welcome", key: "welcome") + $0.add(string: "welcome", key: "welcome") + $0.add(string: "welcome", key: "welcome") + } + fbx.finish() + let buf: ByteBuffer = fbx.sizedByteBuffer + buf.withUnsafeBytes { + // swiftformat:disable all + XCTAssertEqual( + Array($0), + [119, 101, 108, 99, 111, 109, 101, 0, 7, 119, 101, 108, 99, 111, 109, 101, 0, 3, 18, 19, 20, 3, 1, 3, 15, 16, 17, 20, 20, 20, 6, 36, 1] + ) + // swiftformat:enable all + } + } + + func testNestingVectorInMap() { + var fbx = FlexBuffersWriter( + initialSize: 8, + flags: .shareKeysAndStrings) + + fbx.map { map in + map.vector(key: "vec") { v in + v.add(int64: -100) + v.add(string: "Fred") + v.indirect(float32: 4.0) + let lv = v.lastValue() + let blob: [UInt8] = [77] + v.add(blob: blob, length: blob.count) + v.add(bool: false) + v.reuse(value: lv!) + } + let ints: [Int32] = [1, 2, 3] + map.create(vector: ints, key: "bar") + map.createFixed(vector: ints, key: "bar3") + let bools = [true, false, true, false] + map.create(vector: bools, key: "bools") + map.add(bool: true, key: "bool") + map.add(double: 100, key: "foo") + map.map(key: "mymap") { m in + m.add(string: "Fred", key: "foo") + } + } + + fbx.finish() + let buf: ByteBuffer = fbx.sizedByteBuffer + buf.withUnsafeBytes { + // swiftformat:disable all + XCTAssertEqual( + Array($0), + flexbufferGolden + ) + // swiftformat:enable all + } + } + + func testAddingNil() { + var fbx = FlexBuffersWriter( + initialSize: 8, + flags: .shareKeysAndStrings) + + fbx.map { map in + map.addNil(key: "v") + } + + fbx.finish() + let buf: ByteBuffer = fbx.sizedByteBuffer + buf.withUnsafeBytes { + // swiftformat:disable all + XCTAssertEqual( + Array($0), + [118, 0, 1, 3, 1, 1, 1, 0, 0, 2, 36, 1] + ) + // swiftformat:enable all + } + } + + func testAddingManually() { + var fbx = FlexBuffersWriter( + initialSize: 8, + flags: .shareKeysAndStrings) + + let outerMap = fbx.startMap() + + let vector = fbx.startVector(key: "vec") + fbx.add(int64: -100) + fbx.add(string: "Fred") + fbx.indirect(float32: 4.0) + let lv = fbx.lastValue() + let blob: [UInt8] = [77] + fbx.add(blob: blob, length: blob.count) + fbx.add(bool: false) + fbx.reuse(value: lv!) + fbx.endVector(start: vector) + + let ints: [Int32] = [1, 2, 3] + fbx.create(vector: ints, key: "bar") + fbx.createFixed(vector: ints, key: "bar3") + let bools = [true, false, true, false] + fbx.create(vector: bools, key: "bools") + fbx.add(bool: true, key: "bool") + fbx.add(double: 100, key: "foo") + + let innerMap = fbx.startMap(key: "mymap") + fbx.add(string: "Fred", key: "foo") + fbx.endMap(start: innerMap) + + fbx.endMap(start: outerMap) + + + fbx.finish() + let buf: ByteBuffer = fbx.sizedByteBuffer + buf.withUnsafeBytes { + // swiftformat:disable all + XCTAssertEqual( + Array($0), + flexbufferGolden + ) + // swiftformat:enable all + } + } +} diff --git a/tests/swift/tests/Tests/FlexBuffers.Test.SwiftTests/Mocks.swift b/tests/swift/tests/Tests/FlexBuffers.Test.SwiftTests/Mocks.swift new file mode 100644 index 00000000000..3c31b0e4f41 --- /dev/null +++ b/tests/swift/tests/Tests/FlexBuffers.Test.SwiftTests/Mocks.swift @@ -0,0 +1,19 @@ +/* + * Copyright 2024 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// swiftformat:disable all +let flexbufferGolden: [UInt8] = [118, 101, 99, 0, 4, 70, 114, 101, 100, 0, 0, 0, 0, 0, 128, 64, 1, 77, 6, 156, 15, 9, 5, 0, 12, 4, 20, 34, 100, 104, 34, 98, 97, 114, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 98, 97, 114, 51, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 98, 111, 111, 108, 115, 0, 4, 1, 0, 1, 0, 98, 111, 111, 108, 0, 102, 111, 111, 0, 109, 121, 109, 97, 112, 0, 1, 11, 1, 1, 1, 98, 20, 7, 75, 55, 25, 37, 22, 19, 112, 0, 0, 0, 10, 0, 0, 0, 1, 0, 0, 0, 7, 0, 0, 0, 88, 0, 0, 0, 72, 0, 0, 0, 1, 0, 0, 0, 61, 0, 0, 0, 0, 0, 200, 66, 45, 0, 0, 0, 133, 0, 0, 0, 46, 78, 106, 144, 14, 36, 40, 35, 38, 1] +// swiftformat:enable all