apacheGH-37938: [Swift] initial impl of C Data interface

ci/docker/ubuntu-swift.dockerfile

@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-FROM swift:5.7.3
+FROM swift:5.9.0
 
 # Go is needed for generating test data
 RUN apt-get update -y -q && \

dev/release/rat_exclude_files.txt

@@ -150,3 +150,4 @@ r/tools/nixlibs-allowlist.txt
 ruby/red-arrow/.yardopts
 .github/pull_request_template.md
 swift/data-generator/swift-datagen/go.sum
+swift/CDataWGo/go.sum

swift/.swiftlint.yml

@@ -20,6 +20,7 @@ included:
  - Arrow/Tests
  - ArrowFlight/Sources
  - ArrowFlight/Tests
+ - CDataWGo/Sources/go-swift
 excluded:
  - Arrow/Sources/Arrow/File_generated.swift
  - Arrow/Sources/Arrow/Message_generated.swift

swift/Arrow/Package.swift

@@ -36,18 +36,27 @@ let package = Package(
  // and therefore doesn't include the unaligned buffer swift changes.
  // This can be changed back to using the tag once a new version of
  // flatbuffers has been released.
- .package(url: "https://github.com/google/flatbuffers.git", branch: "master")
+ .package(url: "https://github.com/google/flatbuffers.git", branch: "master"),
+ .package(
+ url: "https://github.com/apple/swift-atomics.git",
+ .upToNextMajor(from: "1.2.0") // or `.upToNextMinor
+ )
  ],
  targets: [
  // Targets are the basic building blocks of a package. A target can define a module or a test suite.
  // Targets can depend on other targets in this package, and on products in packages this package depends on.
+ .target(
+ name: "ArrowC",
+ path: "Sources/ArrowC"
+ ),
  .target(
  name: "Arrow",
- dependencies: [
- .product(name: "FlatBuffers", package: "flatbuffers")
+ dependencies: ["ArrowC",
+ .product(name: "FlatBuffers", package: "flatbuffers"),
+ .product(name: "Atomics", package: "swift-atomics")
  ]),
  .testTarget(
  name: "ArrowTests",
- dependencies: ["Arrow"]),
+ dependencies: ["Arrow", "ArrowC"]),
  ]
 )

swift/Arrow/Sources/Arrow/ArrowArray.swift

@@ -17,14 +17,24 @@
 
 import Foundation
 
-public class ArrowArrayHolder {
+public protocol ArrowArrayHolder {
+ var type: ArrowType {get}
+ var length: UInt {get}
+ var nullCount: UInt {get}
+ var array: Any {get}
+ var getBufferData: () -> [Data] {get}
+ var getBufferDataSizes: () -> [Int] {get}
+ var getArrowColumn: (ArrowField, [ArrowArrayHolder]) throws -> ArrowColumn {get}
+}
+
+public class ArrowArrayHolderImpl: ArrowArrayHolder {
  public let type: ArrowType
  public let length: UInt
  public let nullCount: UInt
  public let array: Any
  public let getBufferData: () -> [Data]
  public let getBufferDataSizes: () -> [Int]
- private let getArrowColumn: (ArrowField, [ArrowArrayHolder]) throws -> ArrowColumn
+ public let getArrowColumn: (ArrowField, [ArrowArrayHolder]) throws -> ArrowColumn
  public init<T>(_ arrowArray: ArrowArray<T>) {
  self.array = arrowArray
  self.length = arrowArray.length
@@ -60,19 +70,9 @@ public class ArrowArrayHolder {
  return ArrowColumn(field, chunked: ChunkedArrayHolder(try ChunkedArray<T>(arrays)))
  }
  }
-
- public static func makeArrowColumn(_ field: ArrowField,
- holders: [ArrowArrayHolder]
- ) -> Result<ArrowColumn, ArrowError> {
- do {
- return .success(try holders[0].getArrowColumn(field, holders))
- } catch {
- return .failure(.runtimeError("\(error)"))
- }
- }
 }
 
-public class ArrowArray<T>: AsString {
+public class ArrowArray<T>: AsString, AnyArray {
  public typealias ItemType = T
  public let arrowData: ArrowData
  public var nullCount: UInt {return self.arrowData.nullCount}
@@ -101,6 +101,14 @@ public class ArrowArray<T>: AsString {
 
  return "\(self[index]!)"
  }
+
+ public func asAny(_ index: UInt) -> Any? {
+ if self[index] == nil {
+ return nil
+ }
+
+ return self[index]!
+ }
 }
 
 public class FixedArray<T>: ArrowArray<T> {

swift/Arrow/Sources/Arrow/ArrowBuffer.swift

@@ -23,22 +23,33 @@ public class ArrowBuffer {
  fileprivate(set) var length: UInt
  let capacity: UInt
  let rawPointer: UnsafeMutableRawPointer
+ let isMemoryOwner: Bool
 
- init(length: UInt, capacity: UInt, rawPointer: UnsafeMutableRawPointer) {
+ init(length: UInt, capacity: UInt, rawPointer: UnsafeMutableRawPointer, isMemoryOwner: Bool = true) {
  self.length = length
  self.capacity = capacity
  self.rawPointer = rawPointer
+ self.isMemoryOwner = isMemoryOwner
  }
 
  deinit {
- self.rawPointer.deallocate()
+ if isMemoryOwner {
+ self.rawPointer.deallocate()
+ }
  }
 
  func append(to data: inout Data) {
  let ptr = UnsafePointer(rawPointer.assumingMemoryBound(to: UInt8.self))
  data.append(ptr, count: Int(capacity))
  }
 
+ static func createEmptyBuffer() -> ArrowBuffer {
+ return ArrowBuffer(
+ length: 0,
+ capacity: 0,
+ rawPointer: UnsafeMutableRawPointer.allocate(byteCount: 0, alignment: .zero))
+ }
+
  static func createBuffer(_ data: [UInt8], length: UInt) -> ArrowBuffer {
  let byteCount = UInt(data.count)
  let capacity = alignTo64(byteCount)

swift/Arrow/Sources/Arrow/ArrowCExporter.swift

@@ -0,0 +1,133 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import Foundation
+import ArrowC
+import Atomics
+
+// The memory used by UnsafeAtomic is not automatically
+// reclaimed. Since this value is initialized once
+// and used until the program/app is closed it's
+// memory will be released on program/app exit
+let exportDataCounter: UnsafeAtomic<Int> = .create(0)
+
+public class ArrowCExporter {
+ private class ExportData {
+ let id: Int
+ init() {
+ id = exportDataCounter.loadThenWrappingIncrement(ordering: .relaxed)
+ ArrowCExporter.exportedData[id] = self
+ }
+ }
+
+ private class ExportSchema: ExportData {
+ public let arrowTypeName: UnsafePointer<CChar>
+ public let nameCstr: UnsafePointer<CChar>
+ private let arrowType: ArrowType
+ private let name: String
+ init(_ arrowType: ArrowType, name: String = "") throws {
+ self.arrowType = arrowType
+ // keeping the name str to ensure the cstring buffer remains valid
+ self.name = name
+ self.arrowTypeName = (try arrowType.cDataFormatId as NSString).utf8String!
+ self.nameCstr = (name as NSString).utf8String!
+ super.init()
+ }
+ }
+
+ private class ExportArray: ExportData {
+ private let arrowData: ArrowData
+ private(set) var data = [UnsafeRawPointer?]()
+ private(set) var buffers: UnsafeMutablePointer<UnsafeRawPointer?>
+ init(_ arrowData: ArrowData) {
+ // keep a reference to the ArrowData
+ // obj so the memory doesn't get
+ // deallocated
+ self.arrowData = arrowData
+ for arrowBuffer in arrowData.buffers {
+ data.append(arrowBuffer.rawPointer)
+ }
+
+ self.buffers = UnsafeMutablePointer(mutating: data)
+ super.init()
+ }
+ }
+
+ private static var exportedData = [Int: ExportData]()
+ public init() {}
+
+ public func exportType(_ cSchema: inout ArrowC.ArrowSchema, arrowType: ArrowType, name: String = "") ->
+ Result<Bool, ArrowError> {
+ do {
+ let exportSchema = try ExportSchema(arrowType, name: name)
+ cSchema.format = exportSchema.arrowTypeName
+ cSchema.name = exportSchema.nameCstr
+ cSchema.private_data =
+ UnsafeMutableRawPointer(mutating: UnsafeRawPointer(bitPattern: exportSchema.id))
+ cSchema.release = {(data: UnsafeMutablePointer<ArrowC.ArrowSchema>?) in
+ let arraySchema = data!.pointee
+ let exportId = Int(bitPattern: arraySchema.private_data)
+ guard ArrowCExporter.exportedData[exportId] != nil else {
+ fatalError("Export schema not found with id \(exportId)")
+ }
+
+ // the data associated with this exportSchema object
+ // which includes the C strings for the format and name
+ // be deallocated upon removal
+ ArrowCExporter.exportedData.removeValue(forKey: exportId)
+ ArrowC.ArrowSwiftClearReleaseSchema(data)
+ }
+ } catch {
+ return .failure(.unknownError("\(error)"))
+ }
+ return .success(true)
+ }
+
+ public func exportField(_ schema: inout ArrowC.ArrowSchema, field: ArrowField) ->
+ Result<Bool, ArrowError> {
+ return exportType(&schema, arrowType: field.type, name: field.name)
+ }
+
+ public func exportArray(_ cArray: inout ArrowC.ArrowArray, arrowData: ArrowData) {
+ let exportArray = ExportArray(arrowData)
+ cArray.buffers = exportArray.buffers
+ cArray.length = Int64(arrowData.length)
+ cArray.null_count = Int64(arrowData.nullCount)
+ cArray.n_buffers = Int64(arrowData.buffers.count)
+ // Swift Arrow does not currently support children or dictionaries
+ // This will need to be updated once support has been added
+ cArray.n_children = 0
+ cArray.children = nil
+ cArray.dictionary = nil
+ cArray.private_data =
+ UnsafeMutableRawPointer(mutating: UnsafeRawPointer(bitPattern: exportArray.id))
+ cArray.release = {(data: UnsafeMutablePointer<ArrowC.ArrowArray>?) in
+ let arrayData = data!.pointee
+ let exportId = Int(bitPattern: arrayData.private_data)
+ guard ArrowCExporter.exportedData[exportId] != nil else {
+ fatalError("Export data not found with id \(exportId)")
+ }
+
+ // the data associated with this exportArray object
+ // which includes the entire arrowData object
+ // and the buffers UnsafeMutablePointer[] will
+ // be deallocated upon removal
+ ArrowCExporter.exportedData.removeValue(forKey: exportId)
+ ArrowC.ArrowSwiftClearReleaseArray(data)
+ }
+ }
+}