Merge pull request #122 from giginet/separate-storage

Remove ScipioS3Storage and ScipioStorage
giginet · May 24, 2024 · cd10499 · cd10499
2 parents c70ddfd + 3e1af2c
commit cd10499
Show file tree

Hide file tree

Showing 17 changed files with 43 additions and 57,666 deletions.
diff --git a/Package.swift b/Package.swift
@@ -20,9 +20,6 @@ let package = Package(
         .library(
             name: "ScipioKit",
             targets: ["ScipioKit"]),
-        .library(
-            name: "ScipioS3Storage",
-            targets: ["ScipioS3Storage"]),
     ],
     dependencies: [
         .package(url: "https://github.com/apple/swift-package-manager.git",
@@ -37,10 +34,8 @@ let package = Package(
                  from: "1.0.0"),
         .package(url: "https://github.com/onevcat/Rainbow",
                  .upToNextMinor(from: "4.0.1")),
-        .package(url: "https://github.com/soto-project/soto-codegenerator",
-                 from: "0.6.0"),
-        .package(url: "https://github.com/soto-project/soto-core.git",
-                 from: "6.4.0"),
+        .package(url: "https://github.com/giginet/scipio-cache-storage.git",
+                 from: "1.0.0"),
     ],
     targets: [
         .executableTarget(
@@ -60,6 +55,7 @@ let package = Package(
                 .product(name: "Collections", package: "swift-collections"),
                 .product(name: "Algorithms", package: "swift-algorithms"),
                 .product(name: "Rainbow", package: "Rainbow"),
+                .product(name: "ScipioStorage", package: "scipio-cache-storage"),
             ],
             swiftSettings: swiftSettings,
             plugins: [
@@ -70,17 +66,6 @@ let package = Package(
             name: "GenerateScipioVersion",
             capability: .buildTool()
         ),
-        .target(
-            name: "ScipioS3Storage",
-            dependencies: [
-                .target(name: "ScipioKit"),
-                .product(name: "SotoCore", package: "soto-core"),
-            ],
-            swiftSettings: swiftSettings,
-            plugins: [
-                .plugin(name: "SotoCodeGeneratorPlugin", package: "soto-codegenerator"),
-            ]
-        ),
         .testTarget(
             name: "ScipioKitTests",
             dependencies: [
@@ -90,9 +75,6 @@ let package = Package(
             resources: [.copy("Resources/Fixtures")],
             swiftSettings: swiftSettings
         ),
-        .testTarget(
-            name: "ScipioS3StorageTests",
-            dependencies: ["ScipioS3Storage"]),
     ]
 )
 

diff --git a/Package@swift-5.8.swift b/Package@swift-5.8.swift
@@ -15,9 +15,6 @@ let package = Package(
         .library(
             name: "ScipioKit",
             targets: ["ScipioKit"]),
-        .library(
-            name: "ScipioS3Storage",
-            targets: ["ScipioS3Storage"]),
     ],
     dependencies: [
         .package(url: "https://github.com/giginet/swift-package-manager.git",
@@ -32,10 +29,8 @@ let package = Package(
                  from: "1.0.0"),
         .package(url: "https://github.com/onevcat/Rainbow",
                  .upToNextMinor(from: "4.0.1")),
-        .package(url: "https://github.com/soto-project/soto-codegenerator", 
-                 from: "0.6.0"),
-        .package(url: "https://github.com/soto-project/soto-core.git", 
-                 from: "6.4.0"),
+        .package(url: "https://github.com/giginet/scipio-cache-storage.git",
+                 from: "1.0.0"),
     ],
     targets: [
         .executableTarget(name: "scipio",
@@ -52,6 +47,7 @@ let package = Package(
                 .product(name: "Collections", package: "swift-collections"),
                 .product(name: "Algorithms", package: "swift-algorithms"),
                 .product(name: "Rainbow", package: "Rainbow"),
+                .product(name: "ScipioStorage", package: "scipio-cache-storage"),
             ],
             plugins: [
                 .plugin(name: "GenerateScipioVersion")
@@ -61,26 +57,13 @@ let package = Package(
             name: "GenerateScipioVersion", 
             capability: .buildTool()
         ),
-        .target(
-            name: "ScipioS3Storage",
-            dependencies: [
-                .target(name: "ScipioKit"),
-                .product(name: "SotoCore", package: "soto-core"),
-            ],
-            plugins: [
-                .plugin(name: "SotoCodeGeneratorPlugin", package: "soto-codegenerator"),
-            ]
-        ),
         .testTarget(
             name: "ScipioKitTests",
             dependencies: [
                 .target(name: "ScipioKit"),
             ],
             exclude: ["Resources/Fixtures/"],
             resources: [.copy("Resources/Fixtures")]),
-        .testTarget(
-            name: "ScipioS3StorageTests",
-            dependencies: ["ScipioS3Storage"]),
     ]
 )
 

diff --git a/Package@swift-5.9.swift b/Package@swift-5.9.swift
@@ -15,9 +15,6 @@ let package = Package(
         .library(
             name: "ScipioKit",
             targets: ["ScipioKit"]),
-        .library(
-            name: "ScipioS3Storage",
-            targets: ["ScipioS3Storage"]),
     ],
     dependencies: [
         .package(url: "https://github.com/apple/swift-package-manager.git",
@@ -32,10 +29,8 @@ let package = Package(
                  from: "1.0.0"),
         .package(url: "https://github.com/onevcat/Rainbow",
                  .upToNextMinor(from: "4.0.1")),
-        .package(url: "https://github.com/soto-project/soto-codegenerator", 
-                 from: "0.6.0"),
-        .package(url: "https://github.com/soto-project/soto-core.git", 
-                 from: "6.4.0"),
+        .package(url: "https://github.com/giginet/scipio-cache-storage.git",
+                 from: "1.0.0"),
     ],
     targets: [
         .executableTarget(name: "scipio",
@@ -52,6 +47,7 @@ let package = Package(
                 .product(name: "Collections", package: "swift-collections"),
                 .product(name: "Algorithms", package: "swift-algorithms"),
                 .product(name: "Rainbow", package: "Rainbow"),
+                .product(name: "ScipioStorage", package: "scipio-cache-storage"),
             ],
             plugins: [
                 .plugin(name: "GenerateScipioVersion")
@@ -61,26 +57,13 @@ let package = Package(
             name: "GenerateScipioVersion", 
             capability: .buildTool()
         ),
-        .target(
-            name: "ScipioS3Storage",
-            dependencies: [
-                .target(name: "ScipioKit"),
-                .product(name: "SotoCore", package: "soto-core"),
-            ],
-            plugins: [
-                .plugin(name: "SotoCodeGeneratorPlugin", package: "soto-codegenerator"),
-            ]
-        ),
         .testTarget(
             name: "ScipioKitTests",
             dependencies: [
                 .target(name: "ScipioKit"),
             ],
             exclude: ["Resources/Fixtures/"],
             resources: [.copy("Resources/Fixtures")]),
-        .testTarget(
-            name: "ScipioS3StorageTests",
-            dependencies: ["ScipioS3Storage"]),
     ]
 )
 

diff --git a/Sources/ScipioKit/Producer/Cache/CacheSystem.swift b/Sources/ScipioKit/Producer/Cache/CacheSystem.swift
@@ -1,4 +1,5 @@
 import Foundation
+import ScipioStorage
 import TSCBasic
 import struct TSCUtility.Version
 @preconcurrency import class PackageGraph.PinsStore
@@ -91,7 +92,7 @@ extension PinsStore.PinState: Hashable {
     }
 }
 
-public struct CacheKey: Hashable, Codable, Equatable {
+public struct SwiftPMCacheKey: CacheKey {
     public var targetName: String
     public var pin: PinsStore.PinState
     var buildOptions: BuildOptions
@@ -100,25 +101,6 @@ public struct CacheKey: Hashable, Codable, Equatable {
     public var scipioVersion: String?
 }
 
-extension CacheKey {
-    public var frameworkName: String {
-        "\(targetName.packageNamed()).xcframework"
-    }
-}
-
-public protocol CacheStorage: Sendable {
-    func existsValidCache(for cacheKey: CacheKey) async throws -> Bool
-    func fetchArtifacts(for cacheKey: CacheKey, to destinationDir: URL) async throws
-    func cacheFramework(_ frameworkPath: URL, for cacheKey: CacheKey) async throws
-    var paralellNumber: Int? { get }
-}
-
-extension CacheStorage {
-    public var paralellNumber: Int? {
-        nil
-    }
-}
-
 struct CacheSystem: Sendable {
     static let defaultParalellNumber = 8
     private let pinsStore: PinsStore
@@ -164,7 +146,7 @@ struct CacheSystem: Sendable {
     }
 
     func cacheFrameworks(_ targets: Set<CacheTarget>) async {
-        let chunked = targets.chunks(ofCount: storage?.paralellNumber ?? CacheSystem.defaultParalellNumber)
+        let chunked = targets.chunks(ofCount: storage?.parallelNumber ?? CacheSystem.defaultParalellNumber)
 
         for chunk in chunked {
             await withTaskGroup(of: Void.self) { group in
@@ -205,15 +187,15 @@ struct CacheSystem: Sendable {
         )
     }
 
-    func existsValidCache(cacheKey: CacheKey) async -> Bool {
+    func existsValidCache(cacheKey: SwiftPMCacheKey) async -> Bool {
         do {
             let versionFilePath = versionFilePath(for: cacheKey.targetName)
             guard fileSystem.exists(versionFilePath.absolutePath) else { return false }
             let decoder = JSONDecoder()
             guard let contents = try? fileSystem.readFileContents(versionFilePath.absolutePath).contents else {
                 throw Error.couldNotReadVersionFile(versionFilePath)
             }
-            let versionFileKey = try decoder.decode(CacheKey.self, from: Data(contents))
+            let versionFileKey = try decoder.decode(SwiftPMCacheKey.self, from: Data(contents))
             return versionFileKey == cacheKey
         } catch {
             return false
@@ -246,7 +228,7 @@ struct CacheSystem: Sendable {
         try await storage.fetchArtifacts(for: cacheKey, to: destination)
     }
 
-    func calculateCacheKey(of target: CacheTarget) async throws -> CacheKey {
+    func calculateCacheKey(of target: CacheTarget) async throws -> SwiftPMCacheKey {
         let targetName = target.buildProduct.target.name
         let pin = try retrievePin(product: target.buildProduct)
         let buildOptions = target.buildOptions
@@ -256,7 +238,7 @@ struct CacheSystem: Sendable {
         guard let xcodeVersion = try await XcodeVersionFetcher().fetchXcodeVersion() else {
             throw Error.xcodeVersionNotDetected
         }
-        return CacheKey(
+        return SwiftPMCacheKey(
             targetName: targetName,
             pin: pin.state,
             buildOptions: buildOptions,
@@ -288,25 +270,18 @@ struct CacheSystem: Sendable {
     }
 }
 
-extension CacheKey {
-    public func calculateChecksum() throws -> String {
-        let data = try jsonEncoder.encode(self)
-        return SHA256().hash(ByteString(data)).hexadecimalRepresentation
-    }
-}
-
 public struct VersionFileDecoder {
     private let fileSystem: any FileSystem
 
     public init(fileSystem: any FileSystem = localFileSystem) {
         self.fileSystem = fileSystem
     }
 
-    public func decode(versionFile: URL) throws -> CacheKey {
+    public func decode(versionFile: URL) throws -> SwiftPMCacheKey {
         try jsonDecoder.decode(
             path: versionFile.absolutePath.spmAbsolutePath,
             fileSystem: fileSystem,
-            as: CacheKey.self
+            as: SwiftPMCacheKey.self
         )
     }
 }
diff --git a/Sources/ScipioKit/Producer/Cache/LocalCacheStorage.swift b/Sources/ScipioKit/Producer/Cache/LocalCacheStorage.swift
@@ -1,10 +1,13 @@
 import Foundation
+import ScipioStorage
 import PackageGraph
 import TSCBasic
 
 public struct LocalCacheStorage: CacheStorage {
     private let fileSystem: any FileSystem
 
+    public var parallelNumber: Int? { nil }
+
     enum Error: Swift.Error {
         case cacheDirectoryIsNotFound
     }
@@ -35,11 +38,11 @@ public struct LocalCacheStorage: CacheStorage {
         return cacheDir.appendingPathComponent("Scipio")
     }
 
-    private func xcFrameworkFileName(for cacheKey: CacheKey) -> String {
+    private func xcFrameworkFileName(for cacheKey: some CacheKey) -> String {
         "\(cacheKey.targetName.packageNamed()).xcframework"
     }
 
-    private func cacheFrameworkPath(for cacheKey: CacheKey) throws -> URL {
+    private func cacheFrameworkPath(for cacheKey: some CacheKey) throws -> URL {
         let baseDirectory = try buildBaseDirectoryPath()
         let checksum = try cacheKey.calculateChecksum()
         return baseDirectory
@@ -48,7 +51,7 @@ public struct LocalCacheStorage: CacheStorage {
             .appendingPathComponent(xcFrameworkFileName(for: cacheKey))
     }
 
-    public func existsValidCache(for cacheKey: CacheKey) async -> Bool {
+    public func existsValidCache(for cacheKey: some CacheKey) async -> Bool {
         do {
             let xcFrameworkPath = try cacheFrameworkPath(for: cacheKey)
             return fileSystem.exists(xcFrameworkPath.absolutePath)
@@ -57,7 +60,7 @@ public struct LocalCacheStorage: CacheStorage {
         }
     }
 
-    public func cacheFramework(_ frameworkPath: URL, for cacheKey: CacheKey) async {
+    public func cacheFramework(_ frameworkPath: URL, for cacheKey: some CacheKey) async {
         do {
             let destination = try cacheFrameworkPath(for: cacheKey)
             let directoryPath = destination.deletingLastPathComponent()
@@ -69,7 +72,7 @@ public struct LocalCacheStorage: CacheStorage {
         }
     }
 
-    public func fetchArtifacts(for cacheKey: CacheKey, to destinationDir: URL) async throws {
+    public func fetchArtifacts(for cacheKey: some CacheKey, to destinationDir: URL) async throws {
         let source = try cacheFrameworkPath(for: cacheKey)
         let destination = destinationDir.appendingPathComponent(xcFrameworkFileName(for: cacheKey))
         try fileSystem.copy(from: source.absolutePath, to: destination.absolutePath)

diff --git a/Sources/ScipioKit/Producer/FrameworkProducer.swift b/Sources/ScipioKit/Producer/FrameworkProducer.swift
@@ -1,4 +1,5 @@
 import Foundation
+import ScipioStorage
 import PackageGraph
 import PackageModel
 import Collections
@@ -141,7 +142,7 @@ struct FrameworkProducer {
         availableTargets: Set<CacheSystem.CacheTarget>,
         cacheSystem: CacheSystem
     ) async -> Set<CacheSystem.CacheTarget> {
-        let chunked = availableTargets.chunks(ofCount: cacheStorage?.paralellNumber ?? CacheSystem.defaultParalellNumber)
+        let chunked = availableTargets.chunks(ofCount: cacheStorage?.parallelNumber ?? CacheSystem.defaultParalellNumber)
 
         var restored: Set<CacheSystem.CacheTarget> = []
         for chunk in chunked {

diff --git a/Sources/ScipioKit/Producer/PIF/XCBuildExecutor.swift b/Sources/ScipioKit/Producer/PIF/XCBuildExecutor.swift
@@ -2,6 +2,7 @@ import Logging
 import TSCBasic
 import PackageGraph
 import XCBuildSupport
+import Algorithms
 
 struct XCBuildExecutor {
 

diff --git a/Sources/ScipioKit/Runner.swift b/Sources/ScipioKit/Runner.swift
@@ -1,4 +1,5 @@
 import Foundation
+import ScipioStorage
 import struct TSCBasic.AbsolutePath
 import protocol TSCBasic.FileSystem
 import var TSCBasic.localFileSystem