Merge pull request #156 from ikesyo/multiple-cache-storages

Add multiple cache storages feature
giginet · Nov 18, 2024 · 6c1e30a · 6c1e30a
2 parents 8bab62a + 2720cde
commit 6c1e30a
Show file tree

Hide file tree

Showing 17 changed files with 317 additions and 142 deletions.
diff --git a/.github/workflows/docc.yml b/.github/workflows/docc.yml
@@ -8,7 +8,7 @@ env:
   DEVELOPER_DIR: "/Applications/Xcode_16.0.app/Contents/Developer"
 jobs:
   DocC:
-    runs-on: macos-14
+    runs-on: macos-15
     steps:
       - uses: actions/checkout@v4
       - name: Build DocC

diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
@@ -14,7 +14,7 @@ jobs:
         xcode_version: ["16.0"]
     env: 
       DEVELOPER_DIR: "/Applications/Xcode_${{ matrix.xcode_version }}.app/Contents/Developer"
-    runs-on: macos-14
+    runs-on: macos-15
     steps:
     - uses: actions/checkout@v4
     - name: Install SwiftLint

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -10,7 +10,7 @@ env:
 jobs:
   release:
     name: Build and Upload Artifact Bundle
-    runs-on: macos-14
+    runs-on: macos-15
     steps:
       - uses: actions/checkout@v4
       - name: Resolve Dependencies

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -15,7 +15,7 @@ jobs:
           - "16.0" # 6.0
     env: 
       DEVELOPER_DIR: "/Applications/Xcode_${{ matrix.xcode_version }}.app/Contents/Developer"
-    runs-on: macos-14
+    runs-on: macos-15
     steps:
     - name: Get swift version
       run: swift --version

diff --git a/Sources/ScipioKit/Producer/Cache/CacheSystem.swift b/Sources/ScipioKit/Producer/Cache/CacheSystem.swift
@@ -108,7 +108,6 @@ struct CacheSystem: Sendable {
     static let defaultParalellNumber = 8
     private let pinsStore: PinsStore
     private let outputDirectory: URL
-    private let storage: (any CacheStorage)?
     private let fileSystem: any FileSystem
 
     struct CacheTarget: Hashable, Sendable {
@@ -139,18 +138,23 @@ struct CacheSystem: Sendable {
     init(
         pinsStore: PinsStore,
         outputDirectory: URL,
-        storage: (any CacheStorage)?,
         fileSystem: any FileSystem = localFileSystem
     ) {
         self.pinsStore = pinsStore
         self.outputDirectory = outputDirectory
-        self.storage = storage
         self.fileSystem = fileSystem
     }
 
-    func cacheFrameworks(_ targets: Set<CacheTarget>) async {
-        let chunked = targets.chunks(ofCount: storage?.parallelNumber ?? CacheSystem.defaultParalellNumber)
+    func cacheFrameworks(_ targets: Set<CacheTarget>, to storages: [any CacheStorage]) async {
+        for storage in storages {
+            await cacheFrameworks(targets, to: storage)
+        }
+    }
 
+    private func cacheFrameworks(_ targets: Set<CacheTarget>, to storage: some CacheStorage) async {
+        let chunked = targets.chunks(ofCount: storage.parallelNumber ?? CacheSystem.defaultParalellNumber)
+
+        let storageName = storage.displayName
         for chunk in chunked {
             await withTaskGroup(of: Void.self) { group in
                 for target in chunk {
@@ -159,10 +163,10 @@ struct CacheSystem: Sendable {
                         let frameworkPath = outputDirectory.appendingPathComponent(frameworkName)
                         do {
                             logger.info(
-                                "🚀 Cache \(frameworkName) to cache storage",
+                                "🚀 Cache \(frameworkName) to cache storage: \(storageName)",
                                 metadata: .color(.green)
                             )
-                            try await cacheFramework(target, at: frameworkPath)
+                            try await cacheFramework(target, at: frameworkPath, to: storage)
                         } catch {
                             logger.warning("⚠️ Can't create caches for \(frameworkPath.path)")
                         }
@@ -173,10 +177,10 @@ struct CacheSystem: Sendable {
         }
     }
 
-    private func cacheFramework(_ target: CacheTarget, at frameworkPath: URL) async throws {
+    private func cacheFramework(_ target: CacheTarget, at frameworkPath: URL, to storage: any CacheStorage) async throws {
         let cacheKey = try await calculateCacheKey(of: target)
 
-        try await storage?.cacheFramework(frameworkPath, for: cacheKey)
+        try await storage.cacheFramework(frameworkPath, for: cacheKey)
     }
 
     func generateVersionFile(for target: CacheTarget) async throws {
@@ -210,8 +214,8 @@ struct CacheSystem: Sendable {
         case failed(LocalizedError?)
         case noCache
     }
-    func restoreCacheIfPossible(target: CacheTarget) async -> RestoreResult {
-        guard let storage = storage else { return .noCache }
+
+    func restoreCacheIfPossible(target: CacheTarget, storage: some CacheStorage) async -> RestoreResult {
         do {
             let cacheKey = try await calculateCacheKey(of: target)
             if try await storage.existsValidCache(for: cacheKey) {
@@ -225,12 +229,6 @@ struct CacheSystem: Sendable {
         }
     }
 
-    private func fetchArtifacts(target: CacheTarget, to destination: URL) async throws {
-        guard let storage = storage else { return }
-        let cacheKey = try await calculateCacheKey(of: target)
-        try await storage.fetchArtifacts(for: cacheKey, to: destination)
-    }
-
     func calculateCacheKey(of target: CacheTarget) async throws -> SwiftPMCacheKey {
         let targetName = target.buildProduct.target.name
         let pin = try retrievePin(package: target.buildProduct.package)

diff --git a/...it/Producer/Cache/LocalCacheStorage.swift → ...roducer/Cache/LocalDiskCacheStorage.swift b/...it/Producer/Cache/LocalCacheStorage.swift → ...roducer/Cache/LocalDiskCacheStorage.swift
@@ -3,37 +3,33 @@ import ScipioStorage
 import PackageGraph
 import TSCBasic
 
-public struct LocalCacheStorage: CacheStorage {
+struct LocalDiskCacheStorage: CacheStorage {
     private let fileSystem: any FileSystem
 
-    public var parallelNumber: Int? { nil }
+    var parallelNumber: Int? { nil }
 
     enum Error: Swift.Error {
         case cacheDirectoryIsNotFound
     }
 
-    public enum CacheDirectory: Sendable {
-        case system
-        case custom(URL)
-    }
-
-    private let cacheDirectroy: CacheDirectory
+    private let baseURL: URL?
 
-    public init(cacheDirectory: CacheDirectory = .system, fileSystem: FileSystem = localFileSystem) {
-        self.cacheDirectroy = cacheDirectory
+    /// - Parameters:
+    ///   - baseURL: The base url for the local disk cache. When it is nil, the system cache directory (`~/Library/Caches`) will be used.
+    init(baseURL: URL?, fileSystem: FileSystem = localFileSystem) {
+        self.baseURL = baseURL
         self.fileSystem = fileSystem
     }
 
     private func buildBaseDirectoryPath() throws -> URL {
         let cacheDir: URL
-        switch cacheDirectroy {
-        case .system:
+        if let baseURL {
+            cacheDir = baseURL
+        } else {
             guard let systemCacheDir = fileSystem.cachesDirectory else {
                 throw Error.cacheDirectoryIsNotFound
             }
             cacheDir = systemCacheDir.asURL
-        case .custom(let customPath):
-            cacheDir = customPath
         }
         return cacheDir.appendingPathComponent("Scipio")
     }
@@ -51,7 +47,7 @@ public struct LocalCacheStorage: CacheStorage {
             .appendingPathComponent(xcFrameworkFileName(for: cacheKey))
     }
 
-    public func existsValidCache(for cacheKey: some CacheKey) async -> Bool {
+    func existsValidCache(for cacheKey: some CacheKey) async -> Bool {
         do {
             let xcFrameworkPath = try cacheFrameworkPath(for: cacheKey)
             return fileSystem.exists(xcFrameworkPath.absolutePath)
@@ -60,7 +56,7 @@ public struct LocalCacheStorage: CacheStorage {
         }
     }
 
-    public func cacheFramework(_ frameworkPath: URL, for cacheKey: some CacheKey) async {
+    func cacheFramework(_ frameworkPath: URL, for cacheKey: some CacheKey) async {
         do {
             let destination = try cacheFrameworkPath(for: cacheKey)
             let directoryPath = destination.deletingLastPathComponent()
@@ -72,7 +68,7 @@ public struct LocalCacheStorage: CacheStorage {
         }
     }
 
-    public func fetchArtifacts(for cacheKey: some CacheKey, to destinationDir: URL) async throws {
+    func fetchArtifacts(for cacheKey: some CacheKey, to destinationDir: URL) async throws {
         let source = try cacheFrameworkPath(for: cacheKey)
         let destination = destinationDir.appendingPathComponent(xcFrameworkFileName(for: cacheKey))
         try fileSystem.copy(from: source.absolutePath, to: destination.absolutePath)

diff --git a/Sources/ScipioKit/Producer/Cache/ProjectCacheStorage.swift b/Sources/ScipioKit/Producer/Cache/ProjectCacheStorage.swift
@@ -0,0 +1,10 @@
+import Foundation
+import ScipioStorage
+
+/// The pseudo cache storage for "project cache policy", which treats built frameworks under the project's output directory (e.g. `XCFrameworks`)
+/// as valid caches but does not saving / restoring anything.
+struct ProjectCacheStorage: CacheStorage {
+    func existsValidCache(for cacheKey: some ScipioStorage.CacheKey) async throws -> Bool { false }
+    func fetchArtifacts(for cacheKey: some ScipioStorage.CacheKey, to destinationDir: URL) async throws {}
+    func cacheFramework(_ frameworkPath: URL, for cacheKey: some ScipioStorage.CacheKey) async throws {}
+}
diff --git a/Sources/ScipioKit/Producer/CacheStorage+DisplayName.swift b/Sources/ScipioKit/Producer/CacheStorage+DisplayName.swift
@@ -0,0 +1,9 @@
+import ScipioStorage
+
+extension CacheStorage {
+    /// The display name of the cache storage used for logging purpose
+    var displayName: String {
+        // TODO: Define the property as CacheStorage's requirement in scipio-cache-storage
+        "\(type(of: self))"
+    }
+}