From e126c3dd55f773c2b513ecd1180ccecc9beeb84b Mon Sep 17 00:00:00 2001 From: Matt Greenfield Date: Tue, 11 Sep 2018 18:47:03 +0700 Subject: [PATCH] merged TimelineStore and PersistentTimelineStore #41 --- LocoKit/Timelines/ItemsObserver.swift | 79 ++++ .../Timelines/PersistentTimelineStore.swift | 362 ------------------ .../TimelineObjects/PersistentObject.swift | 7 +- .../TimelineObjects/PersistentPath.swift | 2 +- .../TimelineObjects/PersistentSample.swift | 1 - .../TimelineObjects/PersistentVisit.swift | 2 +- .../TimelineObjects/TimelineSegment.swift | 7 +- LocoKit/Timelines/TimelineProcessor.swift | 20 +- ...s.swift => TimelineStore+Migrations.swift} | 4 +- LocoKit/Timelines/TimelineStore.swift | 280 +++++++++++++- 10 files changed, 362 insertions(+), 402 deletions(-) create mode 100644 LocoKit/Timelines/ItemsObserver.swift delete mode 100644 LocoKit/Timelines/PersistentTimelineStore.swift rename LocoKit/Timelines/{PersistentTimelineStore+Migrations.swift => TimelineStore+Migrations.swift} (98%) diff --git a/LocoKit/Timelines/ItemsObserver.swift b/LocoKit/Timelines/ItemsObserver.swift new file mode 100644 index 00000000..3fd3afdf --- /dev/null +++ b/LocoKit/Timelines/ItemsObserver.swift @@ -0,0 +1,79 @@ +// +// ItemsObserver.swift +// LocoKit +// +// Created by Matt Greenfield on 11/9/18. +// + +import os.log +import GRDB + +class ItemsObserver: TransactionObserver { + + var store: TimelineStore + var changedRowIds: Set = [] + + init(store: TimelineStore) { + self.store = store + } + + // observe updates to next/prev item links + func observes(eventsOfKind eventKind: DatabaseEventKind) -> Bool { + switch eventKind { + case .update(let tableName, let columnNames): + guard tableName == "TimelineItem" else { return false } + let itemEdges: Set = ["previousItemId", "nextItemId"] + return itemEdges.intersection(columnNames).count > 0 + default: return false + } + } + + func databaseDidChange(with event: DatabaseEvent) { + changedRowIds.insert(event.rowID) + } + + func databaseDidCommit(_ db: Database) { + let rowIds: Set = store.mutex.sync { + let rowIds = changedRowIds + changedRowIds = [] + return rowIds + } + + if rowIds.isEmpty { return } + + /** maintain the timeline items linked list locally, for changes made outside the managed environment **/ + + do { + let marks = repeatElement("?", count: rowIds.count).joined(separator: ",") + let query = "SELECT itemId, previousItemId, nextItemId FROM TimelineItem WHERE rowId IN (\(marks))" + let rows = try Row.fetchCursor(db, query, arguments: StatementArguments(rowIds)) + + while let row = try rows.next() { + let previousItemIdString = row["previousItemId"] as String? + let nextItemIdString = row["nextItemId"] as String? + + guard let uuidString = row["itemId"] as String?, let itemId = UUID(uuidString: uuidString) else { continue } + guard let item = store.object(for: itemId) as? TimelineItem else { continue } + + if let uuidString = previousItemIdString, item.previousItemId?.uuidString != uuidString { + item.previousItemId = UUID(uuidString: uuidString) + + } else if previousItemIdString == nil && item.previousItemId != nil { + item.previousItemId = nil + } + + if let uuidString = nextItemIdString, item.nextItemId?.uuidString != uuidString { + item.nextItemId = UUID(uuidString: uuidString) + + } else if nextItemIdString == nil && item.nextItemId != nil { + item.nextItemId = nil + } + } + + } catch { + os_log("SQL Exception: %@", error.localizedDescription) + } + } + + func databaseDidRollback(_ db: Database) {} +} diff --git a/LocoKit/Timelines/PersistentTimelineStore.swift b/LocoKit/Timelines/PersistentTimelineStore.swift deleted file mode 100644 index 98e64d9c..00000000 --- a/LocoKit/Timelines/PersistentTimelineStore.swift +++ /dev/null @@ -1,362 +0,0 @@ -// -// PersistentTimelineStore.swift -// LocoKit -// -// Created by Matt Greenfield on 9/01/18. -// Copyright © 2018 Big Paua. All rights reserved. -// - -import os.log -import GRDB -import LocoKitCore -import CoreLocation - -open class PersistentTimelineStore: TimelineStore { - - open var keepDeletedObjectsFor: TimeInterval = 60 * 60 - public var sqlDebugLogging = false - - public var itemsToSave: Set = [] - public var samplesToSave: Set = [] - - private lazy var itemsObserver = { - return ItemsObserver(store: self) - }() - - open lazy var dbUrl: URL = { - return try! FileManager.default - .url(for: .applicationSupportDirectory, in: .userDomainMask, appropriateFor: nil, create: true) - .appendingPathComponent("LocoKit.sqlite") - }() - - public lazy var poolConfig: Configuration = { - var config = Configuration() - if sqlDebugLogging { - config.trace = { - if self.sqlDebugLogging { os_log("SQL: %@", type: .default, $0) } - } - } - return config - }() - - public lazy var pool: DatabasePool = { - return try! DatabasePool(path: self.dbUrl.path, configuration: self.poolConfig) - }() - - public override init() { - super.init() - migrateDatabase() - pool.add(transactionObserver: itemsObserver) - pool.setupMemoryManagement(in: UIApplication.shared) - } - - // MARK: - Item / Sample creation - - open override func createVisit(from sample: LocomotionSample) -> PersistentVisit { - let visit = PersistentVisit(in: self) - visit.add(sample) - return visit - } - - open override func createPath(from sample: LocomotionSample) -> PersistentPath { - let path = PersistentPath(in: self) - path.add(sample) - return path - } - - open func createVisit(from samples: [LocomotionSample]) -> PersistentVisit { - let visit = PersistentVisit(in: self) - visit.add(samples) - return visit - } - - open func createPath(from samples: [LocomotionSample]) -> PersistentPath { - let path = PersistentPath(in: self) - path.add(samples) - return path - } - - open override func createSample(from sample: ActivityBrainSample) -> PersistentSample { - let sample = PersistentSample(from: sample, in: self) - saveOne(sample) // save the sample immediately, to avoid mystery data loss - return sample - } - - open override func createSample(date: Date, location: CLLocation? = nil, movingState: MovingState = .uncertain, - recordingState: RecordingState) -> PersistentSample { - let sample = PersistentSample(date: date, location: location, recordingState: recordingState, in: self) - saveOne(sample) // save the sample immediately, to avoid mystery data loss - return sample - } - - public func object(for row: Row) -> TimelineObject { - if row["itemId"] as String? != nil { return item(for: row) } - if row["sampleId"] as String? != nil { return sample(for: row) } - fatalError("Couldn't create an object for the row.") - } - - open func item(for row: Row) -> TimelineItem { - guard let itemId = row["itemId"] as String? else { fatalError("MISSING ITEMID") } - if let item = object(for: UUID(uuidString: itemId)!) as? TimelineItem { return item } - guard let isVisit = row["isVisit"] as Bool? else { fatalError("MISSING ISVISIT BOOL") } - return isVisit - ? PersistentVisit(from: row.asDict(in: self), in: self) - : PersistentPath(from: row.asDict(in: self), in: self) - } - - open func sample(for row: Row) -> PersistentSample { - guard let sampleId = row["sampleId"] as String? else { fatalError("MISSING SAMPLEID") } - if let sample = object(for: UUID(uuidString: sampleId)!) as? PersistentSample { return sample } - return PersistentSample(from: row.asDict(in: self), in: self) - } - - // MARK: - Item fetching - - open override var mostRecentItem: TimelineItem? { - return item(where: "deleted = 0 ORDER BY endDate DESC") - } - - open override func item(for itemId: UUID) -> TimelineItem? { - if let item = object(for: itemId) as? TimelineItem { return item } - return item(where: "itemId = ?", arguments: [itemId.uuidString]) - } - - public func item(where query: String, arguments: StatementArguments? = nil) -> TimelineItem? { - return item(for: "SELECT * FROM TimelineItem WHERE " + query + " LIMIT 1", arguments: arguments) - } - - public func items(where query: String, arguments: StatementArguments? = nil) -> [TimelineItem] { - return items(for: "SELECT * FROM TimelineItem WHERE " + query, arguments: arguments) - } - - public func item(for query: String, arguments: StatementArguments? = nil) -> TimelineItem? { - return try! pool.read { db in - guard let row = try Row.fetchOne(db, query, arguments: arguments) else { return nil } - return item(for: row) - } - } - - public func items(for query: String, arguments: StatementArguments? = nil) -> [TimelineItem] { - return try! pool.read { db in - var items: [TimelineItem] = [] - let itemRows = try Row.fetchCursor(db, query, arguments: arguments) - while let row = try itemRows.next() { items.append(item(for: row)) } - return items - } - } - - // MARK: Sample fetching - - open override func sample(for sampleId: UUID) -> PersistentSample? { - if let sample = object(for: sampleId) as? PersistentSample { return sample } - return sample(for: "SELECT * FROM LocomotionSample WHERE sampleId = ?", arguments: [sampleId.uuidString]) - } - - public func samples(where query: String, arguments: StatementArguments? = nil) -> [PersistentSample] { - return samples(for: "SELECT * FROM LocomotionSample WHERE " + query, arguments: arguments) - } - - public func sample(for query: String, arguments: StatementArguments? = nil) -> PersistentSample? { - return try! pool.read { db in - guard let row = try Row.fetchOne(db, query, arguments: arguments) else { return nil } - return sample(for: row) - } - } - - public func samples(for query: String, arguments: StatementArguments? = nil) -> [PersistentSample] { - let rows = try! pool.read { db in - return try Row.fetchAll(db, query, arguments: arguments) - } - return rows.map { sample(for: $0) } - } - - // MARK: - Counting - - public func countItems(where query: String = "1", arguments: StatementArguments? = nil) -> Int { - return try! pool.read { db in - return try Int.fetchOne(db, "SELECT COUNT(*) FROM TimelineItem WHERE " + query, arguments: arguments)! - } - } - - public func countSamples(where query: String = "1", arguments: StatementArguments? = nil) -> Int { - return try! pool.read { db in - return try Int.fetchOne(db, "SELECT COUNT(*) FROM LocomotionSample WHERE " + query, arguments: arguments)! - } - } - - // MARK: - Saving - - public func save(_ object: PersistentObject, immediate: Bool) { - mutex.sync { - if let item = object as? TimelineItem { - itemsToSave.insert(item) - } else if let sample = object as? PersistentSample { - samplesToSave.insert(sample) - } - } - if immediate { save() } - } - - open override func save() { - var savingItems: Set = [] - var savingSamples: Set = [] - - mutex.sync { - savingItems = itemsToSave.filter { ($0 as? PersistentItem)?.needsSave == true } - itemsToSave.removeAll(keepingCapacity: true) - - savingSamples = samplesToSave.filter { $0.needsSave } - samplesToSave.removeAll(keepingCapacity: true) - } - - if !savingItems.isEmpty { - try! pool.write { db in - let now = Date() - for case let item as PersistentObject in savingItems { - item.transactionDate = now - do { try item.save(in: db) } - catch PersistenceError.recordNotFound { os_log("PersistenceError.recordNotFound", type: .error) } - } - db.afterNextTransactionCommit { db in - for case let item as PersistentObject in savingItems { item.lastSaved = item.transactionDate } - } - } - } - if !savingSamples.isEmpty { - try! pool.write { db in - let now = Date() - for case let sample as PersistentObject in savingSamples { - sample.transactionDate = now - do { try sample.save(in: db) } - catch PersistenceError.recordNotFound { os_log("PersistenceError.recordNotFound", type: .error) } - } - db.afterNextTransactionCommit { db in - for case let sample as PersistentObject in savingSamples { sample.lastSaved = sample.transactionDate } - } - } - } - } - - public func saveOne(_ object: PersistentObject) { - do { - try pool.write { db in - object.transactionDate = Date() - do { try object.save(in: db) } - catch PersistenceError.recordNotFound { os_log("PersistenceError.recordNotFound", type: .error) } - db.afterNextTransactionCommit { db in - object.lastSaved = object.transactionDate - } - } - } catch { - os_log("%@", type: .error, error.localizedDescription) - } - } - - // MARK: - Database housekeeping - - open func hardDeleteSoftDeletedObjects() { - let deadline = Date(timeIntervalSinceNow: -keepDeletedObjectsFor) - do { - try pool.write { db in - try db.execute("DELETE FROM LocomotionSample WHERE deleted = 1 AND date < ?", arguments: [deadline]) - try db.execute("DELETE FROM TimelineItem WHERE deleted = 1 AND (endDate < ? OR endDate IS NULL)", arguments: [deadline]) - } - } catch { - os_log("%@", error.localizedDescription) - } - } - - // MARK: - Database creation and migrations - - public var migrator = DatabaseMigrator() - - open func migrateDatabase() { - registerMigrations() - try! migrator.migrate(pool) - } - - open var dateFields: [String] { return ["lastSaved", "lastModified", "startDate", "endDate", "date"] } - open var boolFields: [String] { return ["isVisit", "deleted", "locationIsBogus"] } -} - -class ItemsObserver: TransactionObserver { - - var store: PersistentTimelineStore - var changedRowIds: Set = [] - - init(store: PersistentTimelineStore) { - self.store = store - } - - // observe updates to next/prev item links - func observes(eventsOfKind eventKind: DatabaseEventKind) -> Bool { - switch eventKind { - case .update(let tableName, let columnNames): - guard tableName == "TimelineItem" else { return false } - let itemEdges: Set = ["previousItemId", "nextItemId"] - return itemEdges.intersection(columnNames).count > 0 - default: return false - } - } - - func databaseDidChange(with event: DatabaseEvent) { - changedRowIds.insert(event.rowID) - } - - func databaseDidCommit(_ db: Database) { - let rowIds: Set = store.mutex.sync { - let rowIds = changedRowIds - changedRowIds = [] - return rowIds - } - - if rowIds.isEmpty { return } - - /** maintain the timeline items linked list locally, for changes made outside the managed environment **/ - - do { - let marks = repeatElement("?", count: rowIds.count).joined(separator: ",") - let query = "SELECT itemId, previousItemId, nextItemId FROM TimelineItem WHERE rowId IN (\(marks))" - let rows = try Row.fetchCursor(db, query, arguments: StatementArguments(rowIds)) - - while let row = try rows.next() { - let previousItemIdString = row["previousItemId"] as String? - let nextItemIdString = row["nextItemId"] as String? - - guard let uuidString = row["itemId"] as String?, let itemId = UUID(uuidString: uuidString) else { continue } - guard let item = store.object(for: itemId) as? TimelineItem else { continue } - - if let uuidString = previousItemIdString, item.previousItemId?.uuidString != uuidString { - item.previousItemId = UUID(uuidString: uuidString) - - } else if previousItemIdString == nil && item.previousItemId != nil { - item.previousItemId = nil - } - - if let uuidString = nextItemIdString, item.nextItemId?.uuidString != uuidString { - item.nextItemId = UUID(uuidString: uuidString) - - } else if nextItemIdString == nil && item.nextItemId != nil { - item.nextItemId = nil - } - } - - } catch { - os_log("SQL Exception: %@", error.localizedDescription) - } - } - - func databaseDidRollback(_ db: Database) {} -} - -public extension Row { - func asDict(in store: PersistentTimelineStore) -> [String: Any?] { - let dateFields = store.dateFields - let boolFields = store.boolFields - return Dictionary(self.map { column, value in - if dateFields.contains(column) { return (column, Date.fromDatabaseValue(value)) } - if boolFields.contains(column) { return (column, Bool.fromDatabaseValue(value)) } - return (column, value.storage.value) - }, uniquingKeysWith: { left, _ in left }) - } -} diff --git a/LocoKit/Timelines/TimelineObjects/PersistentObject.swift b/LocoKit/Timelines/TimelineObjects/PersistentObject.swift index 6cc9e8cc..8f001bb2 100644 --- a/LocoKit/Timelines/TimelineObjects/PersistentObject.swift +++ b/LocoKit/Timelines/TimelineObjects/PersistentObject.swift @@ -13,7 +13,6 @@ public typealias PersistentItem = PersistentObject & TimelineItem public protocol PersistentObject: TimelineObject, PersistableRecord { - var persistentStore: PersistentTimelineStore? { get } var transactionDate: Date? { get set } var lastSaved: Date? { get set } var unsaved: Bool { get } @@ -28,7 +27,7 @@ public protocol PersistentObject: TimelineObject, PersistableRecord { public extension PersistentObject { public var unsaved: Bool { return lastSaved == nil } public var needsSave: Bool { return unsaved || hasChanges } - public func save(immediate: Bool = false) { persistentStore?.save(self, immediate: immediate) } + public func save(immediate: Bool = false) { store?.save(self, immediate: immediate) } public func save(in db: Database) throws { if unsaved { try insert(db) } else if hasChanges { try update(db) } hasChanges = false @@ -38,7 +37,3 @@ public extension PersistentObject { } } -public extension PersistentObject where Self: TimelineItem { - public var persistentStore: PersistentTimelineStore? { return store as? PersistentTimelineStore } -} - diff --git a/LocoKit/Timelines/TimelineObjects/PersistentPath.swift b/LocoKit/Timelines/TimelineObjects/PersistentPath.swift index c3a19750..ad671c8a 100644 --- a/LocoKit/Timelines/TimelineObjects/PersistentPath.swift +++ b/LocoKit/Timelines/TimelineObjects/PersistentPath.swift @@ -42,7 +42,7 @@ open class PersistentPath: Path, PersistentObject { if let existing = _samples { return existing } if lastSaved == nil { _samples = [] - } else if let store = persistentStore { + } else if let store = store { _samples = store.samples(where: "timelineItemId = ? AND deleted = 0 ORDER BY date", arguments: [itemId.uuidString]) } else { diff --git a/LocoKit/Timelines/TimelineObjects/PersistentSample.swift b/LocoKit/Timelines/TimelineObjects/PersistentSample.swift index eae83c36..404defc9 100644 --- a/LocoKit/Timelines/TimelineObjects/PersistentSample.swift +++ b/LocoKit/Timelines/TimelineObjects/PersistentSample.swift @@ -97,7 +97,6 @@ open class PersistentSample: LocomotionSample, PersistentObject { // MARK: PersistentObject - public var persistentStore: PersistentTimelineStore? { return store as? PersistentTimelineStore } public var transactionDate: Date? public var lastSaved: Date? public var hasChanges: Bool = false diff --git a/LocoKit/Timelines/TimelineObjects/PersistentVisit.swift b/LocoKit/Timelines/TimelineObjects/PersistentVisit.swift index 86d5b7ef..fdfc7738 100644 --- a/LocoKit/Timelines/TimelineObjects/PersistentVisit.swift +++ b/LocoKit/Timelines/TimelineObjects/PersistentVisit.swift @@ -42,7 +42,7 @@ open class PersistentVisit: Visit, PersistentObject { if let existing = _samples { return existing } if lastSaved == nil { _samples = [] - } else if let store = persistentStore { + } else if let store = store { _samples = store.samples(where: "timelineItemId = ? AND deleted = 0 ORDER BY date", arguments: [itemId.uuidString]) } else { diff --git a/LocoKit/Timelines/TimelineObjects/TimelineSegment.swift b/LocoKit/Timelines/TimelineObjects/TimelineSegment.swift index 90995a95..e46079f9 100644 --- a/LocoKit/Timelines/TimelineObjects/TimelineSegment.swift +++ b/LocoKit/Timelines/TimelineObjects/TimelineSegment.swift @@ -10,7 +10,7 @@ import GRDB public class TimelineSegment: TransactionObserver, Encodable { - public let store: PersistentTimelineStore + public let store: TimelineStore public var onUpdate: (() -> Void)? public var debugLogging = false @@ -32,13 +32,12 @@ public class TimelineSegment: TransactionObserver, Encodable { private var pendingChanges = false private var updatingEnabled = true - public convenience init(for dateRange: DateInterval, in store: PersistentTimelineStore, - onUpdate: (() -> Void)? = nil) { + public convenience init(for dateRange: DateInterval, in store: TimelineStore, onUpdate: (() -> Void)? = nil) { self.init(for: "endDate > ? AND startDate < ? AND deleted = 0 ORDER BY startDate", arguments: [dateRange.start, dateRange.end], in: store) } - public init(for query: String, arguments: StatementArguments? = nil, in store: PersistentTimelineStore, + public init(for query: String, arguments: StatementArguments? = nil, in store: TimelineStore, onUpdate: (() -> Void)? = nil) { self.store = store self.query = "SELECT * FROM TimelineItem WHERE " + query diff --git a/LocoKit/Timelines/TimelineProcessor.swift b/LocoKit/Timelines/TimelineProcessor.swift index fc195afe..f1fa3b5a 100644 --- a/LocoKit/Timelines/TimelineProcessor.swift +++ b/LocoKit/Timelines/TimelineProcessor.swift @@ -196,7 +196,7 @@ public class TimelineProcessor { // MARK: - ItemSegment brexiting - public static func extractItem(for segment: ItemSegment, in store: PersistentTimelineStore, completion: ((TimelineItem?) -> Void)? = nil) { + public static func extractItem(for segment: ItemSegment, in store: TimelineStore, completion: ((TimelineItem?) -> Void)? = nil) { store.process { guard let segmentRange = segment.dateRange else { completion?(nil) @@ -299,7 +299,7 @@ public class TimelineProcessor { } } - public static func extractPathEdgesFor(_ visit: Visit, in store: PersistentTimelineStore) { + public static func extractPathEdgesFor(_ visit: Visit, in store: TimelineStore) { if visit.deleted || visit.isMergeLocked { return } if let previousVisit = visit.previousItem as? Visit { @@ -311,7 +311,7 @@ public class TimelineProcessor { } } - public static func extractPathBetween(visit: Visit, and otherVisit: Visit, in store: PersistentTimelineStore) { + public static func extractPathBetween(visit: Visit, and otherVisit: Visit, in store: TimelineStore) { if visit.deleted || visit.isMergeLocked { return } if otherVisit.deleted || otherVisit.isMergeLocked { return } guard visit.nextItem == otherVisit || visit.previousItem == otherVisit else { return } @@ -342,7 +342,7 @@ public class TimelineProcessor { public static func healEdges(of brokenItem: TimelineItem) { if brokenItem.isMergeLocked { return } if !brokenItem.hasBrokenEdges { return } - guard let store = brokenItem.store as? PersistentTimelineStore else { return } + guard let store = brokenItem.store else { return } store.process { self.healPreviousEdge(of: brokenItem) } store.process { self.healNextEdge(of: brokenItem) } @@ -370,7 +370,7 @@ public class TimelineProcessor { } private static func healNextEdge(of brokenItem: TimelineItem) { - guard let store = brokenItem.store as? PersistentTimelineStore else { return } + guard let store = brokenItem.store else { return } if brokenItem.isMergeLocked { return } guard brokenItem.hasBrokenNextItemEdge else { return } guard let endDate = brokenItem.endDate else { return } @@ -422,7 +422,7 @@ public class TimelineProcessor { } private static func healPreviousEdge(of brokenItem: TimelineItem) { - guard let store = brokenItem.store as? PersistentTimelineStore else { return } + guard let store = brokenItem.store else { return } if brokenItem.isMergeLocked { return } guard brokenItem.hasBrokenPreviousItemEdge else { return } guard let startDate = brokenItem.startDate else { return } @@ -499,13 +499,13 @@ public class TimelineProcessor { // MARK: - Database sanitising - public static func sanitise(store: PersistentTimelineStore) { + public static func sanitise(store: TimelineStore) { orphanSamplesFromDeadParents(in: store) adoptOrphanedSamples(in: store) detachDeadmenEdges(in: store) } - private static func adoptOrphanedSamples(in store: PersistentTimelineStore) { + private static func adoptOrphanedSamples(in store: TimelineStore) { store.process { let orphans = store.samples(where: "timelineItemId IS NULL AND deleted = 0 ORDER BY date DESC") @@ -545,7 +545,7 @@ public class TimelineProcessor { } } - private static func orphanSamplesFromDeadParents(in store: PersistentTimelineStore) { + private static func orphanSamplesFromDeadParents(in store: TimelineStore) { store.process { let orphans = store.samples(for: """ SELECT LocomotionSample.* FROM LocomotionSample @@ -566,7 +566,7 @@ public class TimelineProcessor { } } - private static func detachDeadmenEdges(in store: PersistentTimelineStore) { + private static func detachDeadmenEdges(in store: TimelineStore) { store.process { let deadmen = store.items(where: "deleted = 1 AND (previousItemId IS NOT NULL OR nextItemId IS NOT NULL)") diff --git a/LocoKit/Timelines/PersistentTimelineStore+Migrations.swift b/LocoKit/Timelines/TimelineStore+Migrations.swift similarity index 98% rename from LocoKit/Timelines/PersistentTimelineStore+Migrations.swift rename to LocoKit/Timelines/TimelineStore+Migrations.swift index 71602c40..7717be90 100644 --- a/LocoKit/Timelines/PersistentTimelineStore+Migrations.swift +++ b/LocoKit/Timelines/TimelineStore+Migrations.swift @@ -1,5 +1,5 @@ // -// PersistentTimelineStore+Migrations.swift +// TimelineStore+Migrations.swift // LocoKit // // Created by Matt Greenfield on 4/6/18. @@ -7,7 +7,7 @@ import GRDB -internal extension PersistentTimelineStore { +internal extension TimelineStore { internal func registerMigrations() { diff --git a/LocoKit/Timelines/TimelineStore.swift b/LocoKit/Timelines/TimelineStore.swift index 9d4e321d..7e53bc41 100644 --- a/LocoKit/Timelines/TimelineStore.swift +++ b/LocoKit/Timelines/TimelineStore.swift @@ -9,16 +9,24 @@ import os.log import LocoKitCore import CoreLocation +import GRDB public extension NSNotification.Name { public static let processingStarted = Notification.Name("processingStarted") public static let processingStopped = Notification.Name("processingStopped") } -/// An in-memory timeline data store. For persistent timeline data storage, see `PersistentTimelineStore`. +/// An SQL database backed persistent timeline store. open class TimelineStore { - public init() {} + public init() { + migrateDatabase() + pool.add(transactionObserver: itemsObserver) + pool.setupMemoryManagement(in: UIApplication.shared) + } + + open var keepDeletedObjectsFor: TimeInterval = 60 * 60 + public var sqlDebugLogging = false public var recorder: TimelineRecorder? @@ -38,6 +46,33 @@ open class TimelineStore { public var itemsInStore: Int { return mutex.sync { itemMap.objectEnumerator()?.allObjects.count ?? 0 } } public var samplesInStore: Int { return mutex.sync { sampleMap.objectEnumerator()?.allObjects.count ?? 0 } } + public var itemsToSave: Set = [] + public var samplesToSave: Set = [] + + private lazy var itemsObserver = { + return ItemsObserver(store: self) + }() + + open lazy var dbUrl: URL = { + return try! FileManager.default + .url(for: .applicationSupportDirectory, in: .userDomainMask, appropriateFor: nil, create: true) + .appendingPathComponent("LocoKit.sqlite") + }() + + public lazy var poolConfig: Configuration = { + var config = Configuration() + if sqlDebugLogging { + config.trace = { + if self.sqlDebugLogging { os_log("SQL: %@", type: .default, $0) } + } + } + return config + }() + + public lazy var pool: DatabasePool = { + return try! DatabasePool(path: self.dbUrl.path, configuration: self.poolConfig) + }() + public func object(for objectId: UUID) -> TimelineObject? { return mutex.sync { if let item = itemMap.object(forKey: objectId as NSUUID) { return item } @@ -56,31 +91,66 @@ open class TimelineStore { } } - open var mostRecentItem: TimelineItem? { return nil } + open func sample(for sampleId: UUID) -> LocomotionSample? { return object(for: sampleId) as? LocomotionSample } - open func item(for itemId: UUID) -> TimelineItem? { return object(for: itemId) as? TimelineItem } + // MARK: - Item / Sample creation - open func sample(for sampleId: UUID) -> LocomotionSample? { return object(for: sampleId) as? LocomotionSample } - - open func createVisit(from sample: LocomotionSample) -> Visit { - let visit = Visit(in: self) + open func createVisit(from sample: LocomotionSample) -> PersistentVisit { + let visit = PersistentVisit(in: self) visit.add(sample) return visit } - open func createPath(from sample: LocomotionSample) -> Path { - let path = Path(in: self) + open func createPath(from sample: LocomotionSample) -> PersistentPath { + let path = PersistentPath(in: self) path.add(sample) return path } - open func createSample(from sample: ActivityBrainSample) -> LocomotionSample { - return LocomotionSample(from: sample, in: self) + open func createVisit(from samples: [LocomotionSample]) -> PersistentVisit { + let visit = PersistentVisit(in: self) + visit.add(samples) + return visit + } + + open func createPath(from samples: [LocomotionSample]) -> PersistentPath { + let path = PersistentPath(in: self) + path.add(samples) + return path + } + + open func createSample(from sample: ActivityBrainSample) -> PersistentSample { + let sample = PersistentSample(from: sample, in: self) + saveOne(sample) // save the sample immediately, to avoid mystery data loss + return sample } open func createSample(date: Date, location: CLLocation? = nil, movingState: MovingState = .uncertain, - recordingState: RecordingState) -> LocomotionSample { - return LocomotionSample(date: date, location: location, recordingState: recordingState, in: self) + recordingState: RecordingState) -> PersistentSample { + let sample = PersistentSample(date: date, location: location, recordingState: recordingState, in: self) + saveOne(sample) // save the sample immediately, to avoid mystery data loss + return sample + } + + public func object(for row: Row) -> TimelineObject { + if row["itemId"] as String? != nil { return item(for: row) } + if row["sampleId"] as String? != nil { return sample(for: row) } + fatalError("Couldn't create an object for the row.") + } + + open func item(for row: Row) -> TimelineItem { + guard let itemId = row["itemId"] as String? else { fatalError("MISSING ITEMID") } + if let item = object(for: UUID(uuidString: itemId)!) as? TimelineItem { return item } + guard let isVisit = row["isVisit"] as Bool? else { fatalError("MISSING ISVISIT BOOL") } + return isVisit + ? PersistentVisit(from: row.asDict(in: self), in: self) + : PersistentPath(from: row.asDict(in: self), in: self) + } + + open func sample(for row: Row) -> PersistentSample { + guard let sampleId = row["sampleId"] as String? else { fatalError("MISSING SAMPLEID") } + if let sample = object(for: UUID(uuidString: sampleId)!) as? PersistentSample { return sample } + return PersistentSample(from: row.asDict(in: self), in: self) } open func add(_ timelineItem: TimelineItem) { @@ -91,6 +161,150 @@ open class TimelineStore { mutex.sync { sampleMap.setObject(sample, forKey: sample.sampleId as NSUUID) } } + // MARK: - Item fetching + + open var mostRecentItem: TimelineItem? { + return item(where: "deleted = 0 ORDER BY endDate DESC") + } + + open func item(for itemId: UUID) -> TimelineItem? { + if let item = object(for: itemId) as? TimelineItem { return item } + return item(where: "itemId = ?", arguments: [itemId.uuidString]) + } + + public func item(where query: String, arguments: StatementArguments? = nil) -> TimelineItem? { + return item(for: "SELECT * FROM TimelineItem WHERE " + query + " LIMIT 1", arguments: arguments) + } + + public func items(where query: String, arguments: StatementArguments? = nil) -> [TimelineItem] { + return items(for: "SELECT * FROM TimelineItem WHERE " + query, arguments: arguments) + } + + public func item(for query: String, arguments: StatementArguments? = nil) -> TimelineItem? { + return try! pool.read { db in + guard let row = try Row.fetchOne(db, query, arguments: arguments) else { return nil } + return item(for: row) + } + } + + public func items(for query: String, arguments: StatementArguments? = nil) -> [TimelineItem] { + return try! pool.read { db in + var items: [TimelineItem] = [] + let itemRows = try Row.fetchCursor(db, query, arguments: arguments) + while let row = try itemRows.next() { items.append(item(for: row)) } + return items + } + } + + // MARK: Sample fetching + + open func sample(for sampleId: UUID) -> PersistentSample? { + if let sample = object(for: sampleId) as? PersistentSample { return sample } + return sample(for: "SELECT * FROM LocomotionSample WHERE sampleId = ?", arguments: [sampleId.uuidString]) + } + + public func samples(where query: String, arguments: StatementArguments? = nil) -> [PersistentSample] { + return samples(for: "SELECT * FROM LocomotionSample WHERE " + query, arguments: arguments) + } + + public func sample(for query: String, arguments: StatementArguments? = nil) -> PersistentSample? { + return try! pool.read { db in + guard let row = try Row.fetchOne(db, query, arguments: arguments) else { return nil } + return sample(for: row) + } + } + + public func samples(for query: String, arguments: StatementArguments? = nil) -> [PersistentSample] { + let rows = try! pool.read { db in + return try Row.fetchAll(db, query, arguments: arguments) + } + return rows.map { sample(for: $0) } + } + + // MARK: - Counting + + public func countItems(where query: String = "1", arguments: StatementArguments? = nil) -> Int { + return try! pool.read { db in + return try Int.fetchOne(db, "SELECT COUNT(*) FROM TimelineItem WHERE " + query, arguments: arguments)! + } + } + + public func countSamples(where query: String = "1", arguments: StatementArguments? = nil) -> Int { + return try! pool.read { db in + return try Int.fetchOne(db, "SELECT COUNT(*) FROM LocomotionSample WHERE " + query, arguments: arguments)! + } + } + + // MARK: - Saving + + public func save(_ object: PersistentObject, immediate: Bool) { + mutex.sync { + if let item = object as? TimelineItem { + itemsToSave.insert(item) + } else if let sample = object as? PersistentSample { + samplesToSave.insert(sample) + } + } + if immediate { save() } + } + + open func save() { + var savingItems: Set = [] + var savingSamples: Set = [] + + mutex.sync { + savingItems = itemsToSave.filter { ($0 as? PersistentItem)?.needsSave == true } + itemsToSave.removeAll(keepingCapacity: true) + + savingSamples = samplesToSave.filter { $0.needsSave } + samplesToSave.removeAll(keepingCapacity: true) + } + + if !savingItems.isEmpty { + try! pool.write { db in + let now = Date() + for case let item as PersistentObject in savingItems { + item.transactionDate = now + do { try item.save(in: db) } + catch PersistenceError.recordNotFound { os_log("PersistenceError.recordNotFound", type: .error) } + } + db.afterNextTransactionCommit { db in + for case let item as PersistentObject in savingItems { item.lastSaved = item.transactionDate } + } + } + } + if !savingSamples.isEmpty { + try! pool.write { db in + let now = Date() + for case let sample as PersistentObject in savingSamples { + sample.transactionDate = now + do { try sample.save(in: db) } + catch PersistenceError.recordNotFound { os_log("PersistenceError.recordNotFound", type: .error) } + } + db.afterNextTransactionCommit { db in + for case let sample as PersistentObject in savingSamples { sample.lastSaved = sample.transactionDate } + } + } + } + } + + public func saveOne(_ object: PersistentObject) { + do { + try pool.write { db in + object.transactionDate = Date() + do { try object.save(in: db) } + catch PersistenceError.recordNotFound { os_log("PersistenceError.recordNotFound", type: .error) } + db.afterNextTransactionCommit { db in + object.lastSaved = object.transactionDate + } + } + } catch { + os_log("%@", type: .error, error.localizedDescription) + } + } + + // MARK: - Processing + public func process(changes: @escaping () -> Void) { processingQueue.async { self.processing = true @@ -100,6 +314,42 @@ open class TimelineStore { } } - open func save() {} + // MARK: - Database housekeeping + + open func hardDeleteSoftDeletedObjects() { + let deadline = Date(timeIntervalSinceNow: -keepDeletedObjectsFor) + do { + try pool.write { db in + try db.execute("DELETE FROM LocomotionSample WHERE deleted = 1 AND date < ?", arguments: [deadline]) + try db.execute("DELETE FROM TimelineItem WHERE deleted = 1 AND (endDate < ? OR endDate IS NULL)", arguments: [deadline]) + } + } catch { + os_log("%@", error.localizedDescription) + } + } + + // MARK: - Database creation and migrations + + public var migrator = DatabaseMigrator() + open func migrateDatabase() { + registerMigrations() + try! migrator.migrate(pool) + } + + open var dateFields: [String] { return ["lastSaved", "lastModified", "startDate", "endDate", "date"] } + open var boolFields: [String] { return ["isVisit", "deleted", "locationIsBogus"] } + +} + +public extension Row { + func asDict(in store: TimelineStore) -> [String: Any?] { + let dateFields = store.dateFields + let boolFields = store.boolFields + return Dictionary(self.map { column, value in + if dateFields.contains(column) { return (column, Date.fromDatabaseValue(value)) } + if boolFields.contains(column) { return (column, Bool.fromDatabaseValue(value)) } + return (column, value.storage.value) + }, uniquingKeysWith: { left, _ in left }) + } }