Skip to content

Implement connection pool. #451

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 22 commits into from
Jul 19, 2016
Merged

Implement connection pool. #451

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
6881560
Add connection pool for concurrent access
kdubb Nov 29, 2015
42ebd18
Add delegate to connection pool
kdubb Nov 29, 2015
68f46d5
Better names for protocols and classes
kdubb Apr 16, 2016
66b82b0
Use vfs for exclusivity
kdubb Apr 16, 2016
ccc20e5
Added alternate concurrency test
kdubb Apr 16, 2016
18f612f
Fix imports in ConnectionPool.swift
kdubb Apr 16, 2016
2a5908b
Rename `DBConnection` to `DirectConnection`
kdubb Apr 17, 2016
9548d17
Replace pool delegate with setup closures
kdubb Apr 18, 2016
ffb9274
Fix writable connection initialization
kdubb Apr 19, 2016
b7a0eb1
Cleanup connection pool tests
kdubb Apr 19, 2016
6df0272
Merge branch 'connection_pool' of https://github.com/reTXT/SQLite.swi…
nickmshelley Jun 16, 2016
960309e
Implement connection limit.
nickmshelley Jun 16, 2016
6052c31
Fix import problem for standalone.
nickmshelley Jun 22, 2016
e32db09
Shorten testConcurrentAccess2 time as it is making the travis tests t…
nickmshelley Jun 22, 2016
2fd69b8
Tests are still taking too long.
nickmshelley Jun 23, 2016
6d556b6
Merge branch 'master' into connection-pool
nickmshelley Jun 23, 2016
a688d18
Clean up some spacing.
nickmshelley Jul 18, 2016
a1121c4
Change ConnectionPool API and get rid of Connection protocol.
nickmshelley Jul 18, 2016
d2721ed
Merge branch 'master' into connection-pool
nickmshelley Jul 18, 2016
476cdc5
Make sure every thread has at least one read.
nickmshelley Jul 18, 2016
c7db919
Add connection pool documentation.
nickmshelley Jul 19, 2016
66b61dd
Use the right variable names.
nickmshelley Jul 19, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 27 additions & 28 deletions SQLite/Core/ConnectionPool.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ public final class ConnectionPool {
private var unavailableReadConnections = [DirectConnection]()
private let lockQueue : dispatch_queue_t
private var writeConnection : DirectConnection!
private let connectionSemaphore = dispatch_semaphore_create(5)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is a fine default, but should be configurable.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why do we want this configurable?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Because we do not know that 5 is the right connection pool size for every app.


public var foreignKeys : Bool {
get {
Expand Down Expand Up @@ -91,6 +92,7 @@ public final class ConnectionPool {
self.pool.unavailableReadConnections.removeAtIndex(index)
}
self.pool.availableReadConnections.append(self.connection)
dispatch_semaphore_signal(self.pool.connectionSemaphore)
}
}

Expand Down Expand Up @@ -151,44 +153,41 @@ public final class ConnectionPool {

var borrowed : BorrowedConnection!

repeat {
dispatch_semaphore_wait(connectionSemaphore, DISPATCH_TIME_FOREVER)
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The diff didn't come through very well on the changes below. All I did was remove the repeat loop and add this semaphore wait before the dispatch_sync call.

dispatch_sync(lockQueue) {

dispatch_sync(lockQueue) {

// Ensure database is open
self.writable
// Ensure database is open
self.writable

let connection : DirectConnection

if let availableConnection = self.availableReadConnections.popLast() {
connection = availableConnection
}
else {

let connection : DirectConnection
let flags = SQLITE_OPEN_READONLY | SQLITE_OPEN_WAL | SQLITE_OPEN_NOMUTEX

if let availableConnection = self.availableReadConnections.popLast() {
connection = availableConnection
}
else {

let flags = SQLITE_OPEN_READONLY | SQLITE_OPEN_WAL | SQLITE_OPEN_NOMUTEX

connection = try! DirectConnection(self.location, flags: flags, dispatcher: ImmediateDispatcher(), vfsName: vfsName)
connection.busyTimeout = 2
connection = try! DirectConnection(self.location, flags: flags, dispatcher: ImmediateDispatcher(), vfsName: vfsName)
connection.busyTimeout = 2

for (type, setupProcessor) in self.internalSetup {
if type == .WriteAheadLogging {
continue
}
try! setupProcessor(connection)
}

for setupProcessor in self.setup {
try! setupProcessor(connection)
for (type, setupProcessor) in self.internalSetup {
if type == .WriteAheadLogging {
continue
}

try! setupProcessor(connection)
}

self.unavailableReadConnections.append(connection)
for setupProcessor in self.setup {
try! setupProcessor(connection)
}

borrowed = BorrowedConnection(pool: self, connection: connection)
}

} while borrowed == nil
self.unavailableReadConnections.append(connection)

borrowed = BorrowedConnection(pool: self, connection: connection)
}

return borrowed
}
Expand Down
21 changes: 10 additions & 11 deletions SQLiteTests/ConnectionPoolTests.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,33 +22,32 @@ class ConnectionPoolTests : SQLiteTestCase {

func testConcurrentAccess2() {

let threadCount = 20
let conn = pool.writable
try! conn.execute("DROP TABLE IF EXISTS test; CREATE TABLE test(id INTEGER PRIMARY KEY, name TEXT);")
try! conn.execute("DELETE FROM test")
try! conn.execute("INSERT INTO test(id,name) VALUES(0, 'test0')")
try! conn.execute("INSERT INTO test(id,name) VALUES(1, 'test1')")
try! conn.execute("INSERT INTO test(id,name) VALUES(2, 'test2')")
try! conn.execute("INSERT INTO test(id,name) VALUES(3, 'test3')")
try! conn.execute("INSERT INTO test(id,name) VALUES(4, 'test4')")
for threadNumber in 0..<threadCount {
try! conn.execute("INSERT INTO test(id,name) VALUES(\(threadNumber), 'test\(threadNumber)')")
}

var quit = false
let queue = dispatch_queue_create("Readers", DISPATCH_QUEUE_CONCURRENT)
for x in 0..<5 {
for threadNumber in 0..<threadCount {
var reads = 0

let ex = expectationWithDescription("thread" + String(x))
let ex = expectationWithDescription("thread" + String(threadNumber))

dispatch_async(queue) {

print("started", x)
print("started", threadNumber)

let conn = self.pool.readable

let stmt = try! conn.prepare("SELECT name FROM test WHERE id = ?")
var curr = stmt.scalar(x) as! String
var curr = stmt.scalar(threadNumber) as! String
while !quit {

let now = stmt.scalar(x) as! String
let now = stmt.scalar(threadNumber) as! String
if now != curr {
//print(now)
curr = now
Expand Down Expand Up @@ -90,7 +89,7 @@ class ConnectionPoolTests : SQLiteTestCase {
let q = dispatch_queue_create("Readers/Writers", DISPATCH_QUEUE_CONCURRENT);
var finished = false

for _ in 0..<5 {
for _ in 0..<20 {

dispatch_async(q) {

Expand Down