From 0c8861a677416d972240d6ac30c6635ab73d1333 Mon Sep 17 00:00:00 2001 From: QuadStingray Date: Fri, 26 Apr 2024 23:13:19 +0200 Subject: [PATCH] refactor: added serverAddressList as connection option --- .../mongodb/database/ConfigHelper.scala | 43 +++++++++ .../driver/mongodb/database/MongoConfig.scala | 91 ++++++------------- .../mongodb/pagination/MongoPagination.scala | 1 + .../mongodb/server/H2BackendConfig.scala | 3 + .../driver/mongodb/server/ServerBackend.scala | 6 ++ .../driver/mongodb/server/ServerConfig.scala | 18 ++-- .../mongodb/sync/MongoSyncException.scala | 3 + .../mongodb/sync/MongoSyncOperation.scala | 56 +++--------- .../driver/mongodb/sync/MongoSyncResult.scala | 14 +++ .../driver/mongodb/sync/SyncDirection.scala | 6 ++ .../driver/mongodb/sync/SyncStrategy.scala | 6 ++ 11 files changed, 128 insertions(+), 119 deletions(-) create mode 100644 src/main/scala/dev/mongocamp/driver/mongodb/database/ConfigHelper.scala create mode 100644 src/main/scala/dev/mongocamp/driver/mongodb/server/H2BackendConfig.scala create mode 100644 src/main/scala/dev/mongocamp/driver/mongodb/server/ServerBackend.scala create mode 100644 src/main/scala/dev/mongocamp/driver/mongodb/sync/MongoSyncException.scala create mode 100644 src/main/scala/dev/mongocamp/driver/mongodb/sync/MongoSyncResult.scala create mode 100644 src/main/scala/dev/mongocamp/driver/mongodb/sync/SyncDirection.scala create mode 100644 src/main/scala/dev/mongocamp/driver/mongodb/sync/SyncStrategy.scala diff --git a/src/main/scala/dev/mongocamp/driver/mongodb/database/ConfigHelper.scala b/src/main/scala/dev/mongocamp/driver/mongodb/database/ConfigHelper.scala new file mode 100644 index 00000000..3551705b --- /dev/null +++ b/src/main/scala/dev/mongocamp/driver/mongodb/database/ConfigHelper.scala @@ -0,0 +1,43 @@ +package dev.mongocamp.driver.mongodb.database + +import com.typesafe.config.{ Config, ConfigFactory } + +trait ConfigHelper { + val conf: Config = ConfigFactory.load() + + def stringConfig(configPath: String, key: String, default: String = ""): Option[String] = { + if (conf.hasPath("%s.%s".format(configPath, key))) { + val str = conf.getString("%s.%s".format(configPath, key)) + if (str.nonEmpty) { + Some(str) + } + else { + None + } + } + else if (default.nonEmpty) { + Some(default) + } + else { + None + } + } + + def intConfig(configPath: String, key: String, default: Int = 0): Int = { + if (conf.hasPath("%s.%s".format(configPath, key))) { + conf.getInt("%s.%s".format(configPath, key)) + } + else { + default + } + } + + def booleanConfig(configPath: String, key: String, default: Boolean = false): Boolean = { + if (conf.hasPath("%s.%s".format(configPath, key))) { + conf.getBoolean("%s.%s".format(configPath, key)) + } + else { + default + } + } +} diff --git a/src/main/scala/dev/mongocamp/driver/mongodb/database/MongoConfig.scala b/src/main/scala/dev/mongocamp/driver/mongodb/database/MongoConfig.scala index c0c90657..7f2d3368 100644 --- a/src/main/scala/dev/mongocamp/driver/mongodb/database/MongoConfig.scala +++ b/src/main/scala/dev/mongocamp/driver/mongodb/database/MongoConfig.scala @@ -4,8 +4,8 @@ import java.util.concurrent.TimeUnit import com.mongodb.MongoCompressor import com.mongodb.MongoCredential.createCredential import com.mongodb.event.{CommandListener, ConnectionPoolListener} -import dev.mongocamp.driver.mongodb.database.MongoConfig._ import com.typesafe.config.{Config, ConfigFactory} +import dev.mongocamp.driver.mongodb.database.MongoConfig.{CompressionSnappy, CompressionZlib, CompressionZstd, DefaultApplicationName, DefaultAuthenticationDatabaseName, DefaultHost, DefaultPort} import org.mongodb.scala.connection._ import org.mongodb.scala.{MongoClientSettings, MongoCredential, ServerAddress} @@ -21,10 +21,11 @@ case class MongoConfig( password: Option[String] = None, authDatabase: String = DefaultAuthenticationDatabaseName, poolOptions: MongoPoolOptions = MongoPoolOptions(), - compressors: List[String] = List(), - connectionPoolListener: List[ConnectionPoolListener] = List(), - commandListener: List[CommandListener] = List(), - customClientSettings: Option[MongoClientSettings] = None + compressors: List[String] = List.empty, + connectionPoolListener: List[ConnectionPoolListener] = List.empty, + commandListener: List[CommandListener] = List.empty, + customClientSettings: Option[MongoClientSettings] = None, + serverAddressList: List[ServerAddress] = List.empty ) { val clientSettings: MongoClientSettings = { @@ -32,7 +33,8 @@ case class MongoConfig( customClientSettings.get } else { - val clusterSettings: ClusterSettings = ClusterSettings.builder().hosts(List(new ServerAddress(host, port)).asJava).build() + val internalServerAddressList = if (serverAddressList.nonEmpty) serverAddressList else List(new ServerAddress(host, port)) + val clusterSettings: ClusterSettings = ClusterSettings.builder().hosts(internalServerAddressList.asJava).build() val connectionPoolSettingsBuilder = ConnectionPoolSettings .builder() @@ -47,13 +49,13 @@ case class MongoConfig( val compressorList = new ArrayBuffer[MongoCompressor]() compressors.foreach(compression => { - if (ComressionSnappy.equalsIgnoreCase(compression)) { + if (CompressionSnappy.equalsIgnoreCase(compression)) { compressorList.+=(MongoCompressor.createSnappyCompressor()) } - else if (ComressionZlib.equalsIgnoreCase(compression)) { + else if (CompressionZlib.equalsIgnoreCase(compression)) { compressorList.+=(MongoCompressor.createZlibCompressor()) } - else if (ComressionZstd.equalsIgnoreCase(compression)) { + else if (CompressionZstd.equalsIgnoreCase(compression)) { compressorList.+=(MongoCompressor.createZstdCompressor()) } }) @@ -78,61 +80,21 @@ case class MongoConfig( } } -trait ConfigHelper { - val conf: Config = ConfigFactory.load() - - def stringConfig(configPath: String, key: String, default: String = ""): Option[String] = { - if (conf.hasPath("%s.%s".format(configPath, key))) { - val str = conf.getString("%s.%s".format(configPath, key)) - if (str.nonEmpty) { - Some(str) - } - else { - None - } - } - else if (default.nonEmpty) { - Some(default) - } - else { - None - } - } - - def intConfig(configPath: String, key: String, default: Int = 0): Int = { - if (conf.hasPath("%s.%s".format(configPath, key))) { - conf.getInt("%s.%s".format(configPath, key)) - } - else { - default - } - } - - def booleanConfig(configPath: String, key: String, default: Boolean = false): Boolean = { - if (conf.hasPath("%s.%s".format(configPath, key))) { - conf.getBoolean("%s.%s".format(configPath, key)) - } - else { - default - } - } -} - object MongoConfig extends ConfigHelper { - val DefaultHost = "127.0.0.1" - val DefaultPort = 27017 + val DefaultHost = "127.0.0.1" + val DefaultPort = 27017 val DefaultAuthenticationDatabaseName = "admin" - val DefaultApplicationName = "mongocampdb-app" + val DefaultApplicationName = "mongocampdb-app" - val DefaultPoolMaxConnectionIdleTime = 60 - val DefaultPoolMaxSize = 50 - val DefaultPoolMinSize = 0 - val DefaultPoolMaxWaitQueueSize = 500 + val DefaultPoolMaxConnectionIdleTime = 60 + val DefaultPoolMaxSize = 50 + val DefaultPoolMinSize = 0 + val DefaultPoolMaxWaitQueueSize = 500 val DefaultPoolMaintenanceInitialDelay = 0 - val ComressionSnappy = "snappy" - val ComressionZlib = "zlib" - val ComressionZstd = "zstd" + val CompressionSnappy = "snappy" + val CompressionZlib = "zlib" + val CompressionZstd = "zstd" val DefaultConfigPathPrefix = "mongodb" @@ -158,11 +120,11 @@ object MongoConfig extends ConfigHelper { } } - val host = stringConfig(configPath, "host", DefaultHost).get - val database = stringConfig(configPath, "database").get - val userName = stringConfig(configPath, "userName") - val password = stringConfig(configPath, "password") - val authDatabase = stringConfig(configPath, "authDatabase", DefaultAuthenticationDatabaseName).get + val host = stringConfig(configPath, "host", DefaultHost).get + val database = stringConfig(configPath, "database").get + val userName = stringConfig(configPath, "userName") + val password = stringConfig(configPath, "password") + val authDatabase = stringConfig(configPath, "authDatabase", DefaultAuthenticationDatabaseName).get val applicationName = stringConfig(configPath, "applicationName", DefaultApplicationName).get val poolOptions = MongoPoolOptions( @@ -175,4 +137,5 @@ object MongoConfig extends ConfigHelper { MongoConfig(database, host, port, applicationName, userName, password, authDatabase, poolOptions, compressors) } + } diff --git a/src/main/scala/dev/mongocamp/driver/mongodb/pagination/MongoPagination.scala b/src/main/scala/dev/mongocamp/driver/mongodb/pagination/MongoPagination.scala index dbeee833..569a3c8b 100644 --- a/src/main/scala/dev/mongocamp/driver/mongodb/pagination/MongoPagination.scala +++ b/src/main/scala/dev/mongocamp/driver/mongodb/pagination/MongoPagination.scala @@ -1,4 +1,5 @@ package dev.mongocamp.driver.mongodb.pagination + import dev.mongocamp.driver.mongodb.database.ConfigHelper trait MongoPagination[A <: Any] extends ConfigHelper { diff --git a/src/main/scala/dev/mongocamp/driver/mongodb/server/H2BackendConfig.scala b/src/main/scala/dev/mongocamp/driver/mongodb/server/H2BackendConfig.scala new file mode 100644 index 00000000..a9c5a448 --- /dev/null +++ b/src/main/scala/dev/mongocamp/driver/mongodb/server/H2BackendConfig.scala @@ -0,0 +1,3 @@ +package dev.mongocamp.driver.mongodb.server + +case class H2BackendConfig(inMemory: Boolean = true, path: Option[String] = None) diff --git a/src/main/scala/dev/mongocamp/driver/mongodb/server/ServerBackend.scala b/src/main/scala/dev/mongocamp/driver/mongodb/server/ServerBackend.scala new file mode 100644 index 00000000..38652380 --- /dev/null +++ b/src/main/scala/dev/mongocamp/driver/mongodb/server/ServerBackend.scala @@ -0,0 +1,6 @@ +package dev.mongocamp.driver.mongodb.server + +object ServerBackend extends Enumeration { + type ServerBackend = Value + val Memory, H2 = Value +} diff --git a/src/main/scala/dev/mongocamp/driver/mongodb/server/ServerConfig.scala b/src/main/scala/dev/mongocamp/driver/mongodb/server/ServerConfig.scala index 9181afd9..387a7e3c 100644 --- a/src/main/scala/dev/mongocamp/driver/mongodb/server/ServerConfig.scala +++ b/src/main/scala/dev/mongocamp/driver/mongodb/server/ServerConfig.scala @@ -11,13 +11,6 @@ case class ServerConfig( h2BackendConfig: Option[H2BackendConfig] = None ) -case class H2BackendConfig(inMemory: Boolean = true, path: Option[String] = None) - -object ServerBackend extends Enumeration { - type ServerBackend = Value - val Memory, H2 = Value -} - object ServerConfig extends ConfigHelper { val DefaultServerConfigPathPrefix = "local.mongodb.server" @@ -26,10 +19,12 @@ object ServerConfig extends ConfigHelper { val DefaultPort = 28018 def serverBackendFromString(backendName: String): ServerBackend.Value = - if (ServerBackend.H2.toString.toLowerCase.equals(backendName.toLowerCase)) + if (ServerBackend.H2.toString.toLowerCase.equals(backendName.toLowerCase)) { ServerBackend.H2 - else + } + else { ServerBackend.Memory + } def fromPath(configPath: String = DefaultServerConfigPathPrefix): ServerConfig = { @@ -47,11 +42,12 @@ object ServerConfig extends ConfigHelper { val path = stringConfig(configPath, "h2.path") Some(H2BackendConfig(inMemory, path)) } - else + else { None + } } ServerConfig(name, host, port, serverBackend, h2BackendConfig) - } + } diff --git a/src/main/scala/dev/mongocamp/driver/mongodb/sync/MongoSyncException.scala b/src/main/scala/dev/mongocamp/driver/mongodb/sync/MongoSyncException.scala new file mode 100644 index 00000000..937bc7ab --- /dev/null +++ b/src/main/scala/dev/mongocamp/driver/mongodb/sync/MongoSyncException.scala @@ -0,0 +1,3 @@ +package dev.mongocamp.driver.mongodb.sync + +case class MongoSyncException(message: String) extends Exception(message) diff --git a/src/main/scala/dev/mongocamp/driver/mongodb/sync/MongoSyncOperation.scala b/src/main/scala/dev/mongocamp/driver/mongodb/sync/MongoSyncOperation.scala index 0f5e34b8..0873cdaf 100644 --- a/src/main/scala/dev/mongocamp/driver/mongodb/sync/MongoSyncOperation.scala +++ b/src/main/scala/dev/mongocamp/driver/mongodb/sync/MongoSyncOperation.scala @@ -2,7 +2,7 @@ package dev.mongocamp.driver.mongodb.sync import com.typesafe.scalalogging.LazyLogging import dev.mongocamp.driver.mongodb._ -import dev.mongocamp.driver.mongodb.database.{ ConfigHelper, DatabaseProvider } +import dev.mongocamp.driver.mongodb.database.{ConfigHelper, DatabaseProvider} import dev.mongocamp.driver.mongodb.sync.SyncDirection.SyncDirection import dev.mongocamp.driver.mongodb.sync.SyncStrategy.SyncStrategy import org.mongodb.scala.Document @@ -12,18 +12,6 @@ import org.mongodb.scala.model.Updates._ import java.util.Date -object SyncStrategy extends Enumeration { - type SyncStrategy = Value - val SyncAll = Value -} - -object SyncDirection extends Enumeration { - type SyncDirection = Value - val SourceToTarget, TargetToSource, TwoWay = Value -} - -case class MongoSyncException(message: String) extends Exception(message) - case class MongoSyncOperation( collectionName: String, syncDirection: SyncDirection = SyncDirection.SourceToTarget, @@ -35,10 +23,8 @@ case class MongoSyncOperation( def excecute(source: DatabaseProvider, target: DatabaseProvider): List[MongoSyncResult] = try { - val sourceInfos: Seq[Document] = - source.dao(collectionName).find().projection(includes).results(MongoSyncOperation.MaxWait) - val targetInfos: Seq[Document] = - target.dao(collectionName).find().projection(includes).results(MongoSyncOperation.MaxWait) + val sourceInfos: Seq[Document] = source.dao(collectionName).find().projection(includes).results(MongoSyncOperation.MaxWait) + val targetInfos: Seq[Document] = target.dao(collectionName).find().projection(includes).results(MongoSyncOperation.MaxWait) if (SyncDirection.SourceToTarget == syncDirection) { val diff = sourceInfos.diff(targetInfos) @@ -48,13 +34,15 @@ case class MongoSyncOperation( val diff = targetInfos.diff(sourceInfos) List(syncInternal(target, source, sourceInfos.size, diff)) } - else if (SyncDirection.TwoWay == syncDirection) + else if (SyncDirection.TwoWay == syncDirection) { List( syncInternal(source, target, targetInfos.size, sourceInfos.diff(targetInfos)), syncInternal(target, source, sourceInfos.size, targetInfos.diff(sourceInfos)) ) - else + } + else { List(MongoSyncResult(collectionName)) + } } catch { case e: Exception => @@ -87,7 +75,7 @@ case class MongoSyncOperation( MongoSyncResult( collectionName, syncDate, - true, + acknowleged = true, filteredDocumentsToSync.size, countBefore, countAfter, @@ -100,29 +88,9 @@ object MongoSyncOperation extends ConfigHelper { val MaxWaitDefault = 600 val MaxWait: Int = intConfig(configPath = "dev.mongocamp.mongodb.sync", key = "maxWait", default = MaxWaitDefault) - val SyncColumnLastSync: String = - stringConfig(configPath = "dev.mongocamp.mongodb.sync", key = "syncColumnLastSync", default = "_lastSync").get - val SyncColumnLastUpdate: String = - stringConfig(configPath = "dev.mongocamp.mongodb.sync", key = "syncColumnLastUpdate", default = "_lastUpdate").get + val SyncColumnLastSync: String = stringConfig(configPath = "dev.mongocamp.mongodb.sync", key = "syncColumnLastSync", default = "_lastSync").get + val SyncColumnLastUpdate: String = stringConfig(configPath = "dev.mongocamp.mongodb.sync", key = "syncColumnLastUpdate", default = "_lastUpdate").get - val WriteSyncLogOnMaster = booleanConfig(configPath = "dev.mongocamp.mongodb.sync", key = "writeSyncLogOnMaster") - val SyncLogTableName: String = - stringConfig( - configPath = "dev.mongocamp.mongodb.sync", - key = "syncLogTableName", - default = "mongodb-sync-log" - ).get + val WriteSyncLogOnMaster = booleanConfig(configPath = "dev.mongocamp.mongodb.sync", key = "writeSyncLogOnMaster") + val SyncLogTableName: String = stringConfig(configPath = "dev.mongocamp.mongodb.sync", key = "syncLogTableName", default = "mongodb-sync-log").get } - -//case class MongoSyncInfo(id: Any = new ObjectId(), syncDate: Date = new Date(), updateDate: Date = new Date()) - -case class MongoSyncResult( - collectionName: String, - syncDate: Date = new Date(), - acknowleged: Boolean = false, - synced: Int = -1, - countBefore: Int = -1, - countAfter: Int = -1, - syncTime: Long = -1, - exception: Option[Exception] = None -) diff --git a/src/main/scala/dev/mongocamp/driver/mongodb/sync/MongoSyncResult.scala b/src/main/scala/dev/mongocamp/driver/mongodb/sync/MongoSyncResult.scala new file mode 100644 index 00000000..cb6ec253 --- /dev/null +++ b/src/main/scala/dev/mongocamp/driver/mongodb/sync/MongoSyncResult.scala @@ -0,0 +1,14 @@ +package dev.mongocamp.driver.mongodb.sync + +import java.util.Date + +case class MongoSyncResult( + collectionName: String, + syncDate: Date = new Date(), + acknowleged: Boolean = false, + synced: Int = -1, + countBefore: Int = -1, + countAfter: Int = -1, + syncTime: Long = -1, + exception: Option[Exception] = None +) diff --git a/src/main/scala/dev/mongocamp/driver/mongodb/sync/SyncDirection.scala b/src/main/scala/dev/mongocamp/driver/mongodb/sync/SyncDirection.scala new file mode 100644 index 00000000..391891b5 --- /dev/null +++ b/src/main/scala/dev/mongocamp/driver/mongodb/sync/SyncDirection.scala @@ -0,0 +1,6 @@ +package dev.mongocamp.driver.mongodb.sync + +object SyncDirection extends Enumeration { + type SyncDirection = Value + val SourceToTarget, TargetToSource, TwoWay = Value +} diff --git a/src/main/scala/dev/mongocamp/driver/mongodb/sync/SyncStrategy.scala b/src/main/scala/dev/mongocamp/driver/mongodb/sync/SyncStrategy.scala new file mode 100644 index 00000000..b44f5aba --- /dev/null +++ b/src/main/scala/dev/mongocamp/driver/mongodb/sync/SyncStrategy.scala @@ -0,0 +1,6 @@ +package dev.mongocamp.driver.mongodb.sync + +object SyncStrategy extends Enumeration { + type SyncStrategy = Value + val SyncAll = Value +}