1.5.0

* Add support for caching user email * Convert geodata to postgres * Add request logging * Cleanup channels in ws * Fixed blocking ws problem * Add requestTimingMap cleanup * Convert from gson to kotlin serialization * Add multi-server support for answers
readingbat · Oct 5, 2020 · 73f6dc5 · 73f6dc5
1 parent 4f409fb
commit 73f6dc5
Show file tree

Hide file tree

Showing 64 changed files with 2,116 additions and 1,368 deletions.
diff --git a/build.gradle b/build.gradle
@@ -3,6 +3,7 @@ plugins {
     id 'application'
     id 'maven'  // Required for jitpack.io to do a ./gradlew install
     id 'org.jetbrains.kotlin.jvm' version '1.4.10'
+    id 'org.jetbrains.kotlin.plugin.serialization' version '1.4.10'
     id "com.github.ben-manes.versions" version '0.33.0'
     id 'com.github.johnrengelman.shadow' version '6.0.0'
 }
@@ -30,10 +31,11 @@ targetCompatibility = 1.8
 
 description = 'ReadingBat Core'
 group 'com.github.readingbat'
-version '1.4.0'
+version '1.5.0'
 
 dependencies {
     implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8"
+    implementation "org.jetbrains.kotlinx:kotlinx-serialization-core:$serialization_version"
 
     implementation "com.github.pambrose.common-utils:core-utils:$utils_version"
 

diff --git a/gradle.properties b/gradle.properties
@@ -6,7 +6,7 @@ org.gradle.parallel=true
 org.gradle.caching=true
 org.gradle.jvmargs=-Xmx2048m -XX:MaxPermSize=512m -XX:+HeapDumpOnOutOfMemoryError -Dfile.encoding=UTF-8
 commons_version=1.9
-css_version=1.0.0-pre.122-kotlin-1.4.10
+css_version=1.0.0-pre.123-kotlin-1.4.10
 exposed_version=0.27.1
 flexmark_version=0.62.2
 github_api_version=1.116
@@ -24,5 +24,6 @@ prometheus_version=0.9.0
 proxy_version=d38ab96
 redis_version=3.3.0
 #selenium_version=4.0.0-alpha-6
-sendgrid_version=4.6.5
+serialization_version=1.0.0-RC
+sendgrid_version=4.6.6
 utils_version=10f4e7c
diff --git a/src/main/kotlin/com/github/readingbat/common/ClassCode.kt b/src/main/kotlin/com/github/readingbat/common/ClassCode.kt
@@ -19,10 +19,7 @@ package com.github.readingbat.common
 
 import com.github.pambrose.common.util.randomId
 import com.github.pambrose.common.util.toDoubleQuoted
-import com.github.readingbat.common.CommonUtils.keyOf
 import com.github.readingbat.common.FormFields.DISABLED_MODE
-import com.github.readingbat.common.KeyConstants.CLASS_CODE_KEY
-import com.github.readingbat.common.KeyConstants.CLASS_INFO_KEY
 import com.github.readingbat.common.User.Companion.toUser
 import com.github.readingbat.dsl.InvalidConfigurationException
 import com.github.readingbat.server.Classes
@@ -42,8 +39,6 @@ import kotlin.time.measureTimedValue
 internal data class ClassCode(val value: String) {
   val isNotEnabled by lazy { value == DISABLED_MODE || value.isBlank() }
   val isEnabled by lazy { !isNotEnabled }
-  val classCodeEnrollmentKey by lazy { keyOf(CLASS_CODE_KEY, value) }
-  val classInfoKey by lazy { keyOf(CLASS_INFO_KEY, value) }
 
   val displayedValue get() = if (value == DISABLED_MODE) "" else value
 
@@ -58,7 +53,7 @@ internal data class ClassCode(val value: String) {
             .firstOrNull() ?: throw InvalidConfigurationException("Missing class code $value")
         }
       }.let {
-        logger.info { "Looked up classId in ${it.duration}" }
+        logger.debug { "Looked up classId in ${it.duration}" }
         it.value
       }
 
@@ -70,7 +65,7 @@ internal data class ClassCode(val value: String) {
         .slice(Count(Classes.classCode))
         .select { Classes.classCode eq value }
         .map { it[0] as Long }
-        .first().also { logger.info { "ClassCode.isValid() returned $it for $value" } } > 0
+        .first().also { logger.debug { "ClassCode.isValid() returned $it for $value" } } > 0
     }
 
   fun fetchEnrollees(): List<User> =
@@ -85,10 +80,9 @@ internal data class ClassCode(val value: String) {
             .map { it[0] as Long }
 
         Users
-          .slice(Users.userId)
           .select { Users.id inList userIds }
-          .map { (it[0] as String).toUser(null) }
-          .also { logger.info { "fetchEnrollees() returning $it" } }
+          .map { toUser(it[Users.userId], it) }
+          .also { logger.debug { "fetchEnrollees() returning ${it.size} users" } }
       }
 
   fun deleteClassCode() = Classes.deleteWhere { Classes.classCode eq value }
@@ -113,7 +107,7 @@ internal data class ClassCode(val value: String) {
         .select { Classes.classCode eq value }
         .map { it[0] as String }
         .firstOrNull() ?: "Missing description")
-        .also { logger.info { "fetchClassDesc() returned ${it.toDoubleQuoted()} for $value" } }
+        .also { logger.debug { "fetchClassDesc() returned ${it.toDoubleQuoted()} for $value" } }
     }.let { if (quoted) it.toDoubleQuoted() else it }
 
   fun toDisplayString() = "${fetchClassDesc(true)} [$value]"
@@ -124,7 +118,7 @@ internal data class ClassCode(val value: String) {
         .slice(Users.userId)
         .select { Classes.classCode eq value }
         .map { it[0] as String }
-        .firstOrNull() ?: "").also { logger.info { "fetchClassTeacherId() returned $it" } }
+        .firstOrNull() ?: "").also { logger.debug { "fetchClassTeacherId() returned $it" } }
     }
 
   override fun toString() = value

diff --git a/src/main/kotlin/com/github/readingbat/common/Constants.kt b/src/main/kotlin/com/github/readingbat/common/Constants.kt
@@ -48,6 +48,10 @@ internal object Constants {
   const val PROCESS_USER_ANSWERS_JS_FUNC = "processUserAnswers"
   const val LIKE_DISLIKE_JS_FUNC = "likeDislike"
   const val NO_TRACK_HEADER = "NO_TRACK"
+
+  // Do not change this value without adjusting values in DBMS
+  const val UNKNOWN_USER_ID = "unknown-user"
+
   const val UNKNOWN = "Unknown"
   const val UNASSIGNED = "unassigned"
   const val REDIS_IS_DOWN = "Redis is down"
@@ -76,8 +80,8 @@ internal object PropertyNames {
 
 internal object Endpoints {
   const val ROOT = "/"
+  const val WS_ROOT = "/ws"
 
-  //const val STATIC_ROOT = "/$STATIC"
   const val STATIC_ROOT = "https://static.readingbat.com"
   const val CHALLENGE_ROOT = "/content"
   const val PLAYGROUND_ROOT = "/playground"
@@ -102,6 +106,7 @@ internal object Endpoints {
   const val FAV_ICON_ENDPOINT = "/favicon.ico"
   const val ROBOTS_ENDPOINT = "/robots.txt"
   const val CHALLENGE_ENDPOINT = "/challenge"
+  const val CLOCK_ENDPOINT = "/clock"
   const val CHALLENGE_GROUP_ENDPOINT = "/challenge-group"
   const val CHECK_ANSWERS_ENDPOINT = "/check-answers"
   const val LIKE_DISLIKE_ENDPOINT = "/like-dislike"
@@ -117,7 +122,7 @@ internal object Endpoints {
 
   const val DELETE_CONTENT_IN_REDIS_ENDPOINT = "/clear-caches"
 
-  const val PING = "/ping"
+  const val PING_ENDPOINT = "/ping"
   const val THREAD_DUMP = "/threaddump"
   const val LOGOUT_ENDPOINT = "/logout"
 

diff --git a/src/main/kotlin/com/github/readingbat/common/Cookies.kt b/src/main/kotlin/com/github/readingbat/common/Cookies.kt
@@ -19,16 +19,11 @@ package com.github.readingbat.common
 
 import com.github.readingbat.common.CommonUtils.keyOf
 import com.github.readingbat.common.CommonUtils.md5Of
-import com.github.readingbat.common.KeyConstants.ANSWER_HISTORY_KEY
 import com.github.readingbat.common.KeyConstants.CHALLENGE_ANSWERS_KEY
 import com.github.readingbat.common.KeyConstants.CORRECT_ANSWERS_KEY
-import com.github.readingbat.common.KeyConstants.LIKE_DISLIKE_KEY
 import com.github.readingbat.common.KeyConstants.NO_AUTH_KEY
-import com.github.readingbat.common.User.Companion.gson
-import com.github.readingbat.dsl.InvalidConfigurationException
 import com.github.readingbat.dsl.MissingBrowserSessionException
 import com.github.readingbat.posts.ChallengeHistory
-import com.github.readingbat.posts.ChallengeNames
 import com.github.readingbat.server.BrowserSessions
 import com.github.readingbat.server.ChallengeName
 import com.github.readingbat.server.GroupName
@@ -38,9 +33,13 @@ import com.github.readingbat.server.SessionAnswerHistory
 import io.ktor.application.*
 import io.ktor.auth.*
 import io.ktor.sessions.*
+import kotlinx.serialization.decodeFromString
+import kotlinx.serialization.json.Json
+import mu.KLogging
 import org.jetbrains.exposed.sql.and
 import org.jetbrains.exposed.sql.insertAndGetId
 import org.jetbrains.exposed.sql.select
+import org.jetbrains.exposed.sql.transactions.transaction
 import java.time.Instant
 
 internal data class UserPrincipal(val userId: String, val created: Long = Instant.now().toEpochMilli()) : Principal
@@ -49,39 +48,18 @@ internal data class BrowserSession(val id: String, val created: Long = Instant.n
 
   fun sessionDbmsId() =
     try {
-      id.sessionDbmsId
+      querySessionDbmsId(id)
     } catch (e: MissingBrowserSessionException) {
-      User.logger.info { "Creating BrowserSession for ${e.message}" }
-      createBrowserSession()
+      logger.debug { "Creating BrowserSession for ${e.message}" }
+      createBrowserSession(id)
     }
 
-  fun correctAnswersKey(names: ChallengeNames) =
-    correctAnswersKey(names.languageName, names.groupName, names.challengeName)
-
   fun correctAnswersKey(languageName: LanguageName, groupName: GroupName, challengeName: ChallengeName) =
     keyOf(CORRECT_ANSWERS_KEY, NO_AUTH_KEY, id, md5Of(languageName, groupName, challengeName))
 
-  fun likeDislikeKey(names: ChallengeNames) =
-    likeDislikeKey(names.languageName, names.groupName, names.challengeName)
-
-  fun likeDislikeKey(languageName: LanguageName, groupName: GroupName, challengeName: ChallengeName) =
-    keyOf(LIKE_DISLIKE_KEY, NO_AUTH_KEY, id, md5Of(languageName, groupName, challengeName))
-
-  fun challengeAnswerKey(names: ChallengeNames) =
-    challengeAnswerKey(names.languageName, names.groupName, names.challengeName)
-
   fun challengeAnswerKey(languageName: LanguageName, groupName: GroupName, challengeName: ChallengeName) =
     keyOf(CHALLENGE_ANSWERS_KEY, NO_AUTH_KEY, id, md5Of(languageName, groupName, challengeName))
 
-  fun answerHistoryKey(names: ChallengeNames, invocation: Invocation) =
-    answerHistoryKey(names.languageName, names.groupName, names.challengeName, invocation)
-
-  private fun answerHistoryKey(languageName: LanguageName,
-                               groupName: GroupName,
-                               challengeName: ChallengeName,
-                               invocation: Invocation) =
-    keyOf(ANSWER_HISTORY_KEY, NO_AUTH_KEY, id, md5Of(languageName, groupName, challengeName, invocation))
-
   fun answerHistory(md5: String, invocation: Invocation) =
     SessionAnswerHistory
       .slice(SessionAnswerHistory.invocation,
@@ -91,34 +69,32 @@ internal data class BrowserSession(val id: String, val created: Long = Instant.n
       .select { (SessionAnswerHistory.sessionRef eq sessionDbmsId()) and (SessionAnswerHistory.md5 eq md5) }
       .map {
         val json = it[SessionAnswerHistory.historyJson]
-        val history =
-          mutableListOf<String>().apply { addAll(gson.fromJson(json, List::class.java) as List<String>) }
-
+        val history = Json.decodeFromString<List<String>>(json).toMutableList()
         ChallengeHistory(Invocation(it[SessionAnswerHistory.invocation]),
                          it[SessionAnswerHistory.correct],
                          it[SessionAnswerHistory.incorrectAttempts].toInt(),
                          history)
       }
       .firstOrNull() ?: ChallengeHistory(invocation)
 
-  companion object {
-    fun BrowserSession?.createBrowserSession() =
+  companion object : KLogging() {
+    fun createBrowserSession(id: String) =
       BrowserSessions
         .insertAndGetId { row ->
-          row[session_id] =
-            this@createBrowserSession?.id ?: throw InvalidConfigurationException("Missing browser session")
+          row[session_id] = id
         }.value
+
+    fun querySessionDbmsId(id: String) =
+      transaction {
+        BrowserSessions
+          .slice(BrowserSessions.id)
+          .select { BrowserSessions.session_id eq id }
+          .map { it[BrowserSessions.id].value }
+          .firstOrNull() ?: throw MissingBrowserSessionException(id)
+      }
   }
 }
 
-internal val String.sessionDbmsId: Long
-  get() =
-    BrowserSessions
-      .slice(BrowserSessions.id)
-      .select { BrowserSessions.session_id eq this@sessionDbmsId }
-      .map { it[BrowserSessions.id].value }
-      .firstOrNull() ?: throw MissingBrowserSessionException(this@sessionDbmsId)
-
 internal val ApplicationCall.browserSession get() = sessions.get<BrowserSession>()
 
 internal val ApplicationCall.userPrincipal get() = sessions.get<UserPrincipal>()
diff --git a/src/main/kotlin/com/github/readingbat/common/KeyConstants.kt b/src/main/kotlin/com/github/readingbat/common/KeyConstants.kt
@@ -20,18 +20,8 @@ package com.github.readingbat.common
 internal object KeyConstants {
   const val AUTH_KEY = "auth"
   const val NO_AUTH_KEY = "noauth"
-  const val USER_EMAIL_KEY = "user-email"
-  const val USER_INFO_KEY = "user-info"
-  const val USER_INFO_BROWSER_KEY = "user-info-browser"
-  const val USER_CLASSES_KEY = "user-classes"
   const val CORRECT_ANSWERS_KEY = "correct-answers"
-  const val LIKE_DISLIKE_KEY = "like-dislike"
   const val CHALLENGE_ANSWERS_KEY = "challenge-answers"
-  const val ANSWER_HISTORY_KEY = "answer-history"
-  const val RESET_KEY = "password-reset"
-  const val USER_RESET_KEY = "user-password-reset"
-  const val CLASS_CODE_KEY = "class-code"
-  const val CLASS_INFO_KEY = "class-info"
   const val IPGEO_KEY = "ip-geo"
 
   const val SOURCE_CODE_KEY = "source-code"

diff --git a/src/main/kotlin/com/github/readingbat/common/Keys.kt b/src/main/kotlin/com/github/readingbat/common/Keys.kt
@@ -18,19 +18,14 @@
 package com.github.readingbat.common
 
 import com.github.readingbat.dsl.Challenge
-import com.github.readingbat.posts.ChallengeNames
 import com.github.readingbat.server.ChallengeName
 import com.github.readingbat.server.GroupName
-import com.github.readingbat.server.Invocation
 import com.github.readingbat.server.LanguageName
 
-
-internal fun correctAnswersKey(user: User?, browserSession: BrowserSession?, names: ChallengeNames) =
-  user?.correctAnswersKey(names) ?: browserSession?.correctAnswersKey(names) ?: ""
-
 internal fun correctAnswersKey(user: User?, browserSession: BrowserSession?, challenge: Challenge) =
   user?.correctAnswersKey(challenge.languageName, challenge.groupName, challenge.challengeName)
-    ?: browserSession?.correctAnswersKey(challenge.languageName, challenge.groupName, challenge.challengeName) ?: ""
+    ?: browserSession?.correctAnswersKey(challenge.languageName, challenge.groupName, challenge.challengeName)
+    ?: ""
 
 internal fun correctAnswersKey(user: User?,
                                browserSession: BrowserSession?,
@@ -41,9 +36,6 @@ internal fun correctAnswersKey(user: User?,
     ?: browserSession?.correctAnswersKey(languageName, groupName, challengeName)
     ?: ""
 
-internal fun challengeAnswersKey(user: User?, browserSession: BrowserSession?, names: ChallengeNames) =
-  user?.challengeAnswersKey(names) ?: browserSession?.challengeAnswerKey(names) ?: ""
-
 internal fun challengeAnswersKey(user: User?,
                                  browserSession: BrowserSession?,
                                  languageName: LanguageName,
@@ -58,11 +50,4 @@ internal fun challengeAnswersKey(user: User?, browserSession: BrowserSession?, c
                       browserSession,
                       challenge.languageType.languageName,
                       challenge.groupName,
-                      challenge.challengeName)
-
-internal fun answerHistoryKey(user: User?,
-                              browserSession: BrowserSession?,
-                              names: ChallengeNames,
-                              invocation: Invocation) =
-  user?.answerHistoryKey(names, invocation) ?: browserSession?.answerHistoryKey(names, invocation) ?: ""
-
+                      challenge.challengeName)