Merge pull request #3753 from ProjectSidewalk/develop

v8.0.2

Makefile

@@ -40,10 +40,22 @@ ssh:
 	@docker exec -it projectsidewalk-$${target} /bin/bash
 
 import-users:
-	@docker exec -it projectsidewalk-db sh -c "/opt/import-users.sh"
+	@docker exec -it projectsidewalk-db sh -c "/opt/scripts/import-users.sh"
 
 import-dump:
-	@docker exec -it projectsidewalk-db sh -c "/opt/import-dump.sh $(db)"
+	@docker exec -it projectsidewalk-db sh -c "/opt/scripts/import-dump.sh $(db)"
+
+create-new-schema:
+	@docker exec -it projectsidewalk-db sh -c "/opt/scripts/create-new-schema.sh $(name)"
+
+fill-new-schema:
+	@docker exec -it projectsidewalk-db sh -c "/opt/scripts/fill-new-schema.sh"
+
+hide-streets-without-imagery:
+	@docker exec -it projectsidewalk-db sh -c "/opt/scripts/hide-streets-without-imagery.sh"
+
+reveal-or-hide-neighborhoods:
+	@docker exec -it projectsidewalk-db sh -c "/opt/scripts/reveal-or-hide-neighborhoods.sh"
 
 lint-htmlhint:
 	@echo "Running HTMLHint...";

README.md

@@ -82,7 +82,7 @@ On Windows, we recommend [Windows Powershell](https://docs.microsoft.com/en-us/p
 1. Modify the `MAPBOX_API_KEY`, `GOOGLE_MAPS_API_KEY`, and `GOOGLE_MAPS_SECRET` lines in the `docker-compose.yml` using the keys and secret you've acquired.
 1. Modify the `SIDEWALK_CITY_ID` line in the `docker-compose.yml` to use the ID of the appropriate city, listed [here](https://github.com/ProjectSidewalk/SidewalkWebpage/wiki/Docker-Troubleshooting#first-heres-a-table-that-youll-reference-when-setting-up-your-dev-env) (it's the city that matches your database dump, so check the name of the db dump file).
 1. Modify the `DATABASE_USER` line in the `docker-compose.yml`, replacing "sidewalk" with the username from the table [linked above](https://github.com/ProjectSidewalk/SidewalkWebpage/wiki/Docker-Troubleshooting#first-heres-a-table-that-youll-reference-when-setting-up-your-dev-env).
-1. Rename the database dump file that you got from Mikey to "\<database_user\>-dump" (using the name from the prev step) and put it in the `db/` directory (other files in this dir include `init.sh` and `import-dump.sh`).
+1. Rename the database dump file that you got from Mikey to "\<database_user\>-dump" (using the name from the prev step) and put it in the `db/` directory.
 1. Rename the users dump file that you got from Mikey to "sidewalk_users-dump" and put it in the `db/` directory as well.
 1. From the root SidewalkWebpage dir, run `make dev`. This will take time (20-30 mins or more depending on your Internet connection) as the command downloads the docker images, spins up the containers, and opens a Docker shell into the webpage container in that same terminal. The containers (running Ubuntu Stretch) will have all the necessary packages and tools so no installation is necessary. This command also initializes the database, though we still need to import the data. Successful output of this command will look like:
 

app/models/amt/AMTAssignmentTable.scala

@@ -38,21 +38,21 @@ object AMTAssignmentTable {
   val TURKER_PAY_PER_LABEL_VALIDATION = 0.012D
   val VOLUNTEER_PAY: Double = 0.0D
 
-  def save(asg: AMTAssignment): Int = db.withTransaction { implicit session =>
+  def save(asg: AMTAssignment): Int = db.withSession { implicit session =>
     val asgId: Int =
       (amtAssignments returning amtAssignments.map(_.amtAssignmentId)) += asg
     asgId
   }
 
-  def getConfirmationCode(workerId: String, assignmentId: String): String = db.withTransaction { implicit session =>
+  def getConfirmationCode(workerId: String, assignmentId: String): String = db.withSession { implicit session =>
     amtAssignments.filter(a => a.workerId === workerId && a.assignmentId === assignmentId).sortBy(_.assignmentStart.desc).map(_.confirmationCode).first
   }
 
-  def getMostRecentAssignmentId(workerId: String): String = db.withTransaction { implicit session =>
+  def getMostRecentAssignmentId(workerId: String): String = db.withSession { implicit session =>
     amtAssignments.filter(_.workerId === workerId).sortBy(_.assignmentStart.desc).map(_.assignmentId).first
   }
 
-  def getMostRecentAMTAssignmentId(workerId: String): Int = db.withTransaction { implicit session =>
+  def getMostRecentAMTAssignmentId(workerId: String): Int = db.withSession { implicit session =>
     amtAssignments.filter(_.workerId === workerId).sortBy(_.assignmentStart.desc).map(_.amtAssignmentId).first
   }
 
@@ -84,7 +84,7 @@ object AMTAssignmentTable {
   /**
     * Update the `completed` column of the specified amt_assignment row.
     */
-  def updateCompleted(amtAssignmentId: Int, completed: Boolean): Int = db.withTransaction { implicit session =>
+  def updateCompleted(amtAssignmentId: Int, completed: Boolean): Int = db.withSession { implicit session =>
     val q = for { asg <- amtAssignments if asg.amtAssignmentId === amtAssignmentId } yield asg.completed
     q.update(completed)
   }

app/models/attribute/GlobalAttributeTable.scala

@@ -121,7 +121,7 @@ object GlobalAttributeTable {
     )
   )
 
-  def getAllGlobalAttributes: List[GlobalAttribute] = db.withTransaction { implicit session =>
+  def getAllGlobalAttributes: List[GlobalAttribute] = db.withSession { implicit session =>
     globalAttributes.list
   }
 
@@ -285,11 +285,11 @@ object GlobalAttributeTable {
       .list.map{ case (rId, typeId, count) => (rId, LabelTypeTable.labelTypeIdToLabelType(typeId).get, count) }
   }
 
-  def countGlobalAttributes: Int = db.withTransaction { implicit session =>
-    globalAttributes.length.run
+  def countGlobalAttributes: Int = db.withSession { implicit session =>
+    globalAttributes.size.run
   }
 
-  def save(newSess: GlobalAttribute): Int = db.withTransaction { implicit session =>
+  def save(newSess: GlobalAttribute): Int = db.withSession { implicit session =>
     val newId: Int = (globalAttributes returning globalAttributes.map(_.globalAttributeId)) += newSess
     newId
   }

app/models/attribute/GlobalAttributeUserAttributeTable.scala

@@ -30,7 +30,7 @@ object GlobalAttributeUserAttributeTable {
   val db: slick.Database = play.api.db.slick.DB
   val globalAttributeUserAttributes: TableQuery[GlobalAttributeUserAttributeTable] = TableQuery[GlobalAttributeUserAttributeTable]
 
-  def save(newSess: GlobalAttributeUserAttribute): Int = db.withTransaction { implicit session =>
+  def save(newSess: GlobalAttributeUserAttribute): Int = db.withSession { implicit session =>
     val newId: Int = (globalAttributeUserAttributes returning globalAttributeUserAttributes.map(_.globalAttributeUserAttributeId)) += newSess
     newId
   }

app/models/attribute/GlobalClusteringSessionTable.scala

@@ -57,7 +57,7 @@ object GlobalClusteringSessionTable {
   /**
     * Truncates global_clustering_session, global_attribute, and global_attribute_user_attribute.
     */
-  def truncateTables(): Unit = db.withTransaction { implicit session =>
+  def truncateTables(): Unit = db.withSession { implicit session =>
     Q.updateNA("TRUNCATE TABLE global_clustering_session CASCADE").execute
   }
 
@@ -67,11 +67,11 @@ object GlobalClusteringSessionTable {
    * We run the delete on the `global_clustering_session` table, and it cascades to the `global_attribute` and
    * `global_attribute_user_attribute` tables.
    */
-  def deleteGlobalClusteringSessions(regionIds: List[Int]): Int = db.withTransaction { implicit session =>
+  def deleteGlobalClusteringSessions(regionIds: List[Int]): Int = db.withSession { implicit session =>
     globalClusteringSessions.filter(_.regionId inSet regionIds).delete
   }
 
-  def save(newSess: GlobalClusteringSession): Int = db.withTransaction { implicit session =>
+  def save(newSess: GlobalClusteringSession): Int = db.withSession { implicit session =>
     val newId: Int = (globalClusteringSessions returning globalClusteringSessions.map(_.globalClusteringSessionId)) += newSess
     newId
   }

app/models/attribute/UserAttributeLabelTable.scala

@@ -31,11 +31,11 @@ object UserAttributeLabelTable {
   val db: slick.Database = play.api.db.slick.DB
   val userAttributeLabels: TableQuery[UserAttributeLabelTable] = TableQuery[UserAttributeLabelTable]
 
-  def countUserAttributeLabels: Int = db.withTransaction { implicit session =>
-    userAttributeLabels.length.run
+  def countUserAttributeLabels: Int = db.withSession { implicit session =>
+    userAttributeLabels.size.run
   }
 
-  def save(newSess: UserAttributeLabel): Int = db.withTransaction { implicit session =>
+  def save(newSess: UserAttributeLabel): Int = db.withSession { implicit session =>
     val newId: Int = (userAttributeLabels returning userAttributeLabels.map(_.userAttributeLabelId)) += newSess
     newId
   }

app/models/attribute/UserAttributeTable.scala

@@ -56,11 +56,11 @@ object UserAttributeTable {
   val db: slick.Database = play.api.db.slick.DB
   val userAttributes: TableQuery[UserAttributeTable] = TableQuery[UserAttributeTable]
 
-  def countUserAttributes: Int = db.withTransaction { implicit session =>
-    userAttributes.length.run
+  def countUserAttributes: Int = db.withSession { implicit session =>
+    userAttributes.size.run
   }
 
-  def save(newSess: UserAttribute): Int = db.withTransaction { implicit session =>
+  def save(newSess: UserAttribute): Int = db.withSession { implicit session =>
     val newId: Int = (userAttributes returning userAttributes.map(_.userAttributeId)) += newSess
     newId
   }

app/models/attribute/UserClusteringSessionTable.scala

@@ -89,7 +89,7 @@ object UserClusteringSessionTable {
   /**
     * Gets all clusters from single-user clustering that are in this region, outputs in format needed for clustering.
     */
-  def getClusteredLabelsInRegion(regionId: Int): List[LabelToCluster] = db.withTransaction { implicit session =>
+  def getClusteredLabelsInRegion(regionId: Int): List[LabelToCluster] = db.withSession { implicit session =>
     val labelsInRegion = for {
       _sess <- userClusteringSessions
       _att <- UserAttributeTable.userAttributes if _sess.userClusteringSessionId === _att.userClusteringSessionId
@@ -111,7 +111,7 @@ object UserClusteringSessionTable {
   /**
     * Truncates user_clustering_session, user_attribute, user_attribute_label, and global_attribute_user_attribute.
     */
-  def truncateTables(): Unit = db.withTransaction { implicit session =>
+  def truncateTables(): Unit = db.withSession { implicit session =>
     Q.updateNA("TRUNCATE TABLE user_clustering_session CASCADE").execute
   }
 
@@ -139,7 +139,7 @@ object UserClusteringSessionTable {
     userClusteringSessions.filter(_.userId inSet usersToDelete).delete
   }
 
-  def save(newSess: UserClusteringSession): Int = db.withTransaction { implicit session =>
+  def save(newSess: UserClusteringSession): Int = db.withSession { implicit session =>
     val newId: Int = (userClusteringSessions returning userClusteringSessions.map(_.userClusteringSessionId)) += newSess
     newId
   }

app/models/audit/AuditTaskCommentTable.scala

@@ -49,7 +49,7 @@ object AuditTaskCommentTable {
   /**
     * Get all task records of the given user.
     */
-  def all(username: String): Option[List[AuditTaskComment]] = db.withTransaction { implicit session =>
+  def all(username: String): Option[List[AuditTaskComment]] = db.withSession { implicit session =>
     val comments = (for {
       (c, u) <- auditTaskComments.innerJoin(users).on(_.userId === _.userId).sortBy(_._1.timestamp.desc) if u.username === username
     } yield (c.auditTaskCommentId, c.auditTaskId, c.missionId, c.edgeId, u.username, c.ipAddress, c.gsvPanoramaId,
@@ -61,10 +61,8 @@ object AuditTaskCommentTable {
   /**
     * Insert an audit_task_comment record.
     */
-  def save(comment: AuditTaskComment): Int = db.withTransaction { implicit session =>
-    val auditTaskCommentId: Int =
-      (auditTaskComments returning auditTaskComments.map(_.auditTaskCommentId)) += comment
-    auditTaskCommentId
+  def save(comment: AuditTaskComment): Int = db.withSession { implicit session =>
+    (auditTaskComments returning auditTaskComments.map(_.auditTaskCommentId)) += comment
   }
 
   /**

app/models/audit/AuditTaskEnvironmentTable.scala

@@ -49,9 +49,7 @@ object AuditTaskEnvironmentTable {
   /**
    * Saves a new audit task environment.
    */
-  def save(env: AuditTaskEnvironment): Int = db.withTransaction { implicit session =>
-    val auditTaskEnvironmentId: Int =
-      (auditTaskEnvironments returning auditTaskEnvironments.map(_.auditTaskEnvironmentId)) += env
-    auditTaskEnvironmentId
+  def save(env: AuditTaskEnvironment): Int = db.withSession { implicit session =>
+    (auditTaskEnvironments returning auditTaskEnvironments.map(_.auditTaskEnvironmentId)) += env
   }
 }

app/models/audit/AuditTaskIncompleteTable.scala

@@ -34,9 +34,7 @@ object AuditTaskIncompleteTable {
   /**
    * Saves a new audit task environment.
    */
-  def save(incomplete: AuditTaskIncomplete): Int = db.withTransaction { implicit session =>
-    val auditTaskIncompleteId: Int =
-      (incompletes returning incompletes.map(_.auditTaskIncompleteId)) += incomplete
-    auditTaskIncompleteId
+  def save(incomplete: AuditTaskIncomplete): Int = db.withSession { implicit session =>
+    (incompletes returning incompletes.map(_.auditTaskIncompleteId)) += incomplete
   }
 }

app/models/audit/AuditTaskInteractionTable.scala

@@ -133,7 +133,7 @@ object AuditTaskInteractionTable {
   /**
     * Inserts a sequence of interactions into the audit_task_interaction and audit_task_interaction_small tables.
     */
-  def saveMultiple(interactions: Seq[AuditTaskInteraction]): Seq[Long] = db.withTransaction { implicit session =>
+  def saveMultiple(interactions: Seq[AuditTaskInteraction]): Seq[Long] = db.withSession { implicit session =>
     val savedActions: Seq[AuditTaskInteraction] = (auditTaskInteractions returning auditTaskInteractions) ++= interactions
 
     // Insert copies of a subset of those interactions in audit_task_interaction_small for faster SELECT queries.

app/models/audit/AuditTaskTable.scala

@@ -182,7 +182,7 @@ object AuditTaskTable {
     * Returns the number of tasks completed.
     */
   def countCompletedAudits: Int = db.withSession { implicit session =>
-    completedTasks.length.run
+    completedTasks.size.run
   }
 
   /**
@@ -215,7 +215,7 @@ object AuditTaskTable {
     * Returns the number of tasks completed by the given user.
     */
   def countCompletedAudits(userId: UUID): Int = db.withSession { implicit session =>
-    completedTasks.filter(_.userId === userId.toString).length.run
+    completedTasks.filter(_.userId === userId.toString).size.run
   }
 
   /**

app/models/daos/slick/DBTableDefinitions.scala

@@ -63,28 +63,18 @@ object DBTableDefinitions {
 
     val db = play.api.db.slick.DB
 
-    def find(username: String): Option[DBUser] = db.withTransaction { implicit session =>
-      slickUsers.filter(_.username === username).firstOption match {
-        case Some(user) => Some(user)
-        case None => None
-      }
+    def find(username: String): Option[DBUser] = db.withSession { implicit session =>
+      slickUsers.filter(_.username === username).firstOption
     }
-    def findEmail(email: String): Option[DBUser] = db.withTransaction { implicit session =>
-      slickUsers.filter(_.email === email).firstOption match {
-        case Some(user) => Some(user)
-        case None => None
-      }
+    def findEmail(email: String): Option[DBUser] = db.withSession { implicit session =>
+      slickUsers.filter(_.email === email).firstOption
     }
-    def findById(userId: UUID): Option[DBUser] = db.withTransaction { implicit session =>
-      slickUsers.filter(_.userId === userId.toString).firstOption match {
-        case Some(user) => Some(user)
-        case None => None
-      }
+    def findById(userId: UUID): Option[DBUser] = db.withSession { implicit session =>
+      slickUsers.filter(_.userId === userId.toString).firstOption
     }
 
-    def count: Int = db.withTransaction { implicit session =>
-      val users = slickUsers.list
-      users.length
+    def count: Int = db.withSession { implicit session =>
+      slickUsers.size.run
     }
   }
 }

app/models/daos/slick/UserDAOSlick.scala

@@ -256,7 +256,7 @@ object UserDAOSlick {
     } yield _userTable.userId
 
     // The group by and map does a SELECT DISTINCT, and the list.length does the COUNT.
-    filteredUsers.groupBy(x => x).map(_._1).length.run
+    filteredUsers.groupBy(x => x).map(_._1).size.run
   }
 
   /**
@@ -368,7 +368,7 @@ object UserDAOSlick {
     } yield _user.userId
 
     // The group by and map does a SELECT DISTINCT, and the list.length does the COUNT.
-    users.groupBy(x => x).map(_._1).length.run
+    users.groupBy(x => x).map(_._1).size.run
   }
 
   /**

app/models/gallery/GalleryTaskEnvironmentTable.scala

@@ -46,9 +46,7 @@ object GalleryTaskEnvironmentTable {
    * @param env Data concerning the environment a gallery interaction was done in.
    * @return
    */
-  def save(env: GalleryTaskEnvironment): Int = db.withTransaction { implicit session =>
-    val galleryTaskEnvironmentId: Int =
-      (galleryTaskEnvironments returning galleryTaskEnvironments.map(_.galleryTaskEnvironmentId)) += env
-    galleryTaskEnvironmentId
+  def save(env: GalleryTaskEnvironment): Int = db.withSession { implicit session =>
+    (galleryTaskEnvironments returning galleryTaskEnvironments.map(_.galleryTaskEnvironmentId)) += env
   }
 }

app/models/gallery/GalleryTaskInteractionTable.scala

@@ -32,10 +32,8 @@ object GalleryTaskInteractionTable {
     * @param interaction The interaction to be saved.
     * @return
     */
-  def save(interaction: GalleryTaskInteraction): Int = db.withTransaction { implicit session =>
-    val interactionId: Int =
-      (galleryTaskInteractions returning galleryTaskInteractions.map(_.galleryTaskInteractionId)).insert(interaction)
-    interactionId
+  def save(interaction: GalleryTaskInteraction): Int = db.withSession { implicit session =>
+    (galleryTaskInteractions returning galleryTaskInteractions.map(_.galleryTaskInteractionId)).insert(interaction)
   }
 
   /**
@@ -44,7 +42,7 @@ object GalleryTaskInteractionTable {
     * @param interactions The interactions to be saved.
     * @return
     */
-  def saveMultiple(interactions: Seq[GalleryTaskInteraction]): Seq[Int] = db.withTransaction { implicit session =>
+  def saveMultiple(interactions: Seq[GalleryTaskInteraction]): Seq[Int] = db.withSession { implicit session =>
     (galleryTaskInteractions returning galleryTaskInteractions.map(_.galleryTaskInteractionId)) ++= interactions
   }
 }

app/models/gsv/GSVDataTable.scala

@@ -60,7 +60,7 @@ object GSVDataTable {
     LabelTable.labelsUnfiltered
       .filter(_.gsvPanoramaId =!= "tutorial")
       .groupBy(_.gsvPanoramaId).map(_._1)
-      .length.run
+      .size.run
   }
 
   /**

app/models/gsv/GSVLinkTable.scala

@@ -23,7 +23,7 @@ object GSVLinkTable {
     *
     * @param panoramaId Google Street View panorama id
     */
-  def linkExists(panoramaId: String, targetPanoramaId: String): Boolean = db.withTransaction { implicit session =>
+  def linkExists(panoramaId: String, targetPanoramaId: String): Boolean = db.withSession { implicit session =>
     gsvLinks.filter(x => x.gsvPanoramaId === panoramaId && x.targetGsvPanoramaId === targetPanoramaId).list.nonEmpty
   }
 
@@ -32,7 +32,7 @@ object GSVLinkTable {
     *
     * @param link GSVLink object
     */
-  def save(link: GSVLink): String = db.withTransaction { implicit session =>
+  def save(link: GSVLink): String = db.withSession { implicit session =>
     gsvLinks += link
     link.gsvPanoramaId
   }

app/models/label/LabelPointTable.scala

@@ -55,9 +55,7 @@ object LabelPointTable {
   /**
    * Stores a label point into the label_point table.
    */
-  def save(point: LabelPoint): Int = db.withTransaction { implicit session =>
-    val labelPointId: Int =
-      (labelPoints returning labelPoints.map(_.labelPointId)) += point
-    labelPointId
+  def save(point: LabelPoint): Int = db.withSession { implicit session =>
+    (labelPoints returning labelPoints.map(_.labelPointId)) += point
   }
 }