Skip to content

Commit

Permalink
Merge pull request #3283 from ProjectSidewalk/develop
Browse files Browse the repository at this point in the history 
v7.15.2
  • Loading branch information
@misaugstad
misaugstad authored Jul 6, 2023
2 parents c3913d4 + d998873 commit 9211f2e
Show file tree
Hide file tree
Showing 37 changed files with 256 additions and 595 deletions.
21 changes: 18 additions & 3 deletions app/controllers/AdminController.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -398,9 +398,24 @@ class AdminController @Inject() (implicit val env: Environment[User, SessionAuth
* Get metadata used for 2022 CV project for all labels, and output as JSON.
*/
def getAllLabelMetadataForCV = UserAwareAction.async { implicit request =>
val labels: List[LabelCVMetadata] = LabelTable.getLabelCVMetadata
val json: JsValue = Json.toJson(labels.map(l => Json.toJson(l)))
Future.successful(Ok(json))
val jsonFile = new java.io.File(s"cv_metadata_${new Timestamp(Instant.now.toEpochMilli).toString}.json")
val writer = new java.io.PrintStream(jsonFile)
writer.print("[")

// Grab 10k labels at a time and write them to a JSON file to reduce server memory usage and crashes.
var startIndex: Int = 0
val batchSize: Int = 10000
var moreWork: Boolean = true
while (moreWork) {
val features: List[JsValue] = LabelTable.getLabelCVMetadata(startIndex, batchSize).map(l => Json.toJson(l))
writer.print(features.map(_.toString).mkString(","))
startIndex += batchSize
if (features.length < batchSize) moreWork = false
}
writer.print("]")
writer.close()

Future.successful(Ok.sendFile(content = jsonFile, inline = true, onClose = () => jsonFile.delete()))
}

/**
Expand Down
46 changes: 38 additions & 8 deletions app/controllers/ProjectSidewalkAPIController.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -163,10 +163,25 @@ class ProjectSidewalkAPIController @Inject()(implicit val env: Environment[User,
val shapefile: java.io.File = ShapefilesCreatorHelper.zipShapeFiles("attributeWithLabels", Array("attributes", "labels"))

Future.successful(Ok.sendFile(content = shapefile, onClose = () => shapefile.delete()))
} else { // In GeoJSON format.
val features: List[JsObject] =
GlobalAttributeTable.getGlobalAttributesWithLabelsInBoundingBox(minLat, minLng, maxLat, maxLng, severity).map(_.toJSON)
Future.successful(Ok(Json.obj("type" -> "FeatureCollection", "features" -> features)))
} else {
// In GeoJSON format. Writing 10k objects to a file at a time to reduce server memory usage and crashes.
val attributesJsonFile = new java.io.File(s"attributesWithLabels_${new Timestamp(Instant.now.toEpochMilli).toString}.json")
val writer = new java.io.PrintStream(attributesJsonFile)
writer.print("""{"type":"FeatureCollection","features":[""")

var startIndex: Int = 0
val batchSize: Int = 10000
var moreWork: Boolean = true
while (moreWork) {
val features: List[JsObject] = GlobalAttributeTable.getGlobalAttributesWithLabelsInBoundingBox(minLat, minLng, maxLat, maxLng, severity, Some(startIndex), Some(batchSize)).map(_.toJSON)
writer.print(features.map(_.toString).mkString(","))
startIndex += batchSize
if (features.length < batchSize) moreWork = false
}
writer.print("]}")
writer.close()

Future.successful(Ok.sendFile(content = attributesJsonFile, inline = true, onClose = () => attributesJsonFile.delete()))
}
}

Expand Down Expand Up @@ -206,10 +221,25 @@ class ProjectSidewalkAPIController @Inject()(implicit val env: Environment[User,
ShapefilesCreatorHelper.createAttributeShapeFile("attributes", attributeList)
val shapefile: java.io.File = ShapefilesCreatorHelper.zipShapeFiles("accessAttributes", Array("attributes"));
Future.successful(Ok.sendFile(content = shapefile, onClose = () => shapefile.delete()))
} else { // In GeoJSON format.
val features: List[JsObject] =
GlobalAttributeTable.getGlobalAttributesInBoundingBox(minLat, minLng, maxLat, maxLng, severity).map(_.toJSON)
Future.successful(Ok(Json.obj("type" -> "FeatureCollection", "features" -> features)))
} else {
// In GeoJSON format. Writing 10k objects to a file at a time to reduce server memory usage and crashes.
val attributesJsonFile = new java.io.File(s"attributes_${new Timestamp(Instant.now.toEpochMilli).toString}.json")
val writer = new java.io.PrintStream(attributesJsonFile)
writer.print("""{"type":"FeatureCollection","features":[""")

var startIndex: Int = 0
val batchSize: Int = 10000
var moreWork: Boolean = true
while (moreWork) {
val features: List[JsObject] = GlobalAttributeTable.getGlobalAttributesInBoundingBox(minLat, minLng, maxLat, maxLng, severity, Some(startIndex), Some(batchSize)).map(_.toJSON)
writer.print(features.map(_.toString).mkString(","))
startIndex += batchSize
if (features.length < batchSize) moreWork = false
}
writer.print("]}")
writer.close()

Future.successful(Ok.sendFile(content = attributesJsonFile, inline = true, onClose = () => attributesJsonFile.delete()))
}
}

Expand Down
12 changes: 8 additions & 4 deletions app/models/attribute/GlobalAttributeTable.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -216,7 +216,7 @@ object GlobalAttributeTable {
/**
* Gets global attributes within a bounding box for the public API.
*/
def getGlobalAttributesInBoundingBox(minLat: Float, minLng: Float, maxLat: Float, maxLng: Float, severity: Option[String]): List[GlobalAttributeForAPI] = db.withSession { implicit session =>
def getGlobalAttributesInBoundingBox(minLat: Float, minLng: Float, maxLat: Float, maxLng: Float, severity: Option[String], startIndex: Option[Int] = None, n: Option[Int] = None): List[GlobalAttributeForAPI] = db.withSession { implicit session =>
// Sum the validations counts, average date, and the number of the labels that make up each global attribute.
val validationCounts =
"""SELECT global_attribute.global_attribute_id AS global_attribute_id,
Expand Down Expand Up @@ -285,15 +285,17 @@ object GlobalAttributeTable {
| AND ${severity.getOrElse("") == "none"}
| OR ${severity.isEmpty}
| OR global_attribute.severity = ${toInt(severity).getOrElse(-1)}
| );""".stripMargin
| )
|ORDER BY global_attribute.global_attribute_id
|${if (n.isDefined && startIndex.isDefined) s"LIMIT ${n.get} OFFSET ${startIndex.get}" else ""};""".stripMargin
)
attributes.list
}

/**
* Gets global attributes within a bounding box with the labels that make up those attributes for the public API.
*/
def getGlobalAttributesWithLabelsInBoundingBox(minLat: Float, minLng: Float, maxLat: Float, maxLng: Float, severity: Option[String]): List[GlobalAttributeWithLabelForAPI] = db.withSession { implicit session =>
def getGlobalAttributesWithLabelsInBoundingBox(minLat: Float, minLng: Float, maxLat: Float, maxLng: Float, severity: Option[String], startIndex: Option[Int] = None, n: Option[Int] = None): List[GlobalAttributeWithLabelForAPI] = db.withSession { implicit session =>
val attributesWithLabels = Q.queryNA[GlobalAttributeWithLabelForAPI](
s"""SELECT global_attribute.global_attribute_id,
| label_type.label_type,
Expand Down Expand Up @@ -349,7 +351,9 @@ object GlobalAttributeTable {
| AND ${severity.getOrElse("") == "none"}
| OR ${severity.isEmpty}
| OR global_attribute.severity = ${toInt(severity).getOrElse(-1)}
| );""".stripMargin
| )
|ORDER BY user_attribute_label_id
|${if (n.isDefined && startIndex.isDefined) s"LIMIT ${n.get} OFFSET ${startIndex.get}" else ""};""".stripMargin
)
attributesWithLabels.list
}
Expand Down
7 changes: 4 additions & 3 deletions app/models/label/LabelTable.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1333,7 +1333,7 @@ object LabelTable {
/**
* Get metadata used for 2022 CV project for all labels.
*/
def getLabelCVMetadata: List[LabelCVMetadata] = db.withSession { implicit session =>
def getLabelCVMetadata(startIndex: Int, batchSize: Int): List[LabelCVMetadata] = db.withSession { implicit session =>
(for {
_l <- labels
_lp <- labelPoints if _l.labelId === _lp.labelId
Expand All @@ -1342,7 +1342,8 @@ object LabelTable {
} yield (
_l.labelId, _gsv.gsvPanoramaId, _l.labelTypeId, _l.agreeCount, _l.disagreeCount, _l.notsureCount, _gsv.width,
_gsv.height, _lp.panoX, _lp.panoY, LabelPointTable.canvasWidth, LabelPointTable.canvasHeight, _lp.canvasX,
_lp.canvasY, _lp.zoom, _lp.heading, _lp.pitch, _gsv.cameraHeading.get, _gsv.cameraPitch.get
)).list.map(LabelCVMetadata.tupled)
_lp.canvasY, _lp.zoom, _lp.heading, _lp.pitch, _gsv.cameraHeading.asColumnOf[Float],
_gsv.cameraPitch.asColumnOf[Float]
)).drop(startIndex).take(batchSize).list.map(LabelCVMetadata.tupled)
}
}
125 changes: 2 additions & 123 deletions app/views/explore.scala.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -199,7 +199,7 @@ <h4 class="survey-question">@Html(Messages("audit.survey." + surveyQuestion.surv
</div>
<div id="left-column-zoom-out-button" class="button zoom-control-button" title="@Messages("press.keys", "Shift + Z")" data-toggle="tooltip" data-placement="top">
<img src='@routes.Assets.at("javascripts/SVLabel/img/icons/ZoomOut.svg")' class="zoom-button-icon" alt="Zoom out">
<br>@Messages("audit.left.ui.zoom.out")
<br>@Html(Messages("audit.left.ui.zoom.out"))
</div>
<div id="left-column-sound-button" class="button">
<img src='@routes.Assets.at("javascripts/SVLabel/img/icons/sound.png")' id="left-column-sound-icon" class="visible" alt="Sound icon">
Expand All @@ -220,7 +220,7 @@ <h4 class="survey-question">@Html(Messages("audit.survey." + surveyQuestion.surv
<div id="left-column-feedback-button" class="button" data-toggle="popover" data-template="<div class='feedback-popover' role='tooltip'><div class='arrow'></div><div class='popover-content'></div></div>" data-placement="top" data-content='@Messages("feedback.submitted")' data-trigger="manual">
<img src='@routes.Assets.at("javascripts/SVLabel/img/icons/comment.png")' alt="Comment icon">
<br>
@Messages("audit.left.ui.feedback")
@Html(Messages("feedback"))
</div>
</div>
</div>
Expand Down Expand Up @@ -346,7 +346,6 @@ <h2 id="total-mission-reward" style="display:none"></h2>
<div id="ribbon-street-view-connector"><!-- border --></div>
<!-- You can put an extra div element on top of the Street View div panel to disable any of StreetView control. -->
<div id="pano" class="window-streetview" style="z-index: 0;" ></div>
<div id="overlay-message-holder" class="window-streetview"></div>
<div id="onboarding-holder" class="window-streetview">
<canvas id="onboarding-canvas" class="window-streetview" width="720px" height="480px" style="cursor: default, move;"></canvas>
<div id="hand-gesture-holder"></div>
Expand Down Expand Up @@ -622,126 +621,6 @@ <h3 id="modal-mission-complete-progress-title"></h3>
</div>
</div>
</div>
<div id="modal-curb-ramp-example" class="hidden">
<div class="modal-background"></div>
<div class="modal-foreground" id="modal-curb-ramp-foreground">
<div class="row">
<div class="col-md-12">
<h1 class="text-center bold">@Messages("curb.ramp")</h1>
</div>
<div class="modal-close-button-holder">
<button type="button" class="close modal-example-close-buttons" aria-label="Close"><span aria-hidden="true">&times;</span></button>
</div>
</div>
<div class="row">
<div class="col-md-4">
<img src='@routes.Assets.at("images/examples/CurbRamp_Passable.png")' class="img-responsive" alt="Two good curb ramps" />
<p class="bold">@Messages("audit.help.rating.1")</p>
<p>@Messages("audit.help.curb.ramp.1")</p>
</div>
<div class="col-md-4">
<img src='@routes.Assets.at("images/examples/CurbRamp_HardToPass.png")' class="img-responsive" alt="Water has accumulated in the curb ramp" />
<p class="bold">@Messages("audit.help.rating.3")</p>
<p>@Messages("audit.help.curb.ramp.3")</p>
</div>
<div class="col-md-4">
<img src='@routes.Assets.at("images/examples/CurbRamp_NotPassable.png")' class="img-responsive" alt="Water has accumulated in the curb ramp" />
<p class="bold">@Messages("audit.help.rating.5")</p>
<p>@Messages("audit.help.curb.ramp.5")</p>
</div>
</div>
</div>
</div>
<div id="modal-no-curb-ramp-example" class="hidden">
<div class="modal-background"></div>
<div class="modal-foreground" id="modal-no-curb-ramp-foreground">
<div class="row">
<div class="col-md-12">
<h1 class="text-center bold">@Messages("missing.ramp")</h1>
</div>
<div class="modal-close-button-holder">
<button type="button" class="close modal-example-close-buttons" aria-label="Close"><span aria-hidden="true">&times;</span></button>
</div>
</div>
<div class="row">
<div class="col-md-4">
<img src='@routes.Assets.at("images/examples/NoCurbRamp_Passable.png")' class="img-responsive" alt="Curb ramp is not aligned to the crosswalk" />
<p class="bold">@Messages("audit.help.rating.1")</p>
<p>@Messages("audit.help.missing.ramp.1")</p>
</div>
<div class="col-md-4">
<img src='@routes.Assets.at("images/examples/NoCurbRamp_HardToPass.png")' class="img-responsive" alt="No curb ramp at the end of the crosswalk" />
<p class="bold">@Messages("audit.help.rating.3")</p>
<p>@Messages("audit.help.missing.ramp.3")</p>
</div>
<div class="col-md-4">
<img src='@routes.Assets.at("images/examples/NoCurbRamp_NotPassable.png")' class="img-responsive" alt="No curb ramp at the end of the crosswalk" />
<p class="bold">@Messages("audit.help.rating.5")</p>
<p>@Messages("audit.help.missing.ramp.5")</p>
</div>
</div>
</div>
</div>
<div id="modal-obstacle-example" class="hidden">
<div class="modal-background"></div>
<div class="modal-foreground" id="modal-obstacle-foreground">
<div class="row">
<div class="col-md-12">
<h1 class="text-center bold">@Messages("obstacle")</h1>
</div>
<div class="modal-close-button-holder">
<button type="button" class="close modal-example-close-buttons" aria-label="Close"><span aria-hidden="true">&times;</span></button>
</div>
</div>
<div class="row">
<div class="col-md-4">
<img src='@routes.Assets.at("images/examples/Obstacle_Passable.png")' class="img-responsive" alt="Overgrown bush is partly blocking the path." />
<p class="bold">@Messages("audit.help.rating.1")</p>
<p>@Messages("audit.help.obstacle.1")</p>
</div>
<div class="col-md-4">
<img src='@routes.Assets.at("images/examples/Obstacle_HardToPass.png")' class="img-responsive" alt="Overgrown bush is partly blocking the path." />
<p class="bold">@Messages("audit.help.rating.3")</p>
<p>@Messages("audit.help.obstacle.3")</p>
</div>
<div class="col-md-4">
<img src='@routes.Assets.at("images/examples/Obstacle_NotPassable.png")' class="img-responsive" alt="A tree is completely blocking the path." />
<p class="bold">@Messages("audit.help.rating.5")</p>
<p>@Messages("audit.help.obstacle.5")</p>
</div>
</div>
</div>
</div>
<div id="modal-surface-problem-example" class="hidden">
<div class="modal-background"></div>
<div class="modal-foreground" id="modal-surface-problem-foreground">
<div class="row">
<div class="col-md-12">
<h1 class="text-center bold">@Messages("surface.problem")</h1>
</div>
<div class="modal-close-button-holder">
<button type="button" class="close modal-example-close-buttons" aria-label="Close"><span aria-hidden="true">&times;</span></button>
</div>
</div>
<div class="row">
<div class="col-md-4">
<img src='@routes.Assets.at("images/examples/SurfaceProblem_Passable.png")' class="img-responsive" alt="A partially damaged sidewalk." />
<p class="bold">@Messages("audit.help.rating.1")</p>
<p>@Messages("audit.help.surface.problem.1")</p>
</div>
<div class="col-md-4">
<img src='@routes.Assets.at("images/examples/SurfaceProblem_HardToPass.png")' class="img-responsive" alt="Cobblestone sidewalks and crosswalks." />
<p class="bold">@Messages("audit.help.rating.3")</p>
<p>@Messages("audit.help.surface.problem.3")</p>
</div>
<div class="col-md-4">
<img src='@routes.Assets.at("images/examples/SurfaceProblem_NotPassable.png")' class="img-responsive" alt="A tree is completely blocking the path." />
<p class="bold">@Messages("audit.help.rating.5")</p>
<p>@Messages("audit.help.surface.problem.5")</p>
</div>
</div>
</div>
</div>
</div>
</div>

Expand Down
Loading

0 comments on commit 9211f2e

Please sign in to comment.