Merge pull request #3452 from ProjectSidewalk/sanyam_v_savla

#3413 Shapefile format doesn't allow integers longer than 9 digits

app/controllers/ProjectSidewalkAPIController.scala

@@ -41,7 +41,7 @@ case class NeighborhoodAttributeSignificance (val name: String,
 
 case class StreetAttributeSignificance (val geometry: Array[JTSCoordinate],
                                         val streetID: Int,
-                                        val osmID: Int,
+                                        val osmID: Long,
                                         val regionID: Int,
                                         val score: Double,
                                         val auditCount: Int,
@@ -61,7 +61,7 @@ class ProjectSidewalkAPIController @Inject()(implicit val env: Environment[User,
 
   case class AttributeForAccessScore(lat: Float, lng: Float, labelType: String, avgImageCaptureDate: Timestamp,
                                      avgLabelDate: Timestamp, imageCount: Int, labelCount: Int)
-  case class AccessScoreStreet(streetEdge: StreetEdge, osmId: Int, regionId: Int, score: Double, auditCount: Int,
+  case class AccessScoreStreet(streetEdge: StreetEdge, osmId: Long, regionId: Int, score: Double, auditCount: Int,
                                attributes: Array[Double], significance: Array[Double],
                                avgImageCaptureDate: Option[Timestamp], avgLabelDate: Option[Timestamp], imageCount: Int,
                                labelCount: Int) {
@@ -165,6 +165,7 @@ class ProjectSidewalkAPIController @Inject()(implicit val env: Environment[User,
 
       val shapefile: java.io.File = ShapefilesCreatorHelper.zipShapeFiles("attributeWithLabels", Array("attributes", "labels"))
 
+
       Future.successful(Ok.sendFile(content = shapefile, onClose = () => shapefile.delete()))
     } else {
       // In GeoJSON format. Writing 10k objects to a file at a time to reduce server memory usage and crashes.

app/controllers/helper/ShapefilesCreatorHelper.java

@@ -31,6 +31,7 @@
  */
 public class ShapefilesCreatorHelper {
 
+
     public static void createGeneralShapeFile(String outputFile, SimpleFeatureType TYPE, List<SimpleFeature> features) throws Exception {
         /*
          * Get an output file name and create the new shapefile
@@ -100,7 +101,7 @@ public static void createAttributeShapeFile(String outputFile, List<GlobalAttrib
                         + "id:Integer," // a attribute ID
                         + "labelType:String," // Label type
                         + "streetId:Integer," // Street edge ID of the nearest street
-                        + "osmWayId:Integer," // Street OSM ID of the nearest street
+                        + "osmWayId:String," // Street OSM ID of the nearest street
                         + "neighborhd:String," // Neighborhood Name
                         + "avgImgDate:String," // Image date
                         + "avgLblDate:String," // Label date
@@ -131,7 +132,7 @@ public static void createAttributeShapeFile(String outputFile, List<GlobalAttrib
             featureBuilder.add(a.globalAttributeId());
             featureBuilder.add(a.labelType());
             featureBuilder.add(a.streetEdgeId());
-            featureBuilder.add(a.osmStreetId());
+            featureBuilder.add(String.valueOf(a.osmStreetId()));
             featureBuilder.add(a.neighborhoodName());
             featureBuilder.add(a.avgImageCaptureDate());
             featureBuilder.add(a.avgLabelDate());
@@ -168,7 +169,7 @@ public static void createLabelShapeFile(String outputFile, List<GlobalAttributeW
                         + "attribId:Integer," // attribute ID
                         + "labelType:String," // Label type
                         + "streetId:Integer," // Street edge ID of the nearest street
-                        + "osmWayId:Integer," // Street OSM ID of the nearest street (10 char max)
+                        + "osmWayId:String," // Street OSM ID of the nearest street (10 char max)
                         + "neighborhd:String," // Neighborhood Name
                         + "severity:Integer," // Severity
                         + "temporary:Boolean," // Temporary flag
@@ -212,7 +213,7 @@ public static void createLabelShapeFile(String outputFile, List<GlobalAttributeW
             featureBuilder.add(l.globalAttributeId());
             featureBuilder.add(l.labelType());
             featureBuilder.add(l.streetEdgeId());
-            featureBuilder.add(l.osmStreetId());
+            featureBuilder.add(String.valueOf(l.osmStreetId()));
             featureBuilder.add(l.neighborhoodName());
             featureBuilder.add(l.labelSeverity().getOrElse(new AbstractFunction0<Integer>() {
                 @Override
@@ -261,7 +262,7 @@ public static void createStreetShapefile(String outputFile, List<StreetAttribute
                         "Location",
                         "the_geom:LineString:srid=4326," // the geometry attribute: Line type
                         + "streetId:Integer," // StreetId
-                        + "osmWayId:Integer," // osmWayId
+                        + "osmWayId:String," // osmWayId
                         + "nghborhdId:String," // Region ID
                         + "score:Double," // street score
                         + "auditCount:Integer," // boolean representing whether the street is audited
@@ -293,7 +294,7 @@ public static void createStreetShapefile(String outputFile, List<StreetAttribute
         for (StreetAttributeSignificance s : streets) {
             featureBuilder.add(geometryFactory.createLineString(s.geometry()));
             featureBuilder.add(s.streetID());
-            featureBuilder.add(s.osmID());
+            featureBuilder.add(String.valueOf(s.osmID()));
             featureBuilder.add(s.regionID());
             featureBuilder.add(s.score());
             featureBuilder.add(s.auditCount());

app/models/attribute/GlobalAttributeTable.scala

@@ -32,7 +32,7 @@ case class GlobalAttributeForAPI(val globalAttributeId: Int,
                                  val disagreeCount: Int,
                                  val notsureCount: Int,
                                  val streetEdgeId: Int,
-                                 val osmStreetId: Int,
+                                 val osmStreetId: Long,
                                  val neighborhoodName: String,
                                  val avgImageCaptureDate: Timestamp,
                                  val avgLabelDate: Timestamp,
@@ -74,7 +74,7 @@ case class GlobalAttributeWithLabelForAPI(val globalAttributeId: Int,
                                           val attributeSeverity: Option[Int],
                                           val attributeTemporary: Boolean,
                                           val streetEdgeId: Int,
-                                          val osmStreetId: Int,
+                                          val osmStreetId: Long,
                                           val neighborhoodName: String,
                                           val labelId: Int,
                                           val labelLatLng: (Float, Float),
@@ -188,14 +188,14 @@ object GlobalAttributeTable {
   implicit val GlobalAttributeForAPIConverter = GetResult[GlobalAttributeForAPI](r =>
     GlobalAttributeForAPI(
       r.nextInt, r.nextString, r.nextFloat, r.nextFloat, r.nextIntOption, r.nextBoolean, r.nextInt, r.nextInt,
-      r.nextInt, r.nextInt, r.nextInt, r.nextString, r.nextTimestamp, r.nextTimestamp, r.nextInt, r.nextInt,
+      r.nextInt, r.nextInt, r.nextLong, r.nextString, r.nextTimestamp, r.nextTimestamp, r.nextInt, r.nextInt,
       r.nextString.split(",").toList.distinct
     )
   )
 
   implicit val GlobalAttributeWithLabelForAPIConverter = GetResult[GlobalAttributeWithLabelForAPI](r =>
     GlobalAttributeWithLabelForAPI(
-      r.nextInt, r.nextString, (r.nextFloat, r.nextFloat), r.nextIntOption, r.nextBoolean, r.nextInt, r.nextInt, r.nextString,
+      r.nextInt, r.nextString, (r.nextFloat, r.nextFloat), r.nextIntOption, r.nextBoolean, r.nextInt, r.nextLong, r.nextString,
       r.nextInt, (r.nextFloat, r.nextFloat), r.nextString, (r.nextFloat, r.nextFloat, r.nextInt),
       (r.nextInt, r.nextInt), (r.nextInt, r.nextInt, r.nextInt), r.nextIntOption, r.nextBoolean,
       (r.nextString, r.nextTimestamp), r.nextStringOption.map(tags => tags.split(",").toList).getOrElse(List()),

app/models/street/OsmWayStreetEdgeTable.scala

@@ -5,11 +5,11 @@ import play.api.Play.current
 import play.api.db.slick
 import scala.slick.lifted.{Tag}
 
-case class OsmWayStreetEdge(osmWayStreetEdgeId: Int, osmWayId: Int, streetEdgeId: Int)
+case class OsmWayStreetEdge(osmWayStreetEdgeId: Int, osmWayId: Long, streetEdgeId: Int)
 
 class OsmWayStreetEdgeTable(tag: Tag) extends Table[OsmWayStreetEdge](tag, "osm_way_street_edge") {
   def osmWayStreetEdgeId = column[Int]("osm_way_street_edge_id", O.NotNull, O.PrimaryKey, O.AutoInc)
-  def osmWayId = column[Int]("osm_way_id", O.NotNull)
+  def osmWayId = column[Long]("osm_way_id", O.NotNull)
   def streetEdgeId = column[Int]("street_edge_id", O.NotNull)
 
   def * = (osmWayStreetEdgeId, osmWayId, streetEdgeId) <> ((OsmWayStreetEdge.apply _).tupled, OsmWayStreetEdge.unapply)

app/models/street/StreetEdgeTable.scala

@@ -17,7 +17,7 @@ import scala.slick.jdbc.{GetResult, StaticQuery => Q}
 
 case class StreetEdge(streetEdgeId: Int, geom: LineString, x1: Float, y1: Float, x2: Float, y2: Float, wayType: String, deleted: Boolean, timestamp: Option[Timestamp])
 
-case class StreetEdgeInfo(val street: StreetEdge, osmId: Int, regionId: Int, val auditCount: Int)
+case class StreetEdgeInfo(val street: StreetEdge, osmId: Long, regionId: Int, val auditCount: Int)
 
 class StreetEdgeTable(tag: Tag) extends Table[StreetEdge](tag, "street_edge") {
   def streetEdgeId = column[Int]("street_edge_id", O.PrimaryKey)
@@ -64,7 +64,7 @@ object StreetEdgeTable {
     val wayType = r.nextString
     val deleted = r.nextBoolean
     val timestamp = r.nextTimestampOption
-    val osmId = r.nextInt
+    val osmId = r.nextLong
     val regionId = r.nextInt
     val auditCount = r.nextInt
     StreetEdgeInfo(StreetEdge(streetEdgeId, geometry, x1, y1, x2, y2, wayType, deleted, timestamp), osmId, regionId, auditCount)

conf/evolutions/default/211.sql

@@ -0,0 +1,15 @@
+# --- !Ups
+
+ALTER TABLE osm_way_street_edge
+ALTER COLUMN osm_way_id TYPE BIGINT;
+
+# --- !Downs
+
+ALTER TABLE osm_way_street_edge
+    ADD COLUMN osm_way_id_temp INT;
+UPDATE osm_way_street_edge
+    SET osm_way_id_temp = CAST(LEFT(osm_way_id::TEXT, 9) AS INT);
+ALTER TABLE osm_way_street_edge
+    DROP COLUMN osm_way_id;
+ALTER TABLE osm_way_street_edge
+    RENAME COLUMN osm_way_id_temp TO osm_way_id;