From e5f100e4c05f07f76b0084a6ae55003db3a753ad Mon Sep 17 00:00:00 2001 From: cka-y Date: Sun, 18 Feb 2024 18:45:14 -0500 Subject: [PATCH 1/7] feat: reformat trip and shape dist validator --- .../TripAndShapeDistanceValidator.java | 41 ++++++++++--------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/TripAndShapeDistanceValidator.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/TripAndShapeDistanceValidator.java index 5725ef6e43..953913b1ec 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/TripAndShapeDistanceValidator.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/TripAndShapeDistanceValidator.java @@ -51,32 +51,33 @@ public class TripAndShapeDistanceValidator extends FileValidator { @Override public void validate(NoticeContainer noticeContainer) { - shapeTable - .byShapeIdMap() + tripTable + .getEntities() .forEach( - (shapeId, shape) -> { + trip -> { + String shapeId = trip.shapeId(); + // Get distance for trip + int nbStopTimes = stopTimeTable.byTripId(trip.tripId()).size(); + if (nbStopTimes == 0) { + return; + } + double maxStopTimeDist = + stopTimeTable + .byTripId(trip.tripId()) + .get(nbStopTimes - 1) + .shapeDistTraveled(); + + // Get max shape distance for trip double maxShapeDist = shapeTable.byShapeId(shapeId).stream() .mapToDouble(GtfsShape::shapeDistTraveled) .max() .orElse(Double.NEGATIVE_INFINITY); - - tripTable - .byShapeId(shapeId) - .forEach( - trip -> { - double maxStopTimeDist = - stopTimeTable.byTripId(trip.tripId()).stream() - .mapToDouble(GtfsStopTime::shapeDistTraveled) - .max() - .orElse(Double.NEGATIVE_INFINITY); - - if (maxStopTimeDist > maxShapeDist) { - noticeContainer.addValidationNotice( - new TripDistanceExceedsShapeDistanceNotice( - trip.tripId(), shapeId, maxStopTimeDist, maxShapeDist)); - } - }); + if (maxStopTimeDist > maxShapeDist) { + noticeContainer.addValidationNotice( + new TripDistanceExceedsShapeDistanceNotice( + trip.tripId(), shapeId, maxStopTimeDist, maxShapeDist)); + } }); } From acf506a84006e4b7bb015b35b7941bf4f246c1a3 Mon Sep 17 00:00:00 2001 From: cka-y Date: Sun, 18 Feb 2024 20:42:52 -0500 Subject: [PATCH 2/7] fix: max error --- .../gtfsvalidator/validator/TripAndShapeDistanceValidator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/TripAndShapeDistanceValidator.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/TripAndShapeDistanceValidator.java index 953913b1ec..b55ab97db8 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/TripAndShapeDistanceValidator.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/TripAndShapeDistanceValidator.java @@ -72,7 +72,7 @@ public void validate(NoticeContainer noticeContainer) { shapeTable.byShapeId(shapeId).stream() .mapToDouble(GtfsShape::shapeDistTraveled) .max() - .orElse(Double.NEGATIVE_INFINITY); + .orElse(Double.POSITIVE_INFINITY); if (maxStopTimeDist > maxShapeDist) { noticeContainer.addValidationNotice( new TripDistanceExceedsShapeDistanceNotice( From ac64c10330c430a064ea432182d306b64b51cd03 Mon Sep 17 00:00:00 2001 From: cka-y Date: Sun, 18 Feb 2024 20:43:40 -0500 Subject: [PATCH 3/7] fix: max error --- .../validator/TripAndShapeDistanceValidator.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/TripAndShapeDistanceValidator.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/TripAndShapeDistanceValidator.java index b55ab97db8..00c3d0d966 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/TripAndShapeDistanceValidator.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/TripAndShapeDistanceValidator.java @@ -59,13 +59,10 @@ public void validate(NoticeContainer noticeContainer) { // Get distance for trip int nbStopTimes = stopTimeTable.byTripId(trip.tripId()).size(); if (nbStopTimes == 0) { - return; + return; } double maxStopTimeDist = - stopTimeTable - .byTripId(trip.tripId()) - .get(nbStopTimes - 1) - .shapeDistTraveled(); + stopTimeTable.byTripId(trip.tripId()).get(nbStopTimes - 1).shapeDistTraveled(); // Get max shape distance for trip double maxShapeDist = From 8f9ac7c903961cbe5df3722f6f20cb80510db94b Mon Sep 17 00:00:00 2001 From: cka-y Date: Mon, 19 Feb 2024 14:50:19 -0500 Subject: [PATCH 4/7] feat: added threshold of 1.11m --- .../TripAndShapeDistanceValidator.java | 93 ++++++++++++++++--- .../validator/NoticeFieldsTest.java | 1 + .../TripAndShapeDistanceValidatorTest.java | 74 ++++++++++----- 3 files changed, 134 insertions(+), 34 deletions(-) diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/TripAndShapeDistanceValidator.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/TripAndShapeDistanceValidator.java index 00c3d0d966..424dbb98dc 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/TripAndShapeDistanceValidator.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/TripAndShapeDistanceValidator.java @@ -16,7 +16,10 @@ package org.mobilitydata.gtfsvalidator.validator; import static org.mobilitydata.gtfsvalidator.notice.SeverityLevel.ERROR; +import static org.mobilitydata.gtfsvalidator.notice.SeverityLevel.WARNING; +import static org.mobilitydata.gtfsvalidator.util.S2Earth.getDistanceMeters; +import java.util.Comparator; import javax.inject.Inject; import org.mobilitydata.gtfsvalidator.annotation.GtfsValidationNotice; import org.mobilitydata.gtfsvalidator.annotation.GtfsValidationNotice.FileRefs; @@ -32,21 +35,22 @@ */ @GtfsValidator public class TripAndShapeDistanceValidator extends FileValidator { - private final GtfsTripTableContainer tripTable; - private final GtfsStopTimeTableContainer stopTimeTable; - + private final GtfsStopTableContainer stopTable; private final GtfsShapeTableContainer shapeTable; + private final double DISTANCE_THRESHOLD = 1.11; // distance in meters @Inject TripAndShapeDistanceValidator( GtfsTripTableContainer tripTable, GtfsStopTimeTableContainer stopTimeTable, + GtfsStopTableContainer stopTable, GtfsShapeTableContainer shapeTable) { this.tripTable = tripTable; this.stopTimeTable = stopTimeTable; this.shapeTable = shapeTable; + this.stopTable = stopTable; } @Override @@ -61,24 +65,43 @@ public void validate(NoticeContainer noticeContainer) { if (nbStopTimes == 0) { return; } - double maxStopTimeDist = - stopTimeTable.byTripId(trip.tripId()).get(nbStopTimes - 1).shapeDistTraveled(); + GtfsStopTime lastStopTime = + stopTimeTable.byTripId(trip.tripId()).get(nbStopTimes - 1); + GtfsStop stop = stopTable.byStopId(lastStopTime.stopId()).orElse(null); + if (stop == null) { + return; + } + double maxStopTimeDist = lastStopTime.shapeDistTraveled(); // Get max shape distance for trip - double maxShapeDist = + GtfsShape maxShape = shapeTable.byShapeId(shapeId).stream() - .mapToDouble(GtfsShape::shapeDistTraveled) - .max() - .orElse(Double.POSITIVE_INFINITY); - if (maxStopTimeDist > maxShapeDist) { + .max(Comparator.comparingDouble(GtfsShape::shapeDistTraveled)) + .orElse(null); + if (maxShape == null) { + return; + } + + double maxShapeDist = maxShape.shapeDistTraveled(); + double distanceInMeters = + getDistanceMeters(maxShape.shapePtLatLon(), stop.stopLatLon()); + if (distanceInMeters > DISTANCE_THRESHOLD) { noticeContainer.addValidationNotice( new TripDistanceExceedsShapeDistanceNotice( - trip.tripId(), shapeId, maxStopTimeDist, maxShapeDist)); + trip.tripId(), shapeId, maxStopTimeDist, maxShapeDist, distanceInMeters)); + } else if (distanceInMeters > 0) { + noticeContainer.addValidationNotice( + new TripDistanceExceedsShapeDistanceWarningNotice( + trip.tripId(), shapeId, maxStopTimeDist, maxShapeDist, distanceInMeters)); } }); } - /** The distance traveled by a trip should be less or equal to the max length of its shape. */ + /** + * The distance traveled by a trip should be less or equal to the max length of its shape. + * + *

The distance is greater or equal to the 1.11m threshold. + */ @GtfsValidationNotice( severity = ERROR, files = @FileRefs({GtfsTrip.class, GtfsStopTime.class, GtfsShape.class})) @@ -96,15 +119,59 @@ static class TripDistanceExceedsShapeDistanceNotice extends ValidationNotice { /** The faulty record's shape max distance traveled. */ private final double maxShapeDistanceTraveled; + /** The distance in meters between the shape and the stop. */ + private final double distance; + TripDistanceExceedsShapeDistanceNotice( String tripId, String shapeId, double maxTripDistanceTraveled, - double maxShapeDistanceTraveled) { + double maxShapeDistanceTraveled, + double distance) { + this.tripId = tripId; + this.shapeId = shapeId; + this.maxShapeDistanceTraveled = maxShapeDistanceTraveled; + this.maxTripDistanceTraveled = maxTripDistanceTraveled; + this.distance = distance; + } + } + + /** + * The distance traveled by a trip should be less or equal to the max length of its shape. + * + *

The distance is less than the 1.11m threshold. + */ + @GtfsValidationNotice( + severity = WARNING, + files = @FileRefs({GtfsTrip.class, GtfsStopTime.class, GtfsShape.class})) + static class TripDistanceExceedsShapeDistanceWarningNotice extends ValidationNotice { + + /** The faulty record's trip id. */ + private final String tripId; + + /** The faulty record's shape id. */ + private final String shapeId; + + /** The faulty record's trip max distance traveled. */ + private final double maxTripDistanceTraveled; + + /** The faulty record's shape max distance traveled. */ + private final double maxShapeDistanceTraveled; + + /** The distance in meters between the shape and the stop. */ + private final double distance; + + TripDistanceExceedsShapeDistanceWarningNotice( + String tripId, + String shapeId, + double maxTripDistanceTraveled, + double maxShapeDistanceTraveled, + double distance) { this.tripId = tripId; this.shapeId = shapeId; this.maxShapeDistanceTraveled = maxShapeDistanceTraveled; this.maxTripDistanceTraveled = maxTripDistanceTraveled; + this.distance = distance; } } } diff --git a/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/NoticeFieldsTest.java b/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/NoticeFieldsTest.java index 6ed212c10e..0cba2d1469 100644 --- a/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/NoticeFieldsTest.java +++ b/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/NoticeFieldsTest.java @@ -194,6 +194,7 @@ public void testNoticeClassFieldNames() { "value", "maxShapeDistanceTraveled", "maxTripDistanceTraveled", + "distance", "fileNameA", "fileNameB"); } diff --git a/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/TripAndShapeDistanceValidatorTest.java b/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/TripAndShapeDistanceValidatorTest.java index 03d786f92e..39e005dc9f 100644 --- a/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/TripAndShapeDistanceValidatorTest.java +++ b/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/TripAndShapeDistanceValidatorTest.java @@ -29,15 +29,16 @@ private static List createTripTable(int rows) { return trips; } - private static List createShapeTable(int rows, double shapeDistTraveled) { + private static List createShapeTable( + int rows, double shapeDistTraveled, double lonLat) { ArrayList shapes = new ArrayList<>(); for (int i = 0; i < rows; i++) { shapes.add( new GtfsShape.Builder() .setCsvRowNumber(i + 1) .setShapeId("s" + i) - .setShapePtLat(1.0) - .setShapePtLon(1.0) + .setShapePtLat(lonLat) + .setShapePtLon(lonLat) .setShapePtSequence(0) .setShapeDistTraveled(shapeDistTraveled + i) .build()); @@ -60,12 +61,30 @@ private static List createStopTimesTable(int rows, double shapeDis return stopTimes; } + private static List createStopTable(int rows) { + ArrayList stops = new ArrayList<>(); + for (int i = 0; i < rows; i++) { + stops.add( + new GtfsStop.Builder() + .setCsvRowNumber(i + 1) + .setStopId("st" + i) + .setStopLat(0.0) + .setStopLon(0.0) + .build()); + } + return stops; + } + private static List generateNotices( - List trips, List stopTimes, List shapes) { + List trips, + List stopTimes, + List shapes, + List stops) { NoticeContainer noticeContainer = new NoticeContainer(); new TripAndShapeDistanceValidator( GtfsTripTableContainer.forEntities(trips, noticeContainer), GtfsStopTimeTableContainer.forEntities(stopTimes, noticeContainer), + GtfsStopTableContainer.forEntities(stops, noticeContainer), GtfsShapeTableContainer.forEntities(shapes, noticeContainer)) .validate(noticeContainer); return noticeContainer.getValidationNotices(); @@ -73,25 +92,38 @@ private static List generateNotices( @Test public void testTripDistanceExceedsShapeDistance() { - assertThat( - generateNotices( - createTripTable(1), createStopTimesTable(1, 10.0), createShapeTable(1, 9.0))) - .isNotEmpty(); - } - - @Test - public void testValidTripVsShapeDistance1() { - assertThat( - generateNotices( - createTripTable(1), createStopTimesTable(1, 10.0), createShapeTable(1, 10.0))) - .isEmpty(); + List notices = + generateNotices( + createTripTable(2), + createStopTimesTable(1, 10.0), + createShapeTable(1, 9.0, 10.0), + createStopTable(1)); + boolean found = + notices.stream() + .anyMatch( + notice -> + notice + instanceof + TripAndShapeDistanceValidator.TripDistanceExceedsShapeDistanceNotice); + assertThat(found).isTrue(); } @Test - public void testValidTripVsShapeDistance2() { - assertThat( - generateNotices( - createTripTable(1), createStopTimesTable(1, 9.0), createShapeTable(1, 10.0))) - .isEmpty(); + public void testTripDistanceExceedsShapeDistanceWarning() { + List notices = + generateNotices( + createTripTable(2), + createStopTimesTable(1, 10.0), + createShapeTable(1, 9.0, 0.000001), + createStopTable(1)); + boolean found = + notices.stream() + .anyMatch( + notice -> + notice + instanceof + TripAndShapeDistanceValidator + .TripDistanceExceedsShapeDistanceWarningNotice); + assertThat(found).isTrue(); } } From faf3fefe9d3b478d9df804866d77a7416d87d0ee Mon Sep 17 00:00:00 2001 From: cka-y Date: Mon, 19 Feb 2024 16:12:24 -0500 Subject: [PATCH 5/7] fix: validator error --- .../TripAndShapeDistanceValidator.java | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/TripAndShapeDistanceValidator.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/TripAndShapeDistanceValidator.java index 424dbb98dc..a009eb3209 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/TripAndShapeDistanceValidator.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/TripAndShapeDistanceValidator.java @@ -85,14 +85,16 @@ public void validate(NoticeContainer noticeContainer) { double maxShapeDist = maxShape.shapeDistTraveled(); double distanceInMeters = getDistanceMeters(maxShape.shapePtLatLon(), stop.stopLatLon()); - if (distanceInMeters > DISTANCE_THRESHOLD) { - noticeContainer.addValidationNotice( - new TripDistanceExceedsShapeDistanceNotice( - trip.tripId(), shapeId, maxStopTimeDist, maxShapeDist, distanceInMeters)); - } else if (distanceInMeters > 0) { - noticeContainer.addValidationNotice( - new TripDistanceExceedsShapeDistanceWarningNotice( - trip.tripId(), shapeId, maxStopTimeDist, maxShapeDist, distanceInMeters)); + if (maxStopTimeDist > maxShapeDist) { + if (distanceInMeters > DISTANCE_THRESHOLD) { + noticeContainer.addValidationNotice( + new TripDistanceExceedsShapeDistanceNotice( + trip.tripId(), shapeId, maxStopTimeDist, maxShapeDist, distanceInMeters)); + } else if (distanceInMeters > 0) { + noticeContainer.addValidationNotice( + new TripDistanceExceedsShapeDistanceWarningNotice( + trip.tripId(), shapeId, maxStopTimeDist, maxShapeDist, distanceInMeters)); + } } }); } From ede62bee049ee7e35df1edad9e34e8bcc086e212 Mon Sep 17 00:00:00 2001 From: cka-y Date: Tue, 5 Mar 2024 18:07:52 -0500 Subject: [PATCH 6/7] fix: updated threshold --- .../validator/TripAndShapeDistanceValidator.java | 12 ++++++------ .../validator/TripAndShapeDistanceValidatorTest.java | 3 +-- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/TripAndShapeDistanceValidator.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/TripAndShapeDistanceValidator.java index a009eb3209..1cfb11f011 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/TripAndShapeDistanceValidator.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/TripAndShapeDistanceValidator.java @@ -39,7 +39,7 @@ public class TripAndShapeDistanceValidator extends FileValidator { private final GtfsStopTimeTableContainer stopTimeTable; private final GtfsStopTableContainer stopTable; private final GtfsShapeTableContainer shapeTable; - private final double DISTANCE_THRESHOLD = 1.11; // distance in meters + private final double DISTANCE_THRESHOLD = 11.1; // distance in meters @Inject TripAndShapeDistanceValidator( @@ -92,7 +92,7 @@ public void validate(NoticeContainer noticeContainer) { trip.tripId(), shapeId, maxStopTimeDist, maxShapeDist, distanceInMeters)); } else if (distanceInMeters > 0) { noticeContainer.addValidationNotice( - new TripDistanceExceedsShapeDistanceWarningNotice( + new TripDistanceExceedsShapeDistanceBellowThresholdNotice( trip.tripId(), shapeId, maxStopTimeDist, maxShapeDist, distanceInMeters)); } } @@ -102,7 +102,7 @@ public void validate(NoticeContainer noticeContainer) { /** * The distance traveled by a trip should be less or equal to the max length of its shape. * - *

The distance is greater or equal to the 1.11m threshold. + *

The distance is greater or equal to the 11.1m threshold. */ @GtfsValidationNotice( severity = ERROR, @@ -141,12 +141,12 @@ static class TripDistanceExceedsShapeDistanceNotice extends ValidationNotice { /** * The distance traveled by a trip should be less or equal to the max length of its shape. * - *

The distance is less than the 1.11m threshold. + *

The distance is less than the 11.1m threshold. */ @GtfsValidationNotice( severity = WARNING, files = @FileRefs({GtfsTrip.class, GtfsStopTime.class, GtfsShape.class})) - static class TripDistanceExceedsShapeDistanceWarningNotice extends ValidationNotice { + static class TripDistanceExceedsShapeDistanceBellowThresholdNotice extends ValidationNotice { /** The faulty record's trip id. */ private final String tripId; @@ -163,7 +163,7 @@ static class TripDistanceExceedsShapeDistanceWarningNotice extends ValidationNot /** The distance in meters between the shape and the stop. */ private final double distance; - TripDistanceExceedsShapeDistanceWarningNotice( + TripDistanceExceedsShapeDistanceBellowThresholdNotice( String tripId, String shapeId, double maxTripDistanceTraveled, diff --git a/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/TripAndShapeDistanceValidatorTest.java b/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/TripAndShapeDistanceValidatorTest.java index 39e005dc9f..cd9d8fc10b 100644 --- a/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/TripAndShapeDistanceValidatorTest.java +++ b/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/TripAndShapeDistanceValidatorTest.java @@ -122,8 +122,7 @@ public void testTripDistanceExceedsShapeDistanceWarning() { notice -> notice instanceof - TripAndShapeDistanceValidator - .TripDistanceExceedsShapeDistanceWarningNotice); + TripAndShapeDistanceValidator.TripDistanceExceedsShapeDistanceBellowThresholdNotice); assertThat(found).isTrue(); } } From 8b5e21ef429568a76aa6e7c0bad558d13d8f6d19 Mon Sep 17 00:00:00 2001 From: cka-y Date: Tue, 5 Mar 2024 18:10:33 -0500 Subject: [PATCH 7/7] fix: JF --- .../validator/TripAndShapeDistanceValidatorTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/TripAndShapeDistanceValidatorTest.java b/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/TripAndShapeDistanceValidatorTest.java index cd9d8fc10b..a3c8c3936d 100644 --- a/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/TripAndShapeDistanceValidatorTest.java +++ b/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/TripAndShapeDistanceValidatorTest.java @@ -122,7 +122,8 @@ public void testTripDistanceExceedsShapeDistanceWarning() { notice -> notice instanceof - TripAndShapeDistanceValidator.TripDistanceExceedsShapeDistanceBellowThresholdNotice); + TripAndShapeDistanceValidator + .TripDistanceExceedsShapeDistanceBellowThresholdNotice); assertThat(found).isTrue(); } }