Merge e1ea3f1 into ada0b25

MobilityData · Aug 29, 2024 · f63a52a · f63a52a
2 parents ada0b25 + e1ea3f1
commit f63a52a
Show file tree

Hide file tree

Showing 42 changed files with 351 additions and 477 deletions.
diff --git a/.gitignore b/.gitignore
@@ -104,3 +104,4 @@ app/pkg/bin/
 processor/notices/bin/
 processor/notices/tests/bin/
 web/service/bin/
+/web/service/execution_result.json
diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/AnyTableLoader.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/AnyTableLoader.java
@@ -61,22 +61,19 @@ public static GtfsTableContainer load(
       csvFile = new CsvFile(csvInputStream, gtfsFilename, settings);
     } catch (TextParsingException e) {
       noticeContainer.addValidationNotice(new CsvParsingFailedNotice(gtfsFilename, e));
-      return tableDescriptor.createContainerForInvalidStatus(
-          GtfsTableContainer.TableStatus.INVALID_HEADERS);
+      return tableDescriptor.createContainerForInvalidStatus(TableStatus.INVALID_HEADERS);
     }
     if (csvFile.isEmpty()) {
       noticeContainer.addValidationNotice(new EmptyFileNotice(gtfsFilename));
-      return tableDescriptor.createContainerForInvalidStatus(
-          GtfsTableContainer.TableStatus.EMPTY_FILE);
+      return tableDescriptor.createContainerForInvalidStatus(TableStatus.EMPTY_FILE);
     }
     final CsvHeader header = csvFile.getHeader();
     final ImmutableList<GtfsColumnDescriptor> columnDescriptors = tableDescriptor.getColumns();
     final NoticeContainer headerNotices =
         validateHeaders(validatorProvider, gtfsFilename, header, columnDescriptors);
     noticeContainer.addAll(headerNotices);
     if (headerNotices.hasValidationErrors()) {
-      return tableDescriptor.createContainerForInvalidStatus(
-          GtfsTableContainer.TableStatus.INVALID_HEADERS);
+      return tableDescriptor.createContainerForInvalidStatus(TableStatus.INVALID_HEADERS);
     }
     final int nColumns = columnDescriptors.size();
     final ImmutableMap<String, GtfsFieldLoader> fieldLoadersMap = tableDescriptor.getFieldLoaders();
@@ -133,15 +130,13 @@ public static GtfsTableContainer load(
       }
     } catch (TextParsingException e) {
       noticeContainer.addValidationNotice(new CsvParsingFailedNotice(gtfsFilename, e));
-      return tableDescriptor.createContainerForInvalidStatus(
-          GtfsTableContainer.TableStatus.UNPARSABLE_ROWS);
+      return tableDescriptor.createContainerForInvalidStatus(TableStatus.UNPARSABLE_ROWS);
     } finally {
       logFieldCacheStats(gtfsFilename, fieldCaches, columnDescriptors);
     }
     if (hasUnparsableRows) {
       logger.atSevere().log("Failed to parse some rows in %s", gtfsFilename);
-      return tableDescriptor.createContainerForInvalidStatus(
-          GtfsTableContainer.TableStatus.UNPARSABLE_ROWS);
+      return tableDescriptor.createContainerForInvalidStatus(TableStatus.UNPARSABLE_ROWS);
     }
     GtfsTableContainer table =
         tableDescriptor.createContainerForHeaderAndEntities(header, entities, noticeContainer);
@@ -197,14 +192,12 @@ private static void logFieldCacheStats(
     }
   }
 
-  public static GtfsTableContainer loadMissingFile(
-      GtfsTableDescriptor tableDescriptor,
+  public static GtfsContainer loadMissingFile(
+      GtfsDescriptor tableDescriptor,
       ValidatorProvider validatorProvider,
       NoticeContainer noticeContainer) {
     String gtfsFilename = tableDescriptor.gtfsFilename();
-    GtfsTableContainer table =
-        tableDescriptor.createContainerForInvalidStatus(
-            GtfsTableContainer.TableStatus.MISSING_FILE);
+    GtfsContainer table = tableDescriptor.createContainerForInvalidStatus(TableStatus.MISSING_FILE);
     if (tableDescriptor.isRecommended()) {
       noticeContainer.addValidationNotice(new MissingRecommendedFileNotice(gtfsFilename));
     }

diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsContainer.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsContainer.java
@@ -0,0 +1,50 @@
+package org.mobilitydata.gtfsvalidator.table;
+
+import java.util.List;
+import java.util.Optional;
+
+public abstract class GtfsContainer<T extends GtfsEntity, D extends GtfsDescriptor> {
+
+  private final D descriptor;
+  private final TableStatus tableStatus;
+
+  public GtfsContainer(D descriptor, TableStatus tableStatus) {
+    this.tableStatus = tableStatus;
+    this.descriptor = descriptor;
+  }
+
+  public TableStatus getTableStatus() {
+    return tableStatus;
+  }
+
+  public D getDescriptor() {
+    return descriptor;
+  }
+
+  public abstract Class<T> getEntityClass();
+
+  public int entityCount() {
+    return getEntities().size();
+  }
+
+  public abstract List<T> getEntities();
+
+  public abstract String gtfsFilename();
+
+  public abstract Optional<T> byTranslationKey(String recordId, String recordSubId);
+
+  public boolean isMissingFile() {
+    return tableStatus == TableStatus.MISSING_FILE;
+  }
+
+  public boolean isParsedSuccessfully() {
+    switch (tableStatus) {
+      case PARSABLE_HEADERS_AND_ROWS:
+        return true;
+      case MISSING_FILE:
+        return !descriptor.isRequired();
+      default:
+        return false;
+    }
+  }
+}
diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsDescriptor.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsDescriptor.java
@@ -0,0 +1,27 @@
+package org.mobilitydata.gtfsvalidator.table;
+
+// TODO: review class name maybe GtfsFileDescriptor
+public abstract class GtfsDescriptor<T extends GtfsEntity> {
+
+  public abstract <C extends GtfsContainer> C createContainerForInvalidStatus(
+      TableStatus tableStatus);
+
+  // True if the specified file is required in a feed.
+  private boolean required;
+
+  private TableStatus tableStatus;
+
+  public abstract boolean isRecommended();
+
+  public abstract Class<T> getEntityClass();
+
+  public abstract String gtfsFilename();
+
+  public boolean isRequired() {
+    return this.required;
+  }
+
+  public void setRequired(boolean required) {
+    this.required = required;
+  }
+}
diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedContainer.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedContainer.java
@@ -18,20 +18,19 @@
 
 import com.google.common.base.Ascii;
 import java.util.*;
-import org.mobilitydata.gtfsvalidator.table.GtfsTableContainer.TableStatus;
 
 /**
  * Container for a whole parsed GTFS feed with all its tables.
  *
- * <p>The tables are kept as {@code GtfsTableContainer} instances.
+ * <p>The tables are kept as {@code GtfsContainer} instances.
  */
 public class GtfsFeedContainer {
-  private final Map<String, GtfsTableContainer<?>> tables = new HashMap<>();
-  private final Map<Class<? extends GtfsTableContainer>, GtfsTableContainer<?>> tablesByClass =
+  private final Map<String, GtfsContainer<?, ?>> tables = new HashMap<>();
+  private final Map<Class<? extends GtfsContainer>, GtfsContainer<?, ?>> tablesByClass =
       new HashMap<>();
 
-  public GtfsFeedContainer(List<GtfsTableContainer<?>> tableContainerList) {
-    for (GtfsTableContainer<?> table : tableContainerList) {
+  public GtfsFeedContainer(List<GtfsContainer<?, ?>> tableContainerList) {
+    for (GtfsContainer<?, ?> table : tableContainerList) {
       tables.put(table.gtfsFilename(), table);
       tablesByClass.put(table.getClass(), table);
     }
@@ -49,11 +48,12 @@ public GtfsFeedContainer(List<GtfsTableContainer<?>> tableContainerList) {
    * @param filename file name, including ".txt" extension
    * @return GTFS table or empty if the table is not supported by schema
    */
-  public Optional<GtfsTableContainer<?>> getTableForFilename(String filename) {
-    return Optional.ofNullable(tables.getOrDefault(Ascii.toLowerCase(filename), null));
+  public <T extends GtfsContainer<?, ?>> Optional<T> getTableForFilename(String filename) {
+    return (Optional<T>)
+        Optional.ofNullable(tables.getOrDefault(Ascii.toLowerCase(filename), null));
   }
 
-  public <T extends GtfsTableContainer<?>> T getTable(Class<T> clazz) {
+  public <T extends GtfsContainer<?, ?>> T getTable(Class<T> clazz) {
     return (T) tablesByClass.get(clazz);
   }
 
@@ -65,21 +65,21 @@ public <T extends GtfsTableContainer<?>> T getTable(Class<T> clazz) {
    * @return true if all files were successfully parsed, false otherwise
    */
   public boolean isParsedSuccessfully() {
-    for (GtfsTableContainer<?> table : tables.values()) {
+    for (GtfsContainer<?, ?> table : tables.values()) {
       if (!table.isParsedSuccessfully()) {
         return false;
       }
     }
     return true;
   }
 
-  public Collection<GtfsTableContainer<?>> getTables() {
+  public Collection<GtfsContainer<?, ?>> getTables() {
     return tables.values();
   }
 
   public String tableTotalsText() {
     List<String> totalList = new ArrayList<>();
-    for (GtfsTableContainer<?> table : tables.values()) {
+    for (GtfsContainer<?, ?> table : tables.values()) {
       if (table.getTableStatus() == TableStatus.MISSING_FILE
           && !table.getDescriptor().isRequired()) {
         continue;

diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedLoader.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedLoader.java
@@ -19,6 +19,7 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.flogger.FluentLogger;
 import java.io.InputStream;
+import java.lang.reflect.Modifier;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -46,7 +47,7 @@
  */
 public class GtfsFeedLoader {
   private static final FluentLogger logger = FluentLogger.forEnclosingClass();
-  private final HashMap<String, GtfsTableDescriptor<?>> tableDescriptors = new HashMap<>();
+  private final HashMap<String, GtfsDescriptor<?>> tableDescriptors = new HashMap<>();
   private int numThreads = 1;
 
   /**
@@ -56,12 +57,15 @@ public class GtfsFeedLoader {
   private final List<Class<? extends FileValidator>> multiFileValidatorsWithParsingErrors =
       new ArrayList<>();
 
-  public GtfsFeedLoader(
-      ImmutableList<Class<? extends GtfsTableDescriptor<?>>> tableDescriptorClasses) {
-    for (Class<? extends GtfsTableDescriptor<?>> clazz : tableDescriptorClasses) {
-      GtfsTableDescriptor<?> descriptor;
+  public GtfsFeedLoader(ImmutableList<Class<? extends GtfsDescriptor<?>>> tableDescriptorClasses) {
+    for (Class<? extends GtfsDescriptor<?>> clazz : tableDescriptorClasses) {
+      GtfsDescriptor<?> descriptor;
       try {
-        descriptor = clazz.asSubclass(GtfsTableDescriptor.class).getConstructor().newInstance();
+        //        Skipping abstract classes. Example: GtfsTableDescriptor.
+        if (Modifier.isAbstract(clazz.getModifiers())) {
+          continue;
+        }
+        descriptor = clazz.asSubclass(GtfsDescriptor.class).getConstructor().newInstance();
       } catch (ReflectiveOperationException e) {
         logger.atSevere().withCause(e).log(
             "Possible bug in GTFS annotation processor: expected a constructor without parameters"
@@ -73,7 +77,7 @@ public GtfsFeedLoader(
     }
   }
 
-  public Collection<GtfsTableDescriptor<?>> getTableDescriptors() {
+  public Collection<GtfsDescriptor<?>> getTableDescriptors() {
     return Collections.unmodifiableCollection(tableDescriptors.values());
   }
 
@@ -100,19 +104,36 @@ public GtfsFeedContainer loadAndValidate(
     Map<String, GtfsTableDescriptor<?>> remainingDescriptors =
         (Map<String, GtfsTableDescriptor<?>>) tableDescriptors.clone();
     for (String filename : gtfsInput.getFilenames()) {
-      GtfsTableDescriptor<?> tableDescriptor = remainingDescriptors.remove(filename.toLowerCase());
+      GtfsDescriptor<?> tableDescriptor = remainingDescriptors.remove(filename.toLowerCase());
       if (tableDescriptor == null) {
         noticeContainer.addValidationNotice(new UnknownFileNotice(filename));
       } else {
         loaderCallables.add(
             () -> {
               NoticeContainer loaderNotices = new NoticeContainer();
-              GtfsTableContainer<?> tableContainer;
+              GtfsContainer<?, ?> tableContainer;
               try (InputStream inputStream = gtfsInput.getFile(filename)) {
                 try {
-                  tableContainer =
-                      AnyTableLoader.load(
-                          tableDescriptor, validatorProvider, inputStream, loaderNotices);
+                  if (tableDescriptor instanceof GtfsTableDescriptor) {
+                    tableContainer =
+                        AnyTableLoader.load(
+                            (GtfsTableDescriptor) tableDescriptor,
+                            validatorProvider,
+                            inputStream,
+                            loaderNotices);
+                  } else if (tableDescriptor instanceof GtfsJsonDescriptor) {
+                    tableContainer =
+                        JsonFileLoader.load(
+                            (GtfsJsonDescriptor) tableDescriptor,
+                            validatorProvider,
+                            inputStream,
+                            loaderNotices);
+                  } else {
+                    logger.atSevere().log(
+                        "Runtime exception table descriptor not supported: %s",
+                        tableDescriptor.getClass().getName());
+                    throw new RuntimeException("Table descriptor is not a supported type");
+                  }
                 } catch (RuntimeException e) {
                   // This handler should prevent ExecutionException for
                   // this thread. We catch an exception here for storing
@@ -130,11 +151,10 @@ public GtfsFeedContainer loadAndValidate(
             });
       }
     }
-    ArrayList<GtfsTableContainer<?>> tableContainers = new ArrayList<>();
+    ArrayList<GtfsContainer<?, ?>> tableContainers = new ArrayList<>();
     tableContainers.ensureCapacity(tableDescriptors.size());
-    for (GtfsTableDescriptor<?> tableDescriptor : remainingDescriptors.values()) {
-      tableContainers.add(
-          AnyTableLoader.loadMissingFile(tableDescriptor, validatorProvider, noticeContainer));
+    for (GtfsDescriptor<?> tableDescriptor : remainingDescriptors.values()) {
+      tableContainers.add(AnyTableLoader.loadMissingFile(tableDescriptor, validatorProvider, noticeContainer));
     }
     try {
       for (Future<TableAndNoticeContainers> futureContainer : exec.invokeAll(loaderCallables)) {
@@ -186,11 +206,10 @@ private static void addThreadExecutionError(
   }
 
   static class TableAndNoticeContainers {
-    final GtfsTableContainer tableContainer;
+    final GtfsContainer tableContainer;
     final NoticeContainer noticeContainer;
 
-    public TableAndNoticeContainers(
-        GtfsTableContainer tableContainer, NoticeContainer noticeContainer) {
+    public TableAndNoticeContainers(GtfsContainer tableContainer, NoticeContainer noticeContainer) {
       this.tableContainer = tableContainer;
       this.noticeContainer = noticeContainer;
     }

diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsJsonContainer.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsJsonContainer.java
@@ -0,0 +1,9 @@
+package org.mobilitydata.gtfsvalidator.table;
+
+public abstract class GtfsJsonContainer<T extends GtfsEntity, D extends GtfsDescriptor<T>>
+    extends GtfsContainer<T, D> {
+
+  public GtfsJsonContainer(D descriptor, TableStatus tableStatus) {
+    super(descriptor, tableStatus);
+  }
+}
diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsJsonDescriptor.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsJsonDescriptor.java
@@ -0,0 +1,10 @@
+package org.mobilitydata.gtfsvalidator.table;
+
+import java.util.List;
+import org.mobilitydata.gtfsvalidator.notice.NoticeContainer;
+
+public abstract class GtfsJsonDescriptor<T extends GtfsEntity> extends GtfsDescriptor<T> {
+
+  public abstract GtfsJsonContainer createContainerForEntities(
+      List<T> entities, NoticeContainer noticeContainer);
+}