Merge pull request #90 from dtemkin-volpe/develop

Add int/string id databases

.github/workflows/generate.yaml

@@ -40,10 +40,6 @@ jobs:
         run: pip install -r ${{env.SCRIPTS_FOLDER}}/requirements.txt
         if: steps.getDiff.outputs.diff
 
-      - name: Remove previous files
-        if: steps.getDiff.outputs.diff
-        run: rm ${{env.DOCS_FOLDER}}/*
-
       - name: Run schema generator
         if: steps.getDiff.outputs.diff
         run: python ${{env.SCRIPTS_FOLDER}}/generate.py

.github/workflows/website.yaml

@@ -2,7 +2,7 @@ name: Deploy Documentation Website
 
 on: 
   push:
-    branches: ["main", "develop"]
+    branches: ["main"]
   workflow_dispatch: {}
 
 permissions:

examples/Arlington_Signals/signal_timing_plan.csv

@@ -3,4 +3,3 @@ timing_plan_id,controller_id,time_day,time_day_id,cycle_length,opt_comment
 1,6,01111100_06:00_09:00,,120,M-F 6-9
 2,6,01111100_15:00_19:00,,120,M-F 15-19
 3,6,000000100_11:00_18:00,,110,Sat 11-18
-

scripts/examples.py

@@ -5,7 +5,8 @@
 
 # List of tuples with the database name and path to the relevant example folder
 EXAMPLE_PATHS: list[tuple[str, Path]] = [
-    ("cambridge", EXAMPLES_PATH / "Cambridge_Intersection"),
+    ("cambridge_intersection", EXAMPLES_PATH / "Cambridge_Intersection"),
+    ("arlington_signals", EXAMPLES_PATH / "Arlington_Signals"),
 ]
 
 # Template files that should be present in every example

scripts/generate.py

@@ -1,14 +1,29 @@
-from shared import DOCS_PATH, gmns
+import shutil
+
+from frictionless import Field
+from shared import DB_PATH, DOCS_PATH, check_if_id, gmns
 
 if __name__ == "__main__":
-    # with open(DOCS_PATH / "README.md") as old_file:
-    #     old_docs = old_file.read().split("\n")
+    # Get old docs
+    with open(DOCS_PATH / "README.md") as old_file:
+        old_docs = "".join(old_file.read().split("\n"))
+
+    # Delete old doc files
+    shutil.rmtree(DOCS_PATH)
+
+    new_docs = "".join(gmns.generate_docs().split("\n"))
+
+    if old_docs != new_docs:
+        # Change in schema detected! Generating new database...
+        # gmns.generate_db()
 
-    new_docs = gmns.generate_docs()
+        # Make integer ID database
+        gmns.edit_spec_types(check_if_id, "integer")
+        gmns.generate_db(base_path=DB_PATH / "integer_ids")
 
-    # if old_docs == new_docs:
-    # Change in schema detected! Generating new database...
-    gmns.generate_db()
+        # Make string ID database
+        gmns.edit_spec_types(check_if_id, "string")
+        gmns.generate_db(base_path=DB_PATH / "string_ids")
 
     # with open(SCRIPT_PATH / "datapackage.json", "w+") as file:
     #     json.dump(gmns.json_data, file)

scripts/shared.py

@@ -1,3 +1,4 @@
+import csv
 import json
 import os
 import pathlib
@@ -6,8 +7,9 @@
 from os import listdir
 from os.path import isfile
 from pathlib import Path
+from typing import Callable
 
-from frictionless import Package
+from frictionless import Field, Package
 
 SCRIPT_PATH = pathlib.Path(__file__).parent.resolve()
 SPECS_PATH = SCRIPT_PATH / ".." / "spec"
@@ -16,6 +18,10 @@
 EXAMPLES_PATH = SCRIPT_PATH / ".." / "examples"
 
 
+def check_if_id(field: Field):
+    return "_id" in field.name
+
+
 class GMNS:
     """GMNS class that can be reused throughout different scripts"""
 
@@ -156,11 +162,37 @@ def prep_examples(
                         path_of_file_to_copy, example_path / name_of_file_to_copy
                     )
                     files_to_delete.append(example_path / name_of_file_to_copy)
+
+            id_type = "any"
+            try:
+                with open(example_path / "config.csv", "r") as config_file:
+                    config_found = True
+                    config = csv.DictReader(config_file)
+                    for row in config:
+                        id_type = row["id_type"]
+            except:
+                config_found = False
+
+            if config_found:
+                self.edit_spec_types(check_if_id, id_type)
+
             self.generate_db(example_path, example_name, False)
 
+            if config_found:
+                self.edit_spec_types(check_if_id, "any")
+
         for file_to_delete in files_to_delete:
             os.remove(file_to_delete)
 
+    def edit_spec_types(
+        self, check_field: Callable[[Field], bool], change_type_to: str
+    ):
+        for resource in self._package.resources:
+            schema = resource.schema
+            for field in schema.fields:
+                if check_field(field):
+                    schema.set_field_type(field.name, change_type_to)
+
     def validate_example(
         self,
         example_path: Path,
@@ -206,3 +238,4 @@ def validate_example(
 
 
 gmns = GMNS()
+gmns = GMNS()

usage/database/integer_ids/curb_seg.sql

@@ -0,0 +1,12 @@
+CREATE TABLE IF NOT EXISTS curb_seg (
+	curb_seg_id INTEGER NOT NULL, 
+	link_id INTEGER NOT NULL, 
+	ref_node_id INTEGER NOT NULL, 
+	start_lr FLOAT NOT NULL CHECK (start_lr >= 0), 
+	end_lr FLOAT NOT NULL CHECK (end_lr >= 0), 
+	regulation TEXT, 
+	width FLOAT CHECK (width >= 0), 
+	PRIMARY KEY (curb_seg_id), 
+	FOREIGN KEY(link_id) REFERENCES link (link_id), 
+	FOREIGN KEY(ref_node_id) REFERENCES node (node_id)
+)

usage/database/integer_ids/geometry.sql

@@ -0,0 +1,5 @@
+CREATE TABLE IF NOT EXISTS geometry (
+	geometry_id INTEGER NOT NULL, 
+	geometry TEXT, 
+	PRIMARY KEY (geometry_id)
+)

usage/database/integer_ids/lane.sql

@@ -0,0 +1,11 @@
+CREATE TABLE IF NOT EXISTS lane (
+	lane_id INTEGER NOT NULL, 
+	link_id INTEGER NOT NULL, 
+	lane_num INTEGER NOT NULL CHECK (lane_num >= -10) CHECK (lane_num <= 10), 
+	allowed_uses TEXT, 
+	r_barrier TEXT, 
+	l_barrier TEXT, 
+	width FLOAT CHECK (width >= 0), 
+	PRIMARY KEY (lane_id), 
+	FOREIGN KEY(link_id) REFERENCES link (link_id)
+)

usage/database/integer_ids/lane_tod.sql

@@ -0,0 +1,14 @@
+CREATE TABLE IF NOT EXISTS lane_tod (
+	lane_tod_id INTEGER NOT NULL, 
+	lane_id INTEGER NOT NULL, 
+	timeday_id INTEGER, 
+	time_day TEXT, 
+	lane_num INTEGER NOT NULL CHECK (lane_num <= 10) CHECK (lane_num >= -10), 
+	allowed_uses TEXT, 
+	r_barrier TEXT, 
+	l_barrier TEXT, 
+	width FLOAT CHECK (width >= 0), 
+	PRIMARY KEY (lane_tod_id), 
+	FOREIGN KEY(timeday_id) REFERENCES time_set_definitions (timeday_id), 
+	FOREIGN KEY(lane_id) REFERENCES lane (lane_id)
+)

usage/database/integer_ids/link.sql

@@ -0,0 +1,29 @@
+CREATE TABLE IF NOT EXISTS link (
+	link_id INTEGER NOT NULL, 
+	name TEXT, 
+	from_node_id INTEGER NOT NULL, 
+	to_node_id INTEGER NOT NULL, 
+	directed BOOLEAN NOT NULL, 
+	geometry_id INTEGER, 
+	geometry TEXT, 
+	parent_link_id INTEGER, 
+	dir_flag INTEGER, 
+	length FLOAT CHECK (length >= 0), 
+	grade FLOAT CHECK (grade >= -100) CHECK (grade <= 100), 
+	facility_type TEXT, 
+	capacity FLOAT CHECK (capacity >= 0), 
+	free_speed FLOAT CHECK (free_speed <= 200) CHECK (free_speed >= 0), 
+	lanes INTEGER CHECK (lanes >= 0), 
+	bike_facility TEXT, 
+	ped_facility TEXT, 
+	parking TEXT, 
+	allowed_uses TEXT, 
+	toll FLOAT, 
+	jurisdiction TEXT, 
+	row_width FLOAT CHECK (row_width >= 0), 
+	PRIMARY KEY (link_id), 
+	FOREIGN KEY(from_node_id) REFERENCES node (node_id), 
+	FOREIGN KEY(geometry_id) REFERENCES geometry (geometry_id), 
+	FOREIGN KEY(parent_link_id) REFERENCES link (link_id), 
+	FOREIGN KEY(to_node_id) REFERENCES node (node_id)
+)

usage/database/link_tod.sql → usage/database/integer_ids/link_tod.sql

@@ -1,7 +1,7 @@
 CREATE TABLE IF NOT EXISTS link_tod (
-	link_tod_id TEXT NOT NULL, 
-	link_id TEXT NOT NULL, 
-	timeday_id TEXT, 
+	link_tod_id INTEGER NOT NULL, 
+	link_id INTEGER NOT NULL, 
+	timeday_id INTEGER, 
 	time_day TEXT, 
 	capacity FLOAT CHECK (capacity >= 0), 
 	free_speed FLOAT CHECK (free_speed <= 200) CHECK (free_speed >= 0), 

usage/database/integer_ids/location.sql

@@ -0,0 +1,15 @@
+CREATE TABLE IF NOT EXISTS location (
+	loc_id INTEGER NOT NULL, 
+	link_id INTEGER NOT NULL, 
+	ref_node_id INTEGER NOT NULL, 
+	lr FLOAT NOT NULL CHECK (lr >= 0), 
+	x_coord FLOAT, 
+	y_coord FLOAT, 
+	z_coord FLOAT, 
+	loc_type TEXT, 
+	zone_id INTEGER, 
+	gtfs_stop_id INTEGER, 
+	PRIMARY KEY (loc_id), 
+	FOREIGN KEY(link_id) REFERENCES link (link_id), 
+	FOREIGN KEY(ref_node_id) REFERENCES node (node_id)
+)

usage/database/integer_ids/movement.sql

@@ -0,0 +1,22 @@
+CREATE TABLE IF NOT EXISTS movement (
+	mvmt_id INTEGER NOT NULL, 
+	node_id INTEGER NOT NULL, 
+	name TEXT, 
+	ib_link_id INTEGER NOT NULL, 
+	start_ib_lane INTEGER, 
+	end_ib_lane INTEGER, 
+	ob_link_id INTEGER NOT NULL, 
+	start_ob_lane INTEGER, 
+	end_ob_lane INTEGER, 
+	type TEXT NOT NULL, 
+	penalty FLOAT, 
+	capacity FLOAT, 
+	ctrl_type TEXT, 
+	mvmt_code TEXT, 
+	allowed_uses TEXT, 
+	geometry TEXT, 
+	PRIMARY KEY (mvmt_id), 
+	FOREIGN KEY(ob_link_id) REFERENCES link (link_id), 
+	FOREIGN KEY(node_id) REFERENCES node (node_id), 
+	FOREIGN KEY(ib_link_id) REFERENCES link (link_id)
+)

usage/database/integer_ids/movement_tod.sql

@@ -0,0 +1,23 @@
+CREATE TABLE IF NOT EXISTS movement_tod (
+	mvmt_tod_id INTEGER NOT NULL, 
+	mvmt_id INTEGER NOT NULL, 
+	time_day TEXT, 
+	timeday_id INTEGER, 
+	ib_link_id INTEGER NOT NULL, 
+	start_ib_lane INTEGER, 
+	end_ib_lane INTEGER, 
+	ob_link_id INTEGER NOT NULL, 
+	start_ob_lane INTEGER, 
+	end_ob_lane INTEGER, 
+	type TEXT NOT NULL, 
+	penalty FLOAT, 
+	capacity FLOAT, 
+	ctrl_type TEXT, 
+	mvmt_code TEXT, 
+	allowed_uses TEXT, 
+	PRIMARY KEY (mvmt_tod_id), 
+	FOREIGN KEY(mvmt_id) REFERENCES movement (mvmt_id), 
+	FOREIGN KEY(timeday_id) REFERENCES time_set_definitions (timeday_id), 
+	FOREIGN KEY(ib_link_id) REFERENCES link (link_id), 
+	FOREIGN KEY(ob_link_id) REFERENCES link (link_id)
+)

usage/database/integer_ids/node.sql

@@ -0,0 +1,14 @@
+CREATE TABLE IF NOT EXISTS node (
+	node_id INTEGER NOT NULL, 
+	name TEXT, 
+	x_coord FLOAT NOT NULL, 
+	y_coord FLOAT NOT NULL, 
+	z_coord FLOAT, 
+	node_type TEXT, 
+	ctrl_type TEXT, 
+	zone_id INTEGER, 
+	parent_node_id INTEGER, 
+	PRIMARY KEY (node_id), 
+	FOREIGN KEY(zone_id) REFERENCES zone (zone_id), 
+	FOREIGN KEY(parent_node_id) REFERENCES node (node_id)
+)

usage/database/integer_ids/segment.sql

@@ -0,0 +1,23 @@
+CREATE TABLE IF NOT EXISTS segment (
+	segment_id INTEGER NOT NULL, 
+	link_id INTEGER NOT NULL, 
+	ref_node_id INTEGER NOT NULL, 
+	start_lr FLOAT NOT NULL CHECK (start_lr >= 0), 
+	end_lr FLOAT NOT NULL CHECK (end_lr >= 0), 
+	grade FLOAT CHECK (grade <= 100) CHECK (grade >= -100), 
+	capacity FLOAT CHECK (capacity >= 0), 
+	free_speed FLOAT CHECK (free_speed <= 200) CHECK (free_speed >= 0), 
+	lanes INTEGER, 
+	l_lanes_added INTEGER, 
+	r_lanes_added INTEGER, 
+	bike_facility TEXT, 
+	ped_facility TEXT, 
+	parking TEXT, 
+	allowed_uses TEXT, 
+	toll FLOAT, 
+	jurisdiction TEXT, 
+	row_width FLOAT CHECK (row_width >= 0), 
+	PRIMARY KEY (segment_id), 
+	FOREIGN KEY(link_id) REFERENCES link (link_id), 
+	FOREIGN KEY(ref_node_id) REFERENCES node (node_id)
+)

usage/database/integer_ids/segment_lane.sql

@@ -0,0 +1,12 @@
+CREATE TABLE IF NOT EXISTS segment_lane (
+	segment_lane_id INTEGER NOT NULL, 
+	segment_id INTEGER NOT NULL, 
+	lane_num INTEGER NOT NULL CHECK (lane_num <= 10) CHECK (lane_num >= -10), 
+	parent_lane_id INTEGER, 
+	allowed_uses TEXT, 
+	r_barrier TEXT, 
+	l_barrier TEXT, 
+	width FLOAT CHECK (width >= 0), 
+	PRIMARY KEY (segment_lane_id), 
+	FOREIGN KEY(segment_id) REFERENCES segment (segment_id)
+)

usage/database/integer_ids/segment_lane_tod.sql

@@ -0,0 +1,14 @@
+CREATE TABLE IF NOT EXISTS segment_lane_tod (
+	segment_lane_tod_id INTEGER NOT NULL, 
+	segment_lane_id INTEGER NOT NULL, 
+	timeday_id INTEGER, 
+	time_day TEXT, 
+	lane_num INTEGER NOT NULL CHECK (lane_num >= -10) CHECK (lane_num <= 10), 
+	allowed_uses TEXT, 
+	r_barrier TEXT, 
+	l_barrier TEXT, 
+	width FLOAT CHECK (width >= 0), 
+	PRIMARY KEY (segment_lane_tod_id), 
+	FOREIGN KEY(timeday_id) REFERENCES time_set_definitions (timeday_id), 
+	FOREIGN KEY(segment_lane_id) REFERENCES segment_lane (segment_lane_id)
+)

usage/database/integer_ids/segment_tod.sql

@@ -0,0 +1,19 @@
+CREATE TABLE IF NOT EXISTS segment_tod (
+	segment_tod_id INTEGER NOT NULL, 
+	segment_id INTEGER NOT NULL, 
+	timeday_id INTEGER, 
+	time_day TEXT, 
+	capacity FLOAT CHECK (capacity >= 0), 
+	free_speed FLOAT CHECK (free_speed >= 0) CHECK (free_speed <= 200), 
+	lanes INTEGER, 
+	l_lanes_added INTEGER, 
+	r_lanes_added INTEGER, 
+	bike_facility TEXT, 
+	ped_facility TEXT, 
+	parking TEXT, 
+	toll FLOAT, 
+	allowed_uses TEXT, 
+	PRIMARY KEY (segment_tod_id), 
+	FOREIGN KEY(timeday_id) REFERENCES time_set_definitions (timeday_id), 
+	FOREIGN KEY(segment_id) REFERENCES segment (segment_id)
+)

usage/database/integer_ids/signal_controller.sql

@@ -0,0 +1,4 @@
+CREATE TABLE IF NOT EXISTS signal_controller (
+	controller_id INTEGER NOT NULL, 
+	PRIMARY KEY (controller_id)
+)