Merge pull request #77 from dtemkin-volpe/develop

Add auto-generated SQL scripts and SQLite DB

scripts/generate_docs.py

@@ -1,51 +1,83 @@
 import json
+import os
+import sqlite3
 from frictionless import Package, Resource
 import pathlib
 from os import listdir
 from os.path import isfile, join, split
 
+# Define paths to various folders
+script_path = pathlib.Path(__file__).parent.resolve()
+specs_path = script_path / ".." / "spec"
+docs_path = script_path / ".." / "docs" / "spec"
+db_path = script_path / ".." / "usage" / "database"
 
-current_path = pathlib.Path(__file__).parent.resolve()
-specs_path = current_path / ".." / "spec"
-docs_path = current_path / ".." / "docs" / "spec"
+# Get all files in spec_path
+spec_files = [
+    specs_path / f for f in listdir(specs_path) if isfile(join(specs_path, f))
+]
 
-onlyfiles = [specs_path / f for f in listdir(specs_path) if isfile(join(specs_path, f))]
+# Grab general package data
+with open(specs_path / "datapackage.json") as spec_file:
+    json_data = json.load(spec_file)
 
-for file in onlyfiles:
-    if "datapackage" in str(file):
-        with open(file) as f:
-            json_data = json.load(f)
-        for index, resource in enumerate(json_data["resources"]):
-            if type(resource["schema"]) is str:
-                with open((specs_path / resource["schema"])) as r:
-                    schema_data = json.load(r)
+# Combine every schema file into one datapackage
+for index, resource in enumerate(json_data["resources"]):
+    if type(resource["schema"]) is str:
+        with open((specs_path / resource["schema"])) as r:
+            schema_data = json.load(r)
 
-                    json_data["resources"][index]["schema"] = schema_data
+            json_data["resources"][index]["schema"] = schema_data
 
-                    del json_data["resources"][index]["schema"]["name"]
-                    del json_data["resources"][index]["schema"]["description"]
+            # Get rid of duplicate info...
+            del json_data["resources"][index]["schema"]["name"]
+            del json_data["resources"][index]["schema"]["description"]
 
-                    filename = (
-                        json_data["resources"][index]["name"].split(".")[0] + ".md"
-                    )
+    filename = str(json_data["resources"][index]["name"].split(".")[0] + ".md")
 
-                    path = docs_path / filename
+    path = docs_path / filename
 
-                    resource = Resource(json_data["resources"][index])
-                    generated_markdown = resource.to_markdown(
-                        path.absolute().as_posix(),
-                        True,
-                    )
+    # Make documentation files for every schema/resource
+    resource = Resource(json_data["resources"][index])
+    generated_markdown = resource.to_markdown(
+        path.absolute().as_posix(),
+        True,
+    )
 
-                    new_markdown = generated_markdown.replace("  | name", "\n  | name")
+    # Fix formatting to make table display properly
+    new_markdown = generated_markdown.replace("  | name", "\n  | name")
 
-                    with open(path, "w") as file1:
-                        file1.write(new_markdown)
+    with open(path, "w") as resource_doc:
+        resource_doc.write(new_markdown)
 
-        package = Package(json_data)
-        # er_diagram = package.to_er_diagram((docs_path / "diagram.dot").absolute().as_posix())
+# Make package markdown file as README.md for documentation folder
+package = Package(json_data)
+package.to_markdown((docs_path / "README.md").absolute().as_posix())
 
-        # package.image = er_diagram
-        # pprint(package)
+# Create blank CSV files so that we don't get errors when creating the SQLite DB
+files_to_delete = []
+for resource in package.resources:
+    if type(resource.path) is str:
+        file_to_create = resource.path
+        if not os.path.exists(file_to_create):
+            with open(file_to_create, "a+"):
+                files_to_delete.append(file_to_create)
 
-        package.to_markdown((docs_path / "README.md").absolute().as_posix())
+# Use https://alpha.sqliteviewer.app/ for verification!
+os.remove(db_path / "gmns.sqlite")
+create_db = package.publish(
+    "sqlite:///" + (db_path / "gmns.sqlite").absolute().as_posix()
+)
+
+# Get list of every table and its schema
+connection = sqlite3.connect(db_path / "gmns.sqlite")
+for table_name, table_sql in connection.execute("SELECT name, sql FROM sqlite_master WHERE type='table';"):
+    with open(db_path / f"{table_name}.sql", "w") as table_file:
+        # Add "IF NOT EXISTS" condition
+        table_sql = table_sql.replace("CREATE TABLE", "CREATE TABLE IF NOT EXISTS")
+        table_file.write(table_sql)
+connection.close()
+
+# Remove temp files we created earlier
+for file_to_delete in files_to_delete:
+    os.remove(file_to_delete)

scripts/requirements.txt

@@ -1,4 +1,5 @@
 frictionless==5.17.0
 frictionless[pandas]==5.17.0
+frictionless[sql]==5.17.0
 pandas==2.2.2
 tabulate==0.9.0

usage/database/config.sql

@@ -0,0 +1,10 @@
+CREATE TABLE IF NOT EXISTS config (
+	dataset_name TEXT, 
+	short_length TEXT, 
+	long_length TEXT, 
+	speed TEXT, 
+	crs TEXT, 
+	geometry_field_format TEXT, 
+	currency TEXT, 
+	version_number FLOAT
+)

usage/database/curb_seg.sql

@@ -1,12 +1,12 @@
 CREATE TABLE IF NOT EXISTS curb_seg (
-    curb_seg_id VARCHAR(255) NOT NULL,  -- Primary key.
-    link_id VARCHAR(255) NOT NULL,  -- Required. Foreign key to road_links. The link that the segment is located on.
-    ref_node_id VARCHAR(255) NOT NULL,  -- Required. Foreign key to node where distance is 0.
-    start_lr FLOAT NOT NULL,  -- Required. Distance from `ref_node_id` in short_length units.
-    end_lr FLOAT NOT NULL,  -- Required. Distance from `ref_node_id`in short_length units.
-    regulation VARCHAR(255),  -- Optional. Regulation on this curb segment.
-    width FLOAT,  -- Optional. Width (short_length units) of the curb segment.
-    PRIMARY KEY (curb_seg_id),
-    FOREIGN KEY (link_id) REFERENCES link(link_id),
-    FOREIGN KEY (ref_node_id) REFERENCES node(node_id)
-);
+	curb_seg_id TEXT NOT NULL, 
+	link_id TEXT NOT NULL, 
+	ref_node_id TEXT NOT NULL, 
+	start_lr FLOAT NOT NULL CHECK (start_lr >= 0), 
+	end_lr FLOAT NOT NULL CHECK (end_lr >= 0), 
+	regulation TEXT, 
+	width FLOAT CHECK (width >= 0), 
+	PRIMARY KEY (curb_seg_id), 
+	FOREIGN KEY(ref_node_id) REFERENCES node (node_id), 
+	FOREIGN KEY(link_id) REFERENCES link (link_id)
+)

usage/database/geometry.sql

@@ -1,5 +1,5 @@
 CREATE TABLE IF NOT EXISTS geometry (
-    geometry_id VARCHAR(255) NOT NULL,  -- Primary key - could be SharedStreets Geometry ID
-    geometry VARCHAR(255),  -- Link geometry, in well-known text (WKT) format.  Optionally, other formats supported by geopandas (GeoJSON, PostGIS) may be used if specified in geometry_field_format in gmns.spec.json.
-    PRIMARY KEY (geometry_id)
-);
+	geometry_id TEXT NOT NULL, 
+	geometry TEXT, 
+	PRIMARY KEY (geometry_id)
+)

usage/database/lane.sql

@@ -1,11 +1,11 @@
 CREATE TABLE IF NOT EXISTS lane (
-    lane_id VARCHAR(255) NOT NULL,  -- Primary key
-    link_id VARCHAR(255) NOT NULL,  -- Required. Foreign key to link table.
-    lane_num VARCHAR(255) NOT NULL,  -- Required. e.g., -1, 1, 2 (use left-to-right numbering).
-    allowed_uses VARCHAR(255),  -- Optional. Set of allowed uses that should appear in either the use_definition or use_group tables; comma-separated.
-    r_barrier VARCHAR(255),  -- Optional. Whether a barrier exists to prevent vehicles from changing lanes to the right.<br>- `none` (the default). Indicates that a vehicle can change lanes, provided that the vehicle-type is permitted in the destination lane<br>- `regulatory`. There is a regulatory prohibition (e.g., a double-white solid line) against changing lanes, but no physical barrier<br>- `physical`. A physical barrier (e.g., a curb, Jersey barrier) is in place.
-    l_barrier VARCHAR(255),  -- Optional. Whether a barrier exists to prevent vehicles from changing lanes to the right.<br>- `none` (the default). Indicates that a vehicle can change lanes, provided that the vehicle-type is permitted in the destination lane<br>- `regulatory`. There is a regulatory prohibition (e.g., a double-white solid line) against changing lanes, but no physical barrier<br>- `physical`. A physical barrier (e.g., a curb, Jersey barrier) is in place.
-    width FLOAT,  -- Optional. Width of the lane, short_length units.
-    PRIMARY KEY (lane_id),
-    FOREIGN KEY (link_id) REFERENCES link(link_id)
-);
+	lane_id TEXT NOT NULL, 
+	link_id TEXT NOT NULL, 
+	lane_num INTEGER NOT NULL CHECK (lane_num >= -10) CHECK (lane_num <= 10), 
+	allowed_uses TEXT, 
+	r_barrier VARCHAR(10), 
+	l_barrier VARCHAR(10), 
+	width FLOAT CHECK (width >= 0), 
+	PRIMARY KEY (lane_id), 
+	FOREIGN KEY(link_id) REFERENCES link (link_id)
+)

usage/database/lane_tod.sql

@@ -1,14 +1,14 @@
 CREATE TABLE IF NOT EXISTS lane_tod (
-    lane_tod_id VARCHAR(255) NOT NULL,  -- Primary key.
-    lane_id VARCHAR(255) NOT NULL,  -- Required. Foreign key to `lane`
-    timeday_id VARCHAR(255),  -- Conditionally required (either timeday_id or time_day). Foreign key to time_set_definitions.
-    time_day VARCHAR(255),  -- Conditionally required (either timeday_id or time_day). XXXXXXXX_HHMM_HHMM, where XXXXXXXX is a bitmap of days of the week, Sunday-Saturday, Holiday. The HHMM are the start and end times.
-    lane_num VARCHAR(255) NOT NULL,  -- Required. Lane number identified as offset to the right from the centerline. i.e. -1, 1, 2 (use left-to-rightnumbering).
-    allowed_uses VARCHAR(255),  -- Optional. Set of allowed uses that should appear in either the use_definition or use_group tables; comma-separated.
-    r_barrier VARCHAR(255),  -- Optional. Whether a barrier exists to prevent vehicles from changing lanes to the right.<br>- `none` (the default). Indicates that a vehicle can change lanes, provided that the vehicle-type is permitted in the destination lane<br>- `Regulatory`. There is a regulatory prohibition (e.g., a double-white solid line) against changing lanes, but no physical barrier<br>- `Physical`. A physical barrier (e.g., a curb, Jersey barrier) is in place.
-    l_barrier VARCHAR(255),  -- Optional. Whether a barrier exists to prevent vehicles from changing lanes to the right.<br>- `none` (the default).  Indicates that a vehicle can change lanes, provided that the vehicle-type is permitted in the destination lane<br>- `Regulatory`. There is a regulatory prohibition (e.g., a double-white solid line) against changing lanes, but no physical barrier<br>- `Physical`. A physical barrier (e.g., a curb, Jersey barrier) is in place.
-    width FLOAT,  -- Optional. Width of the lane, short_length units.
-    PRIMARY KEY (lane_tod_id),
-    FOREIGN KEY (lane_id) REFERENCES lane(lane_id),
-    FOREIGN KEY (timeday_id) REFERENCES time_set_definitions(timeday_id)
-);
+	lane_tod_id TEXT NOT NULL, 
+	lane_id TEXT NOT NULL, 
+	timeday_id TEXT, 
+	time_day TEXT, 
+	lane_num INTEGER NOT NULL CHECK (lane_num >= -10) CHECK (lane_num <= 10), 
+	allowed_uses TEXT, 
+	r_barrier VARCHAR(10), 
+	l_barrier VARCHAR(10), 
+	width FLOAT CHECK (width >= 0), 
+	PRIMARY KEY (lane_tod_id), 
+	FOREIGN KEY(lane_id) REFERENCES lane (lane_id), 
+	FOREIGN KEY(timeday_id) REFERENCES time_set_definitions (timeday_id)
+)

usage/database/link.sql

@@ -1,29 +1,29 @@
 CREATE TABLE IF NOT EXISTS link (
-    link_id VARCHAR(255) NOT NULL,  -- Primary key - could be SharedStreets Reference ID
-    parent_link_id VARCHAR(255),  -- Optional. The parent of this link. For example,for a sidewalk, this is the adjacent road.
-    name VARCHAR(255),  -- Optional. Street or Path Name
-    from_node_id VARCHAR(255) NOT NULL,  -- Required: origin node
-    to_node_id VARCHAR(255) NOT NULL,  -- Required: destination node
-    directed VARCHAR(255),  -- Required. Whether the link is directed (travel only occurs from the from_node to the to_node) or undirected.
-    geometry_id VARCHAR(255),  -- Optional. Foreign key (Link_Geometry table).
-    geometry VARCHAR(255),  -- Optional. Link geometry, in well-known text (WKT) format. Optionally, other formats supported by geopandas (GeoJSON, PostGIS) may be used if specified in geometry_field_format in gmns.spec.json
-    dir_flag VARCHAR(255),  -- Optional. <br>1  shapepoints go from from_node to to_node;<br>-1 shapepoints go in the reverse direction;<br>0  link is undirected or no geometry information is provided.
-    length FLOAT,  -- Optional. Length of the link in long_length units
-    grade FLOAT,  -- % grade, negative is downhill
-    facility_type VARCHAR(255),  -- Facility type (e.g., freeway, arterial, etc.)
-    capacity FLOAT,  -- Optional. Capacity (veh / hr / lane)
-    free_speed FLOAT,  -- Optional. Free flow speed, units defined by config file
-    lanes VARCHAR(255),  -- Optional. Number of permanent lanes (not including turn pockets) in the direction of travel open to motor vehicles. It does not include bike lanes, shoulders or parking lanes.
-    bike_facility VARCHAR(255),  -- Optional. Types of bicycle accommodation based on the National Bikeway Network Data Template (Table 1-A at https://nmtdev.ornl.gov/assets/templates/NBN_DataTemplates_final.pdf)
-    ped_facility VARCHAR(255),  -- Optional. Type of pedestrian accommodation: unknown, none, shoulder, sidewalk, offstreet path
-    parking VARCHAR(255),  -- Optional. Type of parking: unknown, none, parallel, angle, other
-    allowed_uses VARCHAR(255),  -- Optional. Set of allowed uses that should appear in either the use_definition or use_group tables; comma-separated.
-    toll FLOAT,  -- Optional.  Toll on the link, in currency units.
-    jurisdiction VARCHAR(255),  -- Optional.  Owner/operator of the link.
-    row_width FLOAT,  -- Optional. Width (short_length units) of the entire right-of-way (both directions).
-    PRIMARY KEY (link_id),
-    FOREIGN KEY (parent_link_id) REFERENCES (link_id),
-    FOREIGN KEY (from_node_id) REFERENCES node(node_id),
-    FOREIGN KEY (to_node_id) REFERENCES node(node_id),
-    FOREIGN KEY (geometry_id) REFERENCES geometry(geometry_id)
-);
+	link_id TEXT NOT NULL, 
+	name TEXT, 
+	from_node_id TEXT NOT NULL, 
+	to_node_id TEXT NOT NULL, 
+	directed BOOLEAN NOT NULL, 
+	geometry_id TEXT, 
+	geometry TEXT, 
+	parent_link_id TEXT, 
+	dir_flag INTEGER, 
+	length FLOAT CHECK (length >= 0), 
+	grade FLOAT CHECK (grade <= 100) CHECK (grade >= -100), 
+	facility_type TEXT, 
+	capacity FLOAT CHECK (capacity >= 0), 
+	free_speed FLOAT CHECK (free_speed >= 0) CHECK (free_speed <= 200), 
+	lanes INTEGER CHECK (lanes >= 0), 
+	bike_facility VARCHAR(22), 
+	ped_facility VARCHAR(14), 
+	parking VARCHAR(8), 
+	allowed_uses TEXT, 
+	toll FLOAT, 
+	jurisdiction TEXT, 
+	row_width FLOAT CHECK (row_width >= 0), 
+	PRIMARY KEY (link_id), 
+	FOREIGN KEY(to_node_id) REFERENCES node (node_id), 
+	FOREIGN KEY(parent_link_id) REFERENCES link (link_id), 
+	FOREIGN KEY(geometry_id) REFERENCES geometry (geometry_id), 
+	FOREIGN KEY(from_node_id) REFERENCES node (node_id)
+)

usage/database/link_tod.sql

@@ -1,17 +1,17 @@
 CREATE TABLE IF NOT EXISTS link_tod (
-    link_tod_id VARCHAR(255) NOT NULL,  -- Primary key
-    link_id VARCHAR(255) NOT NULL,  -- Required. Foreign key, link table
-    timeday_id VARCHAR(255),  -- Conditionally required (either timeday_id or time_day). Foreign key to time_set_definitions.
-    time_day VARCHAR(255),  -- Conditionally required (either timeday_id or time_day). XXXXXXXX_HHMM_HHMM, where XXXXXXXX is a bitmap of days of the week, Sunday-Saturday, Holiday. The HHMM are the start and end times.
-    capacity FLOAT,  -- Optional. Capacity (veh / hr / lane)
-    free_speed FLOAT,  -- Optional. Free flow speed in long_distance units per hour 
-    lanes VARCHAR(255),  -- Optional. Number of permanent lanes (not including turn pockets) in the direction of travel open to motor vehicles. It does not include bike lanes, shoulders or parking lanes.
-    bike_facility VARCHAR(255),  -- Optional. Types of bicycle accommodation based on the National Bikeway Network Data Template (Table 1-A at https://nmtdev.ornl.gov/assets/templates/NBN_DataTemplates_final.pdf)
-    ped_facility VARCHAR(255),  -- Optional. Type of pedestrian accommodation: unknown, none, shoulder, sidewalk, offstreet path
-    parking VARCHAR(255),  -- Optional. Type of parking: unknown, none, parallel, angle, other
-    allowed_uses VARCHAR(255),  -- Optional. Set of allowed uses that should appear in either the use_definition or use_group tables; comma-separated.
-    toll FLOAT,  -- toll in currency units.
-    PRIMARY KEY (link_tod_id),
-    FOREIGN KEY (link_id) REFERENCES link(link_id),
-    FOREIGN KEY (timeday_id) REFERENCES time_set_definitions(timeday_id)
-);
+	link_tod_id TEXT NOT NULL, 
+	link_id TEXT NOT NULL, 
+	timeday_id TEXT, 
+	time_day TEXT, 
+	capacity FLOAT CHECK (capacity >= 0), 
+	free_speed FLOAT CHECK (free_speed <= 200) CHECK (free_speed >= 0), 
+	lanes INTEGER CHECK (lanes >= 0), 
+	bike_facility VARCHAR(22), 
+	ped_facility VARCHAR(14), 
+	parking VARCHAR(8), 
+	allowed_uses TEXT, 
+	toll FLOAT, 
+	PRIMARY KEY (link_tod_id), 
+	FOREIGN KEY(link_id) REFERENCES link (link_id), 
+	FOREIGN KEY(timeday_id) REFERENCES time_set_definitions (timeday_id)
+)