Merge pull request #1366 from cityofaustin/1.41.0-release-candidate

Release VZ 1.41.0 - Toomey Rd

atd-etl/afd_ems_import/afd_incident_upload.py

@@ -40,6 +40,7 @@
 ONEPASSWORD_CONNECT_HOST = os.getenv("OP_CONNECT")  # where we get our secrets
 VAULT_ID = os.getenv("OP_VAULT_ID")
 
+
 def get_secrets():
     REQUIRED_SECRETS = {
         "bastion_host": {
@@ -111,7 +112,12 @@ def get_secrets():
             "opitem": "RDS Bastion Key",
             "opfield": ".private key",
             "opvault": VAULT_ID,
-            },
+        },
+        "record_age_maximum_in_days": {
+            "opitem": "Vision Zero AFD and EMS Import",
+            "opfield": f"{DEPLOYMENT_ENVIRONMENT}.AFD Record Age Maximum in Days",
+            "opvault": VAULT_ID,
+        },
     }
 
     # instantiate a 1Password client
@@ -137,11 +143,11 @@ def get_secrets():
 DB_BASTION_HOST_SSH_USERNAME = secrets["bastion_ssh_username"]
 DB_BASTION_HOST_SSH_PRIVATE_KEY = secrets["bastion_ssh_private_key"]
 DB_RDS_HOST = secrets["database_host"]
-RECORD_AGE_MAXIMUM = None # the default is fine, and we never tune it. 
+RECORD_AGE_MAXIMUM = secrets["record_age_maximum_in_days"]
 
 
 def main():
-    record_age_maximum = RECORD_AGE_MAXIMUM or 15
+    record_age_maximum = int(RECORD_AGE_MAXIMUM) or 15
     timestamp = get_timestamp()
     newest_email = get_most_recent_email()
     attachment_location = extract_email_attachment(newest_email)
@@ -150,7 +156,6 @@ def main():
     upload_token = upload_data_to_postgres(data, record_age_maximum)
 
 
-
 def get_timestamp():
     current = datetime.now()
     return time.mktime(current.timetuple())
@@ -217,9 +222,10 @@ def create_and_parse_dataframe(location):
 
 
 def upload_data_to_postgres(data, age_cutoff):
-
     with SshKeyTempDir() as key_directory:
-        write_key_to_file(key_directory + "/id_ed25519", DB_BASTION_HOST_SSH_PRIVATE_KEY + "\n") 
+        write_key_to_file(
+            key_directory + "/id_ed25519", DB_BASTION_HOST_SSH_PRIVATE_KEY + "\n"
+        )
         ssh_tunnel = SSHTunnelForwarder(
             (DB_BASTION_HOST),
             ssh_username=DB_BASTION_HOST_SSH_USERNAME,
@@ -228,13 +234,12 @@ def upload_data_to_postgres(data, age_cutoff):
         )
         ssh_tunnel.start()
 
-
         pg = psycopg2.connect(
-            host='localhost', 
+            host="localhost",
             port=ssh_tunnel.local_bind_port,
-            user=DB_USERNAME, 
-            password=DB_PASSWORD, 
-            dbname=DB_DATABASE
+            user=DB_USERNAME,
+            password=DB_PASSWORD,
+            dbname=DB_DATABASE,
         )
 
         print(f"Max record age: {age_cutoff}")
@@ -356,12 +361,13 @@ def __init__(self):
         self.path = None
 
     def __enter__(self):
-        self.path = tempfile.mkdtemp(dir='/tmp')
+        self.path = tempfile.mkdtemp(dir="/tmp")
         return self.path
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         shutil.rmtree(self.path)
 
+
 def write_key_to_file(path, content):
     # Open the file with write permissions and create it if it doesn't exist
     fd = os.open(path, os.O_WRONLY | os.O_CREAT, 0o600)
@@ -372,5 +378,6 @@ def write_key_to_file(path, content):
     # Close the file
     os.close(fd)
 
+
 if __name__ == "__main__":
     main()

atd-etl/afd_ems_import/docker-compose.yml

@@ -6,6 +6,4 @@ services:
     volumes:
       - .:/app
     env_file:
-      - ./env
-    environment:
-      - ENVIRONMENT=development
+      - ./.env

atd-etl/afd_ems_import/ems_incident_upload.py

@@ -34,7 +34,8 @@
 import onepasswordconnectsdk
 from onepasswordconnectsdk.client import Client, new_client
 
-# Heads up! Looking for global code? Look just below this get_secrets() function. 
+# Heads up! Looking for global code? Look just below this get_secrets() function.
+
 
 def get_secrets():
     REQUIRED_SECRETS = {
@@ -107,14 +108,20 @@ def get_secrets():
             "opitem": "RDS Bastion Key",
             "opfield": ".private key",
             "opvault": VAULT_ID,
-            },
+        },
+        "record_age_maximum_in_days": {
+            "opitem": "Vision Zero AFD and EMS Import",
+            "opfield": f"{DEPLOYMENT_ENVIRONMENT}.EMS Record Age Maximum in Days",
+            "opvault": VAULT_ID,
+        },
     }
 
     # instantiate a 1Password client
     client: Client = new_client(ONEPASSWORD_CONNECT_HOST, ONEPASSWORD_CONNECT_TOKEN)
     # get the requested secrets from 1Password
     return onepasswordconnectsdk.load_dict(client, REQUIRED_SECRETS)
 
+
 pp = pprint.PrettyPrinter(indent=4)
 
 DEPLOYMENT_ENVIRONMENT = os.environ.get(
@@ -141,11 +148,11 @@ def get_secrets():
 DB_BASTION_HOST_SSH_USERNAME = secrets["bastion_ssh_username"]
 DB_BASTION_HOST_SSH_PRIVATE_KEY = secrets["bastion_ssh_private_key"]
 DB_RDS_HOST = secrets["database_host"]
-RECORD_AGE_MAXIMUM = None # the default is fine, and we never tune it.  
+RECORD_AGE_MAXIMUM = secrets["record_age_maximum_in_days"]
 
 
 def main():
-    record_age_maximum = RECORD_AGE_MAXIMUM or 15
+    record_age_maximum = int(RECORD_AGE_MAXIMUM) or 15
     timestamp = get_timestamp()
     newest_email = get_most_recent_email()
     attachment_location = extract_email_attachment(newest_email)
@@ -154,7 +161,6 @@ def main():
     upload_token = upload_data_to_postgres(data, record_age_maximum)
 
 
-
 def get_timestamp():
     current = datetime.now()
     return time.mktime(current.timetuple())
@@ -216,14 +222,20 @@ def upload_attachment_to_S3(location, timestamp):
 
 def create_and_parse_dataframe(location):
     # Parse CSV into pandas dataframe
-    data = pandas.read_csv(f"{location}/attachment.csv", header=0, on_bad_lines='skip',  encoding='ISO-8859-1')
+    data = pandas.read_csv(
+        f"{location}/attachment.csv",
+        header=0,
+        on_bad_lines="skip",
+        encoding="ISO-8859-1",
+    )
     return data
 
 
 def upload_data_to_postgres(data, age_cutoff):
-
     with SshKeyTempDir() as key_directory:
-        write_key_to_file(key_directory + "/id_ed25519", DB_BASTION_HOST_SSH_PRIVATE_KEY + "\n") 
+        write_key_to_file(
+            key_directory + "/id_ed25519", DB_BASTION_HOST_SSH_PRIVATE_KEY + "\n"
+        )
         ssh_tunnel = SSHTunnelForwarder(
             (DB_BASTION_HOST),
             ssh_username=DB_BASTION_HOST_SSH_USERNAME,
@@ -232,19 +244,20 @@ def upload_data_to_postgres(data, age_cutoff):
         )
         ssh_tunnel.start()
 
-
         pg = psycopg2.connect(
-            host='localhost', 
+            host="localhost",
             port=ssh_tunnel.local_bind_port,
             user=DB_USERNAME,
             password=DB_PASSWORD,
-            dbname=DB_DATABASE
+            dbname=DB_DATABASE,
         )
 
         print(f"Max record age: {age_cutoff}")
         print(f"Input Dataframe shape: {data.shape}")
         if age_cutoff > 0:
-            data["Incident_Date_Received"] = pandas.to_datetime(data["Incident_Date_Received"], format="%Y-%m-%d")
+            data["Incident_Date_Received"] = pandas.to_datetime(
+                data["Incident_Date_Received"], format="%Y-%m-%d"
+            )
             age_threshold = datetime.today() - timedelta(days=age_cutoff)
             data = data[data["Incident_Date_Received"] > age_threshold]
         print(f"Trimmed data shape: {data.shape}")
@@ -254,10 +267,17 @@ def upload_data_to_postgres(data, age_cutoff):
         for index, row in data.iloc[::-1].iterrows():
             if not index % 100:
                 print(f"Row {str(index)} of {str(data.shape[0])}")
-            #print(data["Incident_Date_Received"][index])
-
-            apd_incident_numbers = str(row["APD_Incident_Numbers"]).replace("-", "").replace(" ", "").split(",")
-            apd_incident_numbers[:] = (value for value in apd_incident_numbers if not value == 'nan')
+            # print(data["Incident_Date_Received"][index])
+
+            apd_incident_numbers = (
+                str(row["APD_Incident_Numbers"])
+                .replace("-", "")
+                .replace(" ", "")
+                .split(",")
+            )
+            apd_incident_numbers[:] = (
+                value for value in apd_incident_numbers if not value == "nan"
+            )
             if len(apd_incident_numbers) > 1:
                 pass
                 # print(f"🛎 Found multiple incident numbers: " + str(row["EMS_IncidentNumber"]))
@@ -269,10 +289,10 @@ def upload_data_to_postgres(data, age_cutoff):
             existing = cursor.fetchone()
 
             sql = ""
-            values = []        
+            values = []
 
             if existing["existing"] > 0:
-                #print("Update")
+                # print("Update")
                 sql = """
                     update ems__incidents set
                         incident_date_received = %s,
@@ -368,14 +388,14 @@ def upload_data_to_postgres(data, age_cutoff):
                     row["Incident_Location_Longitude"],
                     row["Incident_Location_Latitude"],
                     "{" + ", ".join(apd_incident_numbers) + "}",
-                    row["PCR_Key"]
+                    row["PCR_Key"],
                 ]
 
                 for i in range(len(values)):
                     if pandas.isna(values[i]):
                         values[i] = None
             else:
-                #print("Insert")
+                # print("Insert")
                 sql = """
                     insert into ems__incidents (
                         pcr_key, 
@@ -498,20 +518,20 @@ def upload_data_to_postgres(data, age_cutoff):
     return True
 
 
-
 # these temp directories are used to store ssh keys, because they will
 # automatically clean themselves up when they go out of scope.
 class SshKeyTempDir:
     def __init__(self):
         self.path = None
 
     def __enter__(self):
-        self.path = tempfile.mkdtemp(dir='/tmp')
+        self.path = tempfile.mkdtemp(dir="/tmp")
         return self.path
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         shutil.rmtree(self.path)
 
+
 def write_key_to_file(path, content):
     # Open the file with write permissions and create it if it doesn't exist
     fd = os.open(path, os.O_WRONLY | os.O_CREAT, 0o600)
@@ -522,5 +542,6 @@ def write_key_to_file(path, content):
     # Close the file
     os.close(fd)
 
+
 if __name__ == "__main__":
     main()

atd-etl/afd_ems_import/env_template

@@ -0,0 +1,4 @@
+ENVIRONMENT=development
+OP_API_TOKEN=
+OP_CONNECT=
+OP_VAULT_ID=

atd-vzd/metadata/databases/default/tables/public_atd__coordination_partners_lkp.yaml

@@ -9,3 +9,18 @@ array_relationships:
         table:
           name: recommendations_partners
           schema: public
+select_permissions:
+  - role: editor
+    permission:
+      columns:
+        - coord_partner_desc
+        - id
+      filter: {}
+    comment: ""
+  - role: readonly
+    permission:
+      columns:
+        - coord_partner_desc
+        - id
+      filter: {}
+    comment: ""

atd-vzd/metadata/databases/default/tables/public_atd__mode_category_lkp.yaml

@@ -1,3 +1,20 @@
 table:
   name: atd__mode_category_lkp
   schema: public
+select_permissions:
+  - role: editor
+    permission:
+      columns:
+        - atd_mode_category_desc
+        - atd_mode_category_mode_name
+        - id
+      filter: {}
+    comment: ""
+  - role: readonly
+    permission:
+      columns:
+        - atd_mode_category_desc
+        - atd_mode_category_mode_name
+        - id
+      filter: {}
+    comment: ""

atd-vzd/metadata/databases/default/tables/public_atd__recommendation_status_lkp.yaml

@@ -9,3 +9,18 @@ array_relationships:
         table:
           name: recommendations
           schema: public
+select_permissions:
+  - role: editor
+    permission:
+      columns:
+        - id
+        - rec_status_desc
+      filter: {}
+    comment: ""
+  - role: readonly
+    permission:
+      columns:
+        - id
+        - rec_status_desc
+      filter: {}
+    comment: ""

atd-vzd/migrations/default/1706291244487_update_crash_based_comp_cost_2024/down.sql

@@ -0,0 +1,27 @@
+UPDATE public.atd_txdot__est_comp_cost_crash_based
+SET est_comp_cost_amount = 3000000
+WHERE est_comp_cost_desc = 'Killed (K)';
+
+UPDATE public.atd_txdot__est_comp_cost_crash_based
+SET est_comp_cost_amount = 2500000
+WHERE est_comp_cost_desc = 'Suspected Serious Injury (A)';
+
+UPDATE public.atd_txdot__est_comp_cost_crash_based
+SET est_comp_cost_amount = 270000
+WHERE est_comp_cost_desc = 'Non-incapacitating Injury (B)';
+
+UPDATE public.atd_txdot__est_comp_cost_crash_based
+SET est_comp_cost_amount = 220000
+WHERE est_comp_cost_desc = 'Possible Injury (C)';
+
+UPDATE public.atd_txdot__est_comp_cost_crash_based
+SET est_comp_cost_amount = 50000
+WHERE est_comp_cost_desc = 'Not Injured';
+
+UPDATE public.atd_txdot__est_comp_cost_crash_based
+SET est_comp_cost_amount = 50000
+WHERE est_comp_cost_desc = 'Unknown Injury (0)';
+
+UPDATE public.atd_txdot__est_comp_cost_crash_based
+SET est_comp_cost_amount = 10000
+WHERE est_comp_cost_desc = 'Non-CR3';