Add created_at and updated_at columns to FidesBase model tables

fidesctl/src/fidesapi/crud.py

@@ -163,6 +163,7 @@ def upsert_resources(sql_model: SqlAlchemyBase, resource_dicts: List[Dict]) -> N
             session.rollback()
             error = errors.QueryError()
             log.bind(error=error.detail["error"]).error("Failed to upsert resources")
+            raise error
         finally:
             session.close()
 

fidesctl/src/fidesapi/database.py

@@ -9,6 +9,7 @@
 from loguru import logger as log
 from sqlalchemy_utils.functions import create_database, database_exists
 
+from fidesapi.errors import QueryError
 from fidesapi.sql_models import sql_model_map, SqlAlchemyBase
 from fidesapi.crud import upsert_resources
 from fideslang import DEFAULT_TAXONOMY
@@ -58,8 +59,15 @@ def load_default_taxonomy() -> None:
     for resource_type in list(DEFAULT_TAXONOMY.__fields_set__):
         log.info(f"Processing {resource_type} resources...")
         resources = list(map(dict, dict(DEFAULT_TAXONOMY)[resource_type]))
-        upsert_resources(sql_model_map[resource_type], resources)
-        log.info(f"Successfully UPSERTED {len(resources)} {resource_type} resources")
+
+        try:
+            upsert_resources(sql_model_map[resource_type], resources)
+        except QueryError:
+            pass  # The upsert_resources function will log the error
+        else:
+            log.info(
+                f"Successfully UPSERTED {len(resources)} {resource_type} resources"
+            )
 
 
 def reset_db(database_url: str) -> None:

fidesctl/src/fidesapi/main.py

@@ -2,9 +2,9 @@
 Contains the code that sets up the API.
 """
 
-import logging
-import time
+from datetime import datetime
 from enum import Enum
+from logging import WARNING
 from typing import Callable, Dict
 
 from fastapi import FastAPI, Request, Response
@@ -13,7 +13,7 @@
 
 from fidesapi import crud, database, db_session, view, visualize
 from fidesapi.logger import setup as setup_logging
-from fidesctl.core.config import get_config, FidesctlConfig
+from fidesctl.core.config import FidesctlConfig, get_config
 
 app = FastAPI(title="fidesctl")
 CONFIG: FidesctlConfig = get_config()
@@ -51,13 +51,13 @@ def setup_server() -> None:
 @app.middleware("http")
 async def log_request(request: Request, call_next: Callable) -> Response:
     "Log basic information about every request handled by the server."
-    start = time.time()
+    start = datetime.now()
     response = await call_next(request)
-    handle_time = time.time() - start
+    handler_time = datetime.now() - start
     log.bind(
         method=request.method,
         status_code=response.status_code,
-        handler_time=f"{handle_time}s",
+        handler_time=f"{handler_time.microseconds * 0.001}ms",
         path=request.url.path,
     ).info("Request received")
     return response
@@ -90,5 +90,5 @@ async def db_action(action: DBActions) -> Dict:
 
 def start_webserver() -> None:
     "Run the webserver."
-    server = Server(Config(app, host="0.0.0.0", port=8080, log_level=logging.WARNING))
+    server = Server(Config(app, host="0.0.0.0", port=8080, log_level=WARNING))
     server.run()

fidesctl/src/fidesapi/migrations/versions/7c851d8a102a_add_created_at_and_updated_at.py

@@ -0,0 +1,46 @@
+"""Add created_at and updated_at
+
+Revision ID: 7c851d8a102a
+Revises: 45c7a349db68
+Create Date: 2021-12-16 12:20:03.801089
+
+"""
+from alembic import op
+from sqlalchemy import Column, DateTime, text
+
+from fidesapi.sql_models import sql_model_map
+
+
+# revision identifiers, used by Alembic.
+revision = "7c851d8a102a"
+down_revision = "312aff72b275"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    for model in sql_model_map.values():
+        op.add_column(
+            model.__tablename__,
+            Column(
+                "created_at",
+                DateTime(timezone=True),
+                server_default=text("now()"),
+                nullable=True,
+            ),
+        )
+        op.add_column(
+            model.__tablename__,
+            Column(
+                "updated_at",
+                DateTime(timezone=True),
+                server_default=text("now()"),
+                nullable=True,
+            ),
+        )
+
+
+def downgrade():
+    for model in sql_model_map.values():
+        op.drop_column(model.__tablename__, "created_at")
+        op.drop_column(model.__tablename__, "updated_at")

fidesctl/src/fidesapi/sql_models.py

@@ -4,8 +4,10 @@
 
 from typing import Dict
 
-from sqlalchemy import Column, Integer, Text, String, ARRAY, JSON
-import sqlalchemy.ext.declarative
+from sqlalchemy import ARRAY, JSON, Column, Integer, String, Text
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.sql import func
+from sqlalchemy.sql.sqltypes import DateTime
 
 
 class SqlModelBase:
@@ -25,9 +27,15 @@ class FidesBase(SqlModelBase):
     organization_fides_key = Column(Text)
     name = Column(Text)
     description = Column(Text)
+    created_at = Column(DateTime(timezone=True), server_default=func.now())
+    updated_at = Column(
+        DateTime(timezone=True),
+        server_default=func.now(),
+        onupdate=func.now(),
+    )
 
 
-SqlAlchemyBase = sqlalchemy.ext.declarative.declarative_base(cls=SqlModelBase)
+SqlAlchemyBase = declarative_base(cls=SqlModelBase)
 
 
 # Privacy Types

fidesctl/src/fidesctl/core/evaluate.py

@@ -527,7 +527,7 @@ def evaluate(
     validate_supported_policy_rules(policies=policies)
     echo_green(
         "Evaluating the following policies:\n- {}".format(
-            "\n- ".join([key.fides_key for key in taxonomy.policy])
+            "\n- ".join([key.fides_key for key in policies])
         )
     )
     print("-" * 10)