From 7fecb906cfc231aed5ee81d0a1cce63f3d14e9b9 Mon Sep 17 00:00:00 2001 From: Phil Salant Date: Thu, 16 Dec 2021 16:31:34 -0500 Subject: [PATCH 1/5] Improve handler time logging, reduce import volume --- fidesctl/src/fidesapi/main.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/fidesctl/src/fidesapi/main.py b/fidesctl/src/fidesapi/main.py index 2115eb95c6..40bf33072f 100644 --- a/fidesctl/src/fidesapi/main.py +++ b/fidesctl/src/fidesapi/main.py @@ -2,9 +2,9 @@ Contains the code that sets up the API. """ -import logging -import time +from datetime import datetime from enum import Enum +from logging import WARNING from typing import Callable, Dict from fastapi import FastAPI, Request, Response @@ -13,7 +13,7 @@ from fidesapi import crud, database, db_session, view, visualize from fidesapi.logger import setup as setup_logging -from fidesctl.core.config import get_config, FidesctlConfig +from fidesctl.core.config import FidesctlConfig, get_config app = FastAPI(title="fidesctl") CONFIG: FidesctlConfig = get_config() @@ -51,13 +51,13 @@ def setup_server() -> None: @app.middleware("http") async def log_request(request: Request, call_next: Callable) -> Response: "Log basic information about every request handled by the server." - start = time.time() + start = datetime.now() response = await call_next(request) - handle_time = time.time() - start + handler_time = datetime.now() - start log.bind( method=request.method, status_code=response.status_code, - handler_time=f"{handle_time}s", + handler_time=f"{handler_time.microseconds * 0.001}ms", path=request.url.path, ).info("Request received") return response @@ -90,5 +90,5 @@ async def db_action(action: DBActions) -> Dict: def start_webserver() -> None: "Run the webserver." - server = Server(Config(app, host="0.0.0.0", port=8080, log_level=logging.WARNING)) + server = Server(Config(app, host="0.0.0.0", port=8080, log_level=WARNING)) server.run() From 46c2761f1dbb1f3872f9d20683bfb5334859eb4d Mon Sep 17 00:00:00 2001 From: Phil Salant Date: Thu, 16 Dec 2021 16:34:29 -0500 Subject: [PATCH 2/5] Don't log a success message when upserts fail --- fidesctl/src/fidesapi/crud.py | 1 + fidesctl/src/fidesapi/database.py | 12 ++++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/fidesctl/src/fidesapi/crud.py b/fidesctl/src/fidesapi/crud.py index 862f490e84..25fb9b0827 100644 --- a/fidesctl/src/fidesapi/crud.py +++ b/fidesctl/src/fidesapi/crud.py @@ -163,6 +163,7 @@ def upsert_resources(sql_model: SqlAlchemyBase, resource_dicts: List[Dict]) -> N session.rollback() error = errors.QueryError() log.bind(error=error.detail["error"]).error("Failed to upsert resources") + raise error finally: session.close() diff --git a/fidesctl/src/fidesapi/database.py b/fidesctl/src/fidesapi/database.py index 5974747f90..311dbeb7e5 100644 --- a/fidesctl/src/fidesapi/database.py +++ b/fidesctl/src/fidesapi/database.py @@ -9,6 +9,7 @@ from loguru import logger as log from sqlalchemy_utils.functions import create_database, database_exists +from fidesapi.errors import QueryError from fidesapi.sql_models import sql_model_map, SqlAlchemyBase from fidesapi.crud import upsert_resources from fideslang import DEFAULT_TAXONOMY @@ -58,8 +59,15 @@ def load_default_taxonomy() -> None: for resource_type in list(DEFAULT_TAXONOMY.__fields_set__): log.info(f"Processing {resource_type} resources...") resources = list(map(dict, dict(DEFAULT_TAXONOMY)[resource_type])) - upsert_resources(sql_model_map[resource_type], resources) - log.info(f"Successfully UPSERTED {len(resources)} {resource_type} resources") + + try: + upsert_resources(sql_model_map[resource_type], resources) + except QueryError: + pass # The upsert_resources function will log the error + else: + log.info( + f"Successfully UPSERTED {len(resources)} {resource_type} resources" + ) def reset_db(database_url: str) -> None: From a3e010f4699ecd4edf44ed395bf3e40ae7514044 Mon Sep 17 00:00:00 2001 From: Phil Salant Date: Thu, 16 Dec 2021 16:37:00 -0500 Subject: [PATCH 3/5] Add `created_at` and `updated_at` columns to all `FidesBase` models --- ...51d8a102a_add_created_at_and_updated_at.py | 46 +++++++++++++++++++ fidesctl/src/fidesapi/sql_models.py | 14 ++++-- 2 files changed, 57 insertions(+), 3 deletions(-) create mode 100644 fidesctl/src/fidesapi/migrations/versions/7c851d8a102a_add_created_at_and_updated_at.py diff --git a/fidesctl/src/fidesapi/migrations/versions/7c851d8a102a_add_created_at_and_updated_at.py b/fidesctl/src/fidesapi/migrations/versions/7c851d8a102a_add_created_at_and_updated_at.py new file mode 100644 index 0000000000..6e99a6f8a1 --- /dev/null +++ b/fidesctl/src/fidesapi/migrations/versions/7c851d8a102a_add_created_at_and_updated_at.py @@ -0,0 +1,46 @@ +"""Add created_at and updated_at + +Revision ID: 7c851d8a102a +Revises: 45c7a349db68 +Create Date: 2021-12-16 12:20:03.801089 + +""" +from alembic import op +from sqlalchemy import Column, DateTime, text + +from fidesapi.sql_models import sql_model_map + + +# revision identifiers, used by Alembic. +revision = "7c851d8a102a" +down_revision = "45c7a349db68" +branch_labels = None +depends_on = None + + +def upgrade(): + for model in sql_model_map.values(): + op.add_column( + model.__tablename__, + Column( + "created_at", + DateTime(timezone=True), + server_default=text("now()"), + nullable=True, + ), + ) + op.add_column( + model.__tablename__, + Column( + "updated_at", + DateTime(timezone=True), + server_default=text("now()"), + nullable=True, + ), + ) + + +def downgrade(): + for model in sql_model_map.values(): + op.drop_column(model.__tablename__, "created_at") + op.drop_column(model.__tablename__, "updated_at") diff --git a/fidesctl/src/fidesapi/sql_models.py b/fidesctl/src/fidesapi/sql_models.py index f0edaa8745..93214c71c5 100644 --- a/fidesctl/src/fidesapi/sql_models.py +++ b/fidesctl/src/fidesapi/sql_models.py @@ -4,8 +4,10 @@ from typing import Dict -from sqlalchemy import Column, Integer, Text, String, ARRAY, JSON -import sqlalchemy.ext.declarative +from sqlalchemy import ARRAY, JSON, Column, Integer, String, Text +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.sql import func +from sqlalchemy.sql.sqltypes import DateTime class SqlModelBase: @@ -25,9 +27,15 @@ class FidesBase(SqlModelBase): organization_fides_key = Column(Text) name = Column(Text) description = Column(Text) + created_at = Column(DateTime(timezone=True), server_default=func.now()) + updated_at = Column( + DateTime(timezone=True), + server_default=func.now(), + onupdate=func.now(), + ) -SqlAlchemyBase = sqlalchemy.ext.declarative.declarative_base(cls=SqlModelBase) +SqlAlchemyBase = declarative_base(cls=SqlModelBase) # Privacy Types From b2cb5c6782e4645c0f30f0b8a392d965fd4f7593 Mon Sep 17 00:00:00 2001 From: Phil Salant Date: Thu, 16 Dec 2021 16:46:12 -0500 Subject: [PATCH 4/5] Fix linter error --- fidesctl/src/fidesctl/core/evaluate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fidesctl/src/fidesctl/core/evaluate.py b/fidesctl/src/fidesctl/core/evaluate.py index f6c1ba7ce3..bee4720ada 100644 --- a/fidesctl/src/fidesctl/core/evaluate.py +++ b/fidesctl/src/fidesctl/core/evaluate.py @@ -542,7 +542,7 @@ def evaluate( validate_supported_policy_rules(policies=policies) echo_green( "Evaluating the following Policies:\n- {}".format( - "\n- ".join([key.fides_key for key in taxonomy.policy]) + "\n- ".join([key.fides_key for key in policies]) ) ) print("-" * 10) From da8f916fe4ee9a106486b7f37042915d9cedef34 Mon Sep 17 00:00:00 2001 From: Phil Salant Date: Mon, 20 Dec 2021 16:05:52 -0500 Subject: [PATCH 5/5] Fix divergent revision chain --- .../versions/7c851d8a102a_add_created_at_and_updated_at.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fidesctl/src/fidesapi/migrations/versions/7c851d8a102a_add_created_at_and_updated_at.py b/fidesctl/src/fidesapi/migrations/versions/7c851d8a102a_add_created_at_and_updated_at.py index 6e99a6f8a1..04a22cb524 100644 --- a/fidesctl/src/fidesapi/migrations/versions/7c851d8a102a_add_created_at_and_updated_at.py +++ b/fidesctl/src/fidesapi/migrations/versions/7c851d8a102a_add_created_at_and_updated_at.py @@ -13,7 +13,7 @@ # revision identifiers, used by Alembic. revision = "7c851d8a102a" -down_revision = "45c7a349db68" +down_revision = "312aff72b275" branch_labels = None depends_on = None