Run migrations on API initialization, add /admin/db/{action} endpoints

docs/fides/docs/api/index.md

@@ -14,18 +14,20 @@ The `fidesctl` API is almost completely programmatic, so it's easier to grasp th
 
 * Except for the `DELETE`, the endpoints accept and/or return JSON objects that represent the named resource. The structure of these objects is given in the [Fides Language: Resources chapter](/language/resources/organization/) -- it's the same structure that's used in the resource manifest files.
 
-That's about all there is to it. There are an additional four endpoints that we'll look at below, but the sets of quintuplet endpoints listed above make up the core of the `fidesctl` API.
+That's about all there is to it. There are an additional six endpoints that we'll look at below, but the sets of quintuplet endpoints listed above make up the core of the `fidesctl` API.
 
-After a brief review of the four addition endpoints, we'll provide a complete API reference followed by a set of cURL calls that you can use to exercise the API on your system.
+After a brief review of the six additional endpoints, we'll provide a complete API reference followed by a set of cURL calls that you can use to exercise the API on your system.
 
 ## Other endpoints
 
-The four additional endpoints are:
+The six additional endpoints are:
 
 * `GET /health` pings the API server to see if it's up and running. The call returns `200` if it's up and ready to receive messages, and `404` if not.
 
 * Three of the taxonomic resources, `/data_category`, `/data_use`, and `/data_qualifier` (but  _not_ `/data_subject`) define a `GET /resource_type/visualize/{figure_type}` endpoint that returns a graph of the resource's taxonomy.  For details, see the **API Reference**, below.
 
+* `POST admin/db/init` and `POST admin/db/reset` reinitialize the database and reset the database, respectfully. They also repopulate the database with the default fideslang taxonomy.
+
 ## API Reference
 
 ---
@@ -39,4 +41,4 @@ The four additional endpoints are:
             location.href = location.href
         }, 200);
     }
-</script>
+</script>

docs/fides/docs/getting_started/docker.md

@@ -29,14 +29,6 @@ The following commands should all be run from the top-level `fides` directory (w
     root@1a742083cedf:/fides/fidesctl#
     ```
 
-1. `fidesctl init-db` -> Builds the required images, spins up the database, and runs the initialization scripts:
-
-    ```bash
-    ~/git/fides% fidesctl init-db
-    INFO  [alembic.runtime.migration] Context impl PostgresqlImpl.
-    INFO  [alembic.runtime.migration] Will assume transactional DDL.
-    ```
-
 1. `fidesctl ping` -> This confirms that your `fidesctl` CLI can reach the server and everything is ready to go!
 
     ```bash
@@ -49,4 +41,4 @@ The following commands should all be run from the top-level `fides` directory (w
 
 Now that you're up and running, you can use `fidesctl` from the shell to get a list of all the possible CLI commands. You're now ready to start enforcing privacy with Fidesctl!
 
-See the [Tutorial](../tutorial/overview.md) page for a step-by-step guide on setting up a Fidesctl data privacy workflow.
+See the [Tutorial](../tutorial/index.md) page for a step-by-step guide on setting up a Fidesctl data privacy workflow.

fidesctl/Dockerfile

@@ -24,4 +24,7 @@ COPY . /fides/fidesctl
 WORKDIR /fides/fidesctl
 RUN pip install -e ".[all]"
 
+# Immediately flush to stdout, globally
+ENV PYTHONUNBUFFERED=TRUE
+
 CMD ["/bin/bash"]

fidesctl/src/fidesapi/crud.py

@@ -8,6 +8,7 @@
 
 from fastapi import APIRouter, status
 from sqlalchemy import update as _update
+from sqlalchemy.dialects.postgresql import Insert as _insert
 
 from fidesapi import db_session
 from fidesapi.sql_models import sql_model_map, SqlAlchemyBase
@@ -76,6 +77,25 @@ def update_resource(sql_model: SqlAlchemyBase, resource_dict: Dict, fides_key: s
     return result_sql_resource
 
 
+def upsert_resources(sql_model: SqlAlchemyBase, resource_dicts: List[Dict]) -> None:
+    """
+    Insert new resources into the database. If a resource already exists,
+    update it by it's fides_key.
+    """
+    session = db_session.create_session()
+    try:
+        insert_stmt = _insert(sql_model).values(resource_dicts)
+        session.execute(
+            insert_stmt.on_conflict_do_update(
+                index_elements=["fides_key"],
+                set_=insert_stmt.excluded,
+            )
+        )
+        session.commit()
+    finally:
+        session.close()
+
+
 def delete_resource(sql_model: SqlAlchemyBase, fides_key: str) -> Dict:
     """Delete a resource by its fides_key."""
     result_sql_resource = get_resource(sql_model, fides_key)

fidesctl/src/fidesapi/database.py

@@ -7,11 +7,10 @@
 from alembic.config import Config
 from alembic.migration import MigrationContext
 
-from fidesapi.sql_models import SqlAlchemyBase
-from fidesctl.core.apply import apply
-from fidesctl.core.config import FidesctlConfig
-from fidesctl.core.utils import get_db_engine
+from fidesapi.sql_models import sql_model_map, SqlAlchemyBase
+from fidesapi.crud import upsert_resources
 from fideslang import DEFAULT_TAXONOMY
+from fidesctl.core.utils import get_db_engine
 
 
 def get_alembic_config(database_url: str) -> Config:
@@ -33,23 +32,25 @@ def upgrade_db(alembic_config: Config, revision: str = "head") -> None:
     command.upgrade(alembic_config, revision)
 
 
-def init_db(database_url: str, fidesctl_config: FidesctlConfig) -> None:
+def init_db(database_url: str) -> None:
     """
     Runs the migrations and creates all of the database objects.
     """
     alembic_config = get_alembic_config(database_url)
     upgrade_db(alembic_config)
-    load_default_taxonomy(fidesctl_config)
+    load_default_taxonomy()
 
 
-def load_default_taxonomy(fidesctl_config: FidesctlConfig) -> None:
-    "Loads the default taxonomy into the database."
-    config = fidesctl_config
-    apply(
-        url=config.cli.server_url,
-        taxonomy=DEFAULT_TAXONOMY,
-        headers=config.user.request_headers,
-    )
+def load_default_taxonomy() -> None:
+    "Upserts the default taxonomy into the database."
+    print("UPSERTING the default fideslang taxonomy")
+    for resource_type in list(DEFAULT_TAXONOMY.__fields_set__):
+        print("-" * 10)
+        print(f"Processing {resource_type} resources...")
+        resources = list(map(dict, dict(DEFAULT_TAXONOMY)[resource_type]))
+        upsert_resources(sql_model_map[resource_type], resources)
+        print(f"UPSERTED {len(resources)} {resource_type} resources.")
+    print("-" * 10)
 
 
 def reset_db(database_url: str) -> None:

fidesctl/src/fidesapi/main.py

@@ -2,17 +2,23 @@
 Contains the code that sets up the API.
 """
 
+from enum import Enum
 from typing import Dict
 
 import uvicorn
 from fastapi import FastAPI
-
-from fidesapi import crud, db_session, visualize
+from fidesapi import crud, database, db_session, visualize
 from fidesctl.core.config import get_config
 
 app = FastAPI(title="fidesctl")
 
 
+class DBActions(str, Enum):
+    "The available path parameters for the `/admin/db/{action}` endpoint."
+    init = "init"
+    reset = "reset"
+
+
 def configure_routes() -> None:
     "Include all of the routers not defined here."
     for router in crud.routers:
@@ -22,15 +28,30 @@ def configure_routes() -> None:
         app.include_router(router)
 
 
-def configure_db(database_url: str) -> None:
+def configure_db() -> None:
     "Set up the db to be used by the app."
-    db_session.global_init(database_url)
+    db_session.global_init(config.api.database_url)
+    database.init_db(config.api.database_url)
 
 
 @app.get("/health", tags=["Health"])
 async def health() -> Dict:
     "Confirm that the API is running and healthy."
-    return {"data": {"message": "Fides service is healthy!"}}
+    return {"data": {"message": "Fidesctl service is healthy!"}}
+
+
+@app.post("/admin/db/{action}", tags=["Admin"])
+async def db_action(action: DBActions) -> Dict:
+    """
+    Initiate one of the enumerated DBActions.
+    """
+    action_text = "initialized"
+    if action == DBActions.reset:
+        database.reset_db(config.api.database_url)
+        action_text = DBActions.reset
+
+    configure_db()
+    return {"data": {"message": f"Fidesctl database {action_text}"}}
 
 
 def start_webserver() -> None:
@@ -40,4 +61,4 @@ def start_webserver() -> None:
 
 config = get_config()
 configure_routes()
-configure_db(config.api.database_url)
+configure_db()

fidesctl/src/fidesctl/cli/cli.py

@@ -3,7 +3,6 @@
 
 import click
 
-from fidesapi import database
 from fidesctl.cli.options import (
     dry_flag,
     fides_key_argument,
@@ -218,7 +217,7 @@ def init_db(ctx: click.Context) -> None:
 
     """
     config = ctx.obj["CONFIG"]
-    database.init_db(config.api.database_url, fidesctl_config=config)
+    handle_cli_response(_api.db_action(config.cli.server_url, "init"))
 
 
 @click.command()
@@ -287,17 +286,16 @@ def reset_db(ctx: click.Context, yes: bool) -> None:
 
     """
     config = ctx.obj["CONFIG"]
-    database_url = config.api.database_url
     if yes:
         are_you_sure = "y"
     else:
-        echo_red("This will drop all data from the Fides database!")
-        are_you_sure = input("Are you sure [y/n]?")
+        echo_red(
+            "This will drop all data from the Fides database and reload the default taxonomy!"
+        )
+        are_you_sure = input("Are you sure [y/n]? ")
 
     if are_you_sure.lower() == "y":
-        database.reset_db(database_url)
-        database.init_db(database_url, config)
-        echo_green("Database reset!")
+        handle_cli_response(_api.db_action(config.cli.server_url, "reset"))
     else:
         print("Aborting!")
 

fidesctl/src/fidesctl/core/api.py

@@ -99,3 +99,10 @@ def evaluate(
     resource_url = generate_resource_url(url, resource_type)
     url = f"{resource_url}evaluate/{fides_key}"
     return requests.get(url, headers=headers, params={"tag": tag, "message": message})
+
+
+def db_action(server_url: str, action: str) -> requests.Response:
+    """
+    Tell the API to perform a database action.
+    """
+    return requests.post(f"{server_url}/admin/db/{action}")

fidesctl/tests/conftest.py

@@ -1,13 +1,13 @@
 """Common fixtures to be used across tests."""
 from typing import Any, Dict
 
+import os
 import pytest
 import yaml
-import os
 
 from fideslang import models
 from fidesctl.core.config import get_config
-from fidesapi import database
+from fidesctl.core import api
 
 TEST_CONFIG_PATH = "tests/test_config.toml"
 
@@ -25,16 +25,13 @@ def test_config(test_config_path):
 @pytest.fixture(scope="session", autouse=True)
 def setup_db(test_config):
     "Sets up the database for testing."
-    database_url = test_config.api.database_url
-    database.reset_db(database_url)
-    database.init_db(database_url, test_config)
-    yield
+    yield api.db_action(test_config.cli.server_url, "reset")
 
 
 @pytest.fixture(scope="session")
 def resources_dict():
     """
-    Yields an resource containing sample representations of different
+    Yields a resource containing sample representations of different
     Fides resources.
     """
     resources_dict: Dict[str, Any] = {