Code Reorganization (#926)

* Code Reorganization

* add fides as a CLI entrypoint

* move all of the tests/testing infra into tests/ctl/

* pytest passing

* rename fidesctl -> fides

* further separation of files, api -> api/ctl, core + connectors -> ctl.core + ctl.connectors

* update dir references

* more tweaks

* update more paths

* fix a bug

* fix a missing __init__.py

* Revert "fix a missing __init__.py"

This reverts commit 484cb6e.

* Revert "fix a bug"

This reverts commit 50c7161.

* Revert "update more paths"

This reverts commit 4692ca7.

* Revert "more tweaks"

This reverts commit d488c7b.

* Revert "update dir references"

This reverts commit d69498b.

* Revert "further separation of files, api -> api/ctl, core + connectors -> ctl.core + ctl.connectors"

This reverts commit 329b72c.

* Revert "rename fidesctl -> fides"

This reverts commit 4890409.

* pytest passing

* update more src/fidesctl/api references

* move core and connectors into fidesctl.ctl.core and fidesctl.core.connectors

* add the __init__.py to fidesctl.ctl

* update code paths for fidesctl.ctl.connectors where it was missed

* fix some missed fidesctl.core renames

* fix a pointer to the datamap template

* updated the changelog

* Update .gitignore

* move the deps.py file after the rebase

* get rid of the misplaced deps.py file

* update paths from the merge

* fix sneaky path issues for bigquery

.gitignore

@@ -1,7 +1,6 @@
-fidesctl/api/src/main/resources/application.conf
 docs/fides/docs/api/openapi.json
 docs/fides/docs/schemas/config_schema.json
-fidesctl/api/build/static
+fidesctl/api/ctl/build/static
 
 ## generic files to ignore
 *~

CHANGELOG.md

@@ -39,6 +39,7 @@ The types of changes are:
 * Added taxonomy page to UI [#902](https://github.com/ethyca/fides/pull/902)
   * Added a nested accordion component for displaying taxonomy data [#910](https://github.com/ethyca/fides/pull/910)
 * Add lru cache to get_config [927](https://github.com/ethyca/fides/pull/927)
+* `fides` is now an alias for `fidesctl` as a CLI entrypoint [#926](https://github.com/ethyca/fides/pull/926)
 * Add user auth routes [929](https://github.com/ethyca/fides/pull/929)
 * Bump fideslib to 3.0.1 and remove patch code[931](https://github.com/ethyca/fides/pull/931)
 

Dockerfile

@@ -91,7 +91,7 @@ ENV PYTHONUNBUFFERED=TRUE
 ENV RUNNING_IN_DOCKER=TRUE
 
 # Make a static files directory
-RUN mkdir -p src/fidesctl/api/build/static
+RUN mkdir -p src/fidesctl/api/ctl/build/static
 
 EXPOSE 8080
 CMD ["fidesctl", "webserver"]
@@ -116,4 +116,4 @@ RUN python setup.py sdist
 RUN pip install dist/fidesctl-*.tar.gz
 
 # Copy frontend build over
-COPY --from=frontend /fides/clients/admin-ui/out/ /fides/src/fidesctl/api/build/static/
+COPY --from=frontend /fides/clients/admin-ui/out/ /fides/src/fidesctl/api/ctl/build/static/

MANIFEST.in

@@ -2,5 +2,5 @@ include LICENSE
 include requirements.txt
 include dev-requirements.txt
 include versioneer.py
-include src/fidesctl/api/alembic.ini
+include src/fidesctl/api/ctl/alembic.ini
 include src/fidesctl/templates/fides_datamap_template.xlsx

clients/admin-ui/package.json

@@ -14,7 +14,7 @@
     "analyze:server": "cross-env BUNDLE_ANALYZE=server next build",
     "analyze:browser": "cross-env BUNDLE_ANALYZE=browser next build",
     "export": "next build && next export",
-    "copy-export": "rsync -a --delete out/ ../../src/fidesctl/api/build/static/",
+    "copy-export": "rsync -a --delete out/ ../../src/fidesctl/api/ctl/build/static/",
     "prod-export": "npm run export && npm run copy-export",
     "cy:open": "cypress open",
     "cy:run": "cypress run",

docs/fides/docs/development/database_migration.md

@@ -2,12 +2,12 @@
 
 ---
 
-Changes to fidesctl could require a change to the database model. This includes scenarios where you want to persist a new field or replace an existing field. Changes made to the fidesctl database are done through alembic migration scripts. Migrations can be found in the following direcotry: `fidesctl/src/fidesctl/api/migrations/versions`
+Changes to fidesctl could require a change to the database model. This includes scenarios where you want to persist a new field or replace an existing field. Changes made to the fidesctl database are done through alembic migration scripts. Migrations can be found in the following direcotry: `src/fidesctl/api/ctl/migrations/versions`
 
 To create a new migration we use the `alembic revision` command:
 
 ```bash
-cd fidesctl/src/fidesctl/api
+cd src/fidesctl/api/ctl
 alembic revision --autogenerate -m "migration message"
 ```
 

noxfiles/constants_nox.py

@@ -16,7 +16,7 @@ def get_current_tag() -> str:
 # Files
 COMPOSE_FILE = "docker-compose.yml"
 INTEGRATION_COMPOSE_FILE = "docker-compose.integration-tests.yml"
-WITH_TEST_CONFIG = ("-f", "tests/test_config.toml")
+WITH_TEST_CONFIG = ("-f", "tests/ctl/test_config.toml")
 
 # Image Names & Tags
 REGISTRY = "ethyca"

pyproject.toml

@@ -78,9 +78,8 @@ good-names="_,i,setUp,tearDown,maxDiff,default_app_config"
 [tool.pytest.ini_options]
 testpaths="tests"
 log_level = "DEBUG"
-addopts = ["--cov=fidesctl.core",
+addopts = ["--cov=fidesctl.ctl.core",
             "--cov=fidesctl.cli",
-            "--cov=fideslang",
             "--cov-report=term-missing",
             "-vv",
             "--no-cov-on-fail",

setup.py

@@ -44,7 +44,7 @@
     long_description=long_description,
     long_description_content_type="text/markdown",
     url="https://github.com/ethyca/fides",
-    entry_points={"console_scripts": ["fidesctl=fidesctl.cli:cli"]},
+    entry_points={"console_scripts": ["fidesctl=fidesctl.cli:cli","fides=fidesctl.cli:cli"]},
     python_requires=">=3.8, <4",
     package_dir={"": "src"},
     packages=find_packages(where="src"),

src/fidesctl.egg-info/entry_points.txt

@@ -1,2 +1,3 @@
 [console_scripts]
+fides = fidesctl.cli:cli
 fidesctl = fidesctl.cli:cli

src/fidesctl/api/database/crud.py → src/fidesctl/api/ctl/database/crud.py

@@ -13,8 +13,8 @@
 from sqlalchemy.exc import SQLAlchemyError
 from sqlalchemy.future import select
 
-from fidesctl.api.database.session import async_session
-from fidesctl.api.utils import errors
+from fidesctl.api.ctl.database.session import async_session
+from fidesctl.api.ctl.utils import errors
 
 
 # CRUD Functions

src/fidesctl/api/database/database.py → src/fidesctl/api/ctl/database/database.py

@@ -12,13 +12,13 @@
 from sqlalchemy import create_engine
 from sqlalchemy_utils.functions import create_database, database_exists
 
-from fidesctl.api.sql_models import sql_model_map
-from fidesctl.api.utils.errors import (
+from fidesctl.api.ctl.sql_models import sql_model_map
+from fidesctl.api.ctl.utils.errors import (
     AlreadyExistsError,
     QueryError,
     get_full_exception_name,
 )
-from fidesctl.core.utils import get_db_engine
+from fidesctl.ctl.core.utils import get_db_engine
 
 from .crud import create_resource, upsert_resources
 

src/fidesctl/api/database/session.py → src/fidesctl/api/ctl/database/session.py

@@ -2,7 +2,7 @@
 from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine
 from sqlalchemy.orm import Session, sessionmaker
 
-from fidesctl.core.config import get_config
+from fidesctl.ctl.core.config import get_config
 
 config = get_config()
 engine = create_async_engine(

src/fidesctl/api/deps.py → src/fidesctl/api/ctl/deps.py

@@ -15,10 +15,10 @@
 from fideslib.oauth.scopes import SCOPES
 from sqlalchemy.orm import Session
 
-from fidesctl.api.database.session import sync_session
-from fidesctl.api.routes.util import API_PREFIX
-from fidesctl.api.sql_models import ClientDetail, FidesUser
-from fidesctl.core.config import get_config
+from fidesctl.api.ctl.database.session import sync_session
+from fidesctl.api.ctl.routes.util import API_PREFIX
+from fidesctl.api.ctl.sql_models import ClientDetail, FidesUser
+from fidesctl.ctl.core.config import get_config
 
 oauth2_scheme = OAuth2ClientCredentialsBearer(
     tokenUrl=(f"{API_PREFIX}/oauth/token"),

src/fidesctl/api/migrations/env.py → src/fidesctl/api/ctl/migrations/env.py

@@ -3,8 +3,8 @@
 from alembic import context
 from sqlalchemy import engine_from_config, pool
 
-from fidesctl.api.utils.logger import setup as setup_fidesapi_logger
-from fidesctl.core.config import get_config
+from fidesctl.api.ctl.utils.logger import setup as setup_fidesapi_logger
+from fidesctl.ctl.core.config import get_config
 
 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.
@@ -24,7 +24,7 @@
 # for 'autogenerate' support
 # from myapp import mymodel
 # target_metadata = mymodel.Base.metadata
-from fidesctl.api.sql_models import Base
+from fidesctl.api.ctl.sql_models import Base
 
 target_metadata = Base.metadata
 

src/fidesctl/api/migrations/versions/7c851d8a102a_add_created_at_and_updated_at.py → src/fidesctl/api/ctl/migrations/versions/7c851d8a102a_add_created_at_and_updated_at.py

@@ -8,7 +8,7 @@
 from alembic import op
 from sqlalchemy import Column, DateTime, text
 
-from fidesctl.api.sql_models import sql_model_map
+from fidesctl.api.ctl.sql_models import sql_model_map
 
 # revision identifiers, used by Alembic.
 revision = "7c851d8a102a"

src/fidesctl/api/routes/admin.py → src/fidesctl/api/ctl/routes/admin.py

@@ -1,10 +1,10 @@
 from enum import Enum
 from typing import Dict
 
-from fidesctl.api.database import database
-from fidesctl.api.routes.util import API_PREFIX
-from fidesctl.api.utils.api_router import APIRouter
-from fidesctl.core.config import FidesctlConfig, get_config
+from fidesctl.api.ctl.database import database
+from fidesctl.api.ctl.routes.util import API_PREFIX
+from fidesctl.api.ctl.utils.api_router import APIRouter
+from fidesctl.ctl.core.config import FidesctlConfig, get_config
 
 CONFIG: FidesctlConfig = get_config()
 router = APIRouter(prefix=API_PREFIX, tags=["Admin"])

src/fidesctl/api/routes/crud.py → src/fidesctl/api/ctl/routes/crud.py

@@ -11,17 +11,17 @@
 from fastapi import Response, status
 from fideslang import model_map
 
-from fidesctl.api.database.crud import (
+from fidesctl.api.ctl.database.crud import (
     create_resource,
     delete_resource,
     get_resource,
     list_resource,
     update_resource,
     upsert_resources,
 )
-from fidesctl.api.routes.util import API_PREFIX, get_resource_type
-from fidesctl.api.sql_models import sql_model_map
-from fidesctl.api.utils.api_router import APIRouter
+from fidesctl.api.ctl.routes.util import API_PREFIX, get_resource_type
+from fidesctl.api.ctl.sql_models import sql_model_map
+from fidesctl.api.ctl.utils.api_router import APIRouter
 
 # CRUD Endpoints
 routers = []

src/fidesctl/api/routes/datamap.py → src/fidesctl/api/ctl/routes/datamap.py

@@ -8,13 +8,13 @@
 from loguru import logger as log
 from pandas import DataFrame
 
-from fidesctl.api.routes.crud import get_resource, list_resource
-from fidesctl.api.routes.util import API_PREFIX
-from fidesctl.api.sql_models import sql_model_map
-from fidesctl.api.utils.api_router import APIRouter
-from fidesctl.api.utils.errors import DatabaseUnavailableError, NotFoundError
-from fidesctl.core.export import build_joined_dataframe
-from fidesctl.core.export_helpers import DATAMAP_COLUMNS
+from fidesctl.api.ctl.routes.crud import get_resource, list_resource
+from fidesctl.api.ctl.routes.util import API_PREFIX
+from fidesctl.api.ctl.sql_models import sql_model_map
+from fidesctl.api.ctl.utils.api_router import APIRouter
+from fidesctl.api.ctl.utils.errors import DatabaseUnavailableError, NotFoundError
+from fidesctl.ctl.core.export import build_joined_dataframe
+from fidesctl.ctl.core.export_helpers import DATAMAP_COLUMNS
 
 router = APIRouter(tags=["Datamap"], prefix=f"{API_PREFIX}/datamap")
 

src/fidesctl/api/routes/generate.py → src/fidesctl/api/ctl/routes/generate.py

@@ -9,24 +9,24 @@
 from loguru import logger as log
 from pydantic import BaseModel, root_validator
 
-from fidesctl.api.routes.crud import get_resource
-from fidesctl.api.routes.util import (
+from fidesctl.api.ctl.routes.crud import get_resource
+from fidesctl.api.ctl.routes.util import (
     API_PREFIX,
     route_requires_aws_connector,
     route_requires_bigquery_connector,
     route_requires_okta_connector,
 )
-from fidesctl.api.sql_models import sql_model_map
-from fidesctl.api.utils.api_router import APIRouter
-from fidesctl.connectors.models import (
+from fidesctl.api.ctl.sql_models import sql_model_map
+from fidesctl.api.ctl.utils.api_router import APIRouter
+from fidesctl.ctl.connectors.models import (
     AWSConfig,
     BigQueryConfig,
     ConnectorAuthFailureException,
     DatabaseConfig,
     OktaConfig,
 )
-from fidesctl.core.dataset import generate_bigquery_datasets, generate_db_datasets
-from fidesctl.core.system import generate_aws_systems, generate_okta_systems
+from fidesctl.ctl.core.dataset import generate_bigquery_datasets, generate_db_datasets
+from fidesctl.ctl.core.system import generate_aws_systems, generate_okta_systems
 
 
 class ValidTargets(str, Enum):

src/fidesctl/api/routes/health.py → src/fidesctl/api/ctl/routes/health.py

@@ -3,10 +3,10 @@
 from fastapi import HTTPException, status
 
 import fidesctl
-from fidesctl.api.database.database import get_db_health
-from fidesctl.api.routes.util import API_PREFIX
-from fidesctl.api.utils.api_router import APIRouter
-from fidesctl.core.config import FidesctlConfig, get_config
+from fidesctl.api.ctl.database.database import get_db_health
+from fidesctl.api.ctl.routes.util import API_PREFIX
+from fidesctl.api.ctl.utils.api_router import APIRouter
+from fidesctl.ctl.core.config import FidesctlConfig, get_config
 
 CONFIG: FidesctlConfig = get_config()
 

src/fidesctl/api/routes/user.py → src/fidesctl/api/ctl/routes/user.py

@@ -13,10 +13,10 @@
     HTTP_404_NOT_FOUND,
 )
 
-from fidesctl.api.deps import get_current_user, get_db, verify_oauth_client
-from fidesctl.api.routes.util import API_PREFIX
-from fidesctl.api.utils.api_router import APIRouter
-from fidesctl.core.config import FidesctlConfig, get_config
+from fidesctl.api.ctl.deps import get_current_user, get_db, verify_oauth_client
+from fidesctl.api.ctl.routes.util import API_PREFIX
+from fidesctl.api.ctl.utils.api_router import APIRouter
+from fidesctl.ctl.core.config import FidesctlConfig, get_config
 
 router = APIRouter(tags=["Users"], prefix=f"{API_PREFIX}")
 

src/fidesctl/api/routes/util.py → src/fidesctl/api/ctl/routes/util.py

@@ -4,7 +4,7 @@
 
 from fastapi import HTTPException, status
 
-from fidesctl.api.utils.api_router import APIRouter
+from fidesctl.api.ctl.utils.api_router import APIRouter
 
 API_PREFIX = "/api/v1"
 WEBAPP_DIRECTORY = Path("src/fidesctl/api/build/static")
@@ -31,7 +31,7 @@ def route_requires_aws_connector(func: Callable) -> Callable:
 
     def wrapper_func(*args, **kwargs) -> Any:  # type: ignore
         try:
-            import fidesctl.connectors.aws  # pylint: disable=unused-import
+            import fidesctl.ctl.connectors.aws  # pylint: disable=unused-import
         except ModuleNotFoundError:
             raise HTTPException(
                 status_code=status.HTTP_400_BAD_REQUEST,
@@ -50,7 +50,7 @@ def route_requires_okta_connector(func: Callable) -> Callable:
 
     def wrapper_func(*args, **kwargs) -> Any:  # type: ignore
         try:
-            import fidesctl.connectors.okta  # pylint: disable=unused-import
+            import fidesctl.ctl.connectors.okta  # pylint: disable=unused-import
         except ModuleNotFoundError:
             raise HTTPException(
                 status_code=status.HTTP_400_BAD_REQUEST,
@@ -69,7 +69,7 @@ def route_requires_bigquery_connector(func: Callable) -> Callable:
 
     def wrapper_func(*args, **kwargs) -> Any:  # type: ignore
         try:
-            import fidesctl.connectors.bigquery  # pylint: disable=unused-import
+            import fidesctl.ctl.connectors.bigquery  # pylint: disable=unused-import
         except ModuleNotFoundError:
             raise HTTPException(
                 status_code=status.HTTP_400_BAD_REQUEST,

src/fidesctl/api/routes/validate.py → src/fidesctl/api/ctl/routes/validate.py

@@ -7,14 +7,14 @@
 from fastapi import Response, status
 from pydantic import BaseModel
 
-from fidesctl.api.routes.util import (
+from fidesctl.api.ctl.routes.util import (
     API_PREFIX,
     route_requires_aws_connector,
     route_requires_bigquery_connector,
     route_requires_okta_connector,
 )
-from fidesctl.api.utils.api_router import APIRouter
-from fidesctl.connectors.models import (
+from fidesctl.api.ctl.utils.api_router import APIRouter
+from fidesctl.ctl.connectors.models import (
     AWSConfig,
     BigQueryConfig,
     ConnectorAuthFailureException,
@@ -105,7 +105,7 @@ async def validate_aws(aws_config: AWSConfig) -> None:
     Validates that given aws credentials are valid. Dependency
     exception is raised if failure occurs.
     """
-    import fidesctl.connectors.aws as aws_connector
+    import fidesctl.ctl.connectors.aws as aws_connector
 
     aws_connector.validate_credentials(aws_config=aws_config)
 
@@ -116,7 +116,7 @@ async def validate_bigquery(bigquery_config: BigQueryConfig) -> None:
     Validates that given GCP BigQuery credentials are valid. Dependency
     exception is raised if failure occurs.
     """
-    import fidesctl.connectors.bigquery as bigquery_connector
+    import fidesctl.ctl.connectors.bigquery as bigquery_connector
 
     bigquery_engine = bigquery_connector.get_bigquery_engine(bigquery_config)
     bigquery_connector.validate_bigquery_engine(bigquery_engine)
@@ -128,6 +128,6 @@ async def validate_okta(okta_config: OktaConfig) -> None:
     Validates that given okta credentials are valid. Dependency
     exception is raised if failure occurs.
     """
-    import fidesctl.connectors.okta as okta_connector
+    import fidesctl.ctl.connectors.okta as okta_connector
 
     await okta_connector.validate_credentials(okta_config=okta_config)

src/fidesctl/api/routes/visualize.py → src/fidesctl/api/ctl/routes/visualize.py

@@ -8,11 +8,11 @@
 from fastapi.responses import HTMLResponse
 from fideslang import model_map
 
-from fidesctl.api.routes.crud import list_resource
-from fidesctl.api.routes.util import API_PREFIX, get_resource_type
-from fidesctl.api.sql_models import sql_model_map
-from fidesctl.api.utils.api_router import APIRouter
-from fidesctl.core import visualize
+from fidesctl.api.ctl.routes.crud import list_resource
+from fidesctl.api.ctl.routes.util import API_PREFIX, get_resource_type
+from fidesctl.api.ctl.sql_models import sql_model_map
+from fidesctl.api.ctl.utils.api_router import APIRouter
+from fidesctl.ctl.core import visualize
 
 # pylint: disable=redefined-outer-name,cell-var-from-loop