Merge pull request #18 from GSA-TTS/submissions

Data Submissions

.gitignore

@@ -160,4 +160,7 @@ cython_debug/
 #.idea/
 
 # Development databases
-*.sqlite3
+*.sqlite3
+
+# Development storage
+storage/

nad_ch/application_context.py

@@ -1,40 +1,67 @@
 import os
 import logging
+from nad_ch.config import STORAGE_PATH
 from nad_ch.infrastructure.database import (
     session_scope,
-    SqlAlchemyDataProviderRepository
+    SqlAlchemyDataProviderRepository,
+    SqlAlchemyDataSubmissionRepository
 )
 from nad_ch.infrastructure.logger import Logger
-from tests.mocks import MockDataProviderRepository
+from nad_ch.infrastructure.storage import LocalStorage
+from tests.fakes import (
+    FakeDataProviderRepository,
+    FakeDataSubmissionRepository,
+    FakeStorage
+)
 
 
 class ApplicationContext:
     def __init__(self):
         self._providers = SqlAlchemyDataProviderRepository(session_scope)
+        self._submissions = SqlAlchemyDataSubmissionRepository(session_scope)
         self._logger = Logger(__name__)
+        self._storage = LocalStorage(STORAGE_PATH)
 
     @property
     def providers(self):
         return self._providers
 
+    @property
+    def submissions(self):
+        return self._submissions
+
     @property
     def logger(self):
         return self._logger
 
+    @property
+    def storage(self):
+        return self._storage
+
 
 class TestApplicationContext(ApplicationContext):
     def __init__(self):
-        self._providers = MockDataProviderRepository()
+        self._providers = FakeDataProviderRepository()
+        self._submissions = FakeDataSubmissionRepository()
         self._logger = Logger(__name__, logging.DEBUG)
+        self._storage = FakeStorage()
 
     @property
     def providers(self):
         return self._providers
 
+    @property
+    def submissions(self):
+        return self._submissions
+
     @property
     def logger(self):
         return self._logger
 
+    @property
+    def storage(self):
+        return self._storage
+
 
 def create_app_context():
     if os.environ.get('APP_ENV') == 'test':

nad_ch/config.py

@@ -7,3 +7,4 @@
 
 APP_ENV = os.getenv('APP_ENV')
 DATABASE_URL = os.getenv('DATABASE_URL')
+STORAGE_PATH = os.getenv('STORAGE_PATH')

nad_ch/controllers/cli.py

@@ -3,6 +3,7 @@
     add_data_provider,
     list_data_providers,
     ingest_data_submission,
+    list_data_submissions_by_provider
 )
 
 
@@ -29,8 +30,16 @@ def list_providers(ctx):
 
 @cli.command()
 @click.pass_context
-@click.argument('filepath')
+@click.argument('file_path')
 @click.argument('provider')
 def ingest(ctx, file_path, provider):
     context = ctx.obj
     ingest_data_submission(context, file_path, provider)
+
+
+@cli.command()
+@click.pass_context
+@click.argument('provider')
+def list_submissions_by_provider(ctx, provider):
+    context = ctx.obj
+    list_data_submissions_by_provider(context, provider)

nad_ch/domain/entities.py

@@ -1,11 +1,40 @@
-class DataProvider:
-    def __init__(self, name: str, id: int = None):
+class Entity:
+    def __init__(self, id: int = None):
         self.id = id
+        self.created_at = None
+        self.updated_at = None
+
+    def set_created_at(self, created_at):
+        self.created_at = created_at
+
+    def set_updated_at(self, updated_at):
+        self.updated_at = updated_at
+
+
+class DataProvider(Entity):
+    def __init__(self, name: str, id: int = None):
+        super().__init__(id)
         self.name = name
 
+    def __repr__(self):
+        return f'DataProvider {self.id}, {self.name} \
+            (created: {self.created_at}; updated: {self.updated_at})'
 
-class DataSubmission:
-    def __init__(self, file_path: str, provider: DataProvider, id: int = None):
-        self.id = id
-        self.file_path = file_path
+
+class DataSubmission(Entity):
+    def __init__(
+        self,
+        file_name: str,
+        url: str,
+        provider: DataProvider,
+        id: int = None,
+    ):
+        super().__init__(id)
+        self.file_name = file_name
+        self.url = url
         self.provider = provider
+
+    def __repr__(self):
+        return f'DataSubmission \
+            {self.id}, {self.file_name}, {self.url}, {self.provider} \
+                (created: {self.created_at}; updated: {self.updated_at})'

nad_ch/domain/repositories.py

@@ -1,14 +1,22 @@
 from typing import Protocol
 from collections.abc import Iterable
-from nad_ch.domain.entities import DataProvider
+from nad_ch.domain.entities import DataProvider, DataSubmission
 
 
 class DataProviderRepository(Protocol):
-    def add(self, provider: DataProvider) -> None:
+    def add(self, provider: DataProvider) -> DataProvider:
         ...
 
     def get_by_name(self, name: str) -> DataProvider:
         ...
 
     def get_all(self) -> Iterable[DataProvider]:
         ...
+
+
+class DataSubmissionRepository(Protocol):
+    def add(self, submission: DataSubmission) -> DataSubmission:
+        ...
+
+    def get_by_provider(self, provider: DataProvider) -> Iterable[DataSubmission]:
+        ...

nad_ch/infrastructure/database.py

@@ -1,10 +1,11 @@
 from typing import List, Optional
-from sqlalchemy import Column, Integer, String, create_engine
-from sqlalchemy.orm import sessionmaker, declarative_base
+from sqlalchemy import Column, Integer, String, create_engine, ForeignKey, DateTime
+from sqlalchemy.orm import sessionmaker, declarative_base, relationship
+from sqlalchemy.sql import func
 import contextlib
 from nad_ch.config import DATABASE_URL
-from nad_ch.domain.entities import DataProvider
-from nad_ch.domain.repositories import DataProviderRepository
+from nad_ch.domain.entities import DataProvider, DataSubmission
+from nad_ch.domain.repositories import DataProviderRepository, DataSubmissionRepository
 
 
 engine = create_engine(DATABASE_URL)
@@ -27,28 +28,89 @@ def session_scope():
 ModelBase = declarative_base()
 
 
-class DataProviderModel(ModelBase):
-    __tablename__ = 'data_providers'
+class CommonBase(ModelBase):
+    __abstract__ = True
 
     id = Column(Integer, primary_key=True)
+    created_at = Column(DateTime(timezone=True), server_default=func.now())
+    updated_at = Column(
+        DateTime(timezone=True), server_default=func.now(), onupdate=func.now()
+    )
+
+
+class DataProviderModel(CommonBase):
+    __tablename__ = 'data_providers'
+
     name = Column(String)
 
+    data_submissions = relationship(
+        'DataSubmissionModel',
+        back_populates='data_provider'
+    )
+
     @staticmethod
     def from_entity(provider):
-        return DataProviderModel(id=provider.id, name=provider.name)
+        model = DataProviderModel(id=provider.id, name=provider.name)
+        return model
 
     def to_entity(self):
-        return DataProvider(id=self.id, name=self.name)
+        entity = DataProvider(id=self.id, name=self.name)
+
+        if self.created_at is not None:
+            entity.set_created_at(self.created_at)
+
+        if self.updated_at is not None:
+            entity.set_updated_at(self.updated_at)
+
+        return entity
+
+
+class DataSubmissionModel(CommonBase):
+    __tablename__ = 'data_submissions'
+
+    file_name = Column(String)
+    url = Column(String)
+    data_provider_id = Column(Integer, ForeignKey('data_providers.id'))
+
+    data_provider = relationship('DataProviderModel', back_populates='data_submissions')
+
+    @staticmethod
+    def from_entity(submission):
+        model = DataSubmissionModel(
+            id=submission.id,
+            file_name=submission.file_name,
+            url=submission.url,
+            data_provider_id=submission.provider.id
+        )
+        return model
+
+    def to_entity(self, provider: DataProvider):
+        entity = DataSubmission(
+            id=self.id,
+            file_name=self.file_name,
+            url=self.url,
+            provider=provider
+        )
+
+        if self.created_at is not None:
+            entity.set_created_at(self.created_at)
+
+        if self.updated_at is not None:
+            entity.set_updated_at(self.updated_at)
+
+        return entity
 
 
 class SqlAlchemyDataProviderRepository(DataProviderRepository):
     def __init__(self, session_factory):
         self.session_factory = session_factory
 
-    def add(self, provider: DataProvider):
+    def add(self, provider: DataProvider) -> DataProvider:
         with self.session_factory() as session:
             provider_model = DataProviderModel.from_entity(provider)
             session.add(provider_model)
+            session.commit()
+            session.refresh(provider_model)
             return provider_model.to_entity()
 
     def get_by_name(self, name: str) -> Optional[DataProvider]:
@@ -65,3 +127,51 @@ def get_all(self) -> List[DataProvider]:
             provider_models = session.query(DataProviderModel).all()
             providers_entities = [provider.to_entity() for provider in provider_models]
             return providers_entities
+
+
+class SqlAlchemyDataSubmissionRepository(DataSubmissionRepository):
+    def __init__(self, session_factory):
+        self.session_factory = session_factory
+
+    def add(self, submission: DataSubmission) -> DataSubmission:
+        with self.session_factory() as session:
+            submission_model = DataSubmissionModel.from_entity(submission)
+            session.add(submission_model)
+            session.commit()
+            session.refresh(submission_model)
+            provider_model = (
+                session.query(DataProviderModel)
+                .filter(DataProviderModel.id == submission_model.data_provider_id)
+                .first()
+            )
+            return submission_model.to_entity(provider_model.to_entity())
+
+    def get_by_name(self, file_name: str) -> Optional[DataSubmission]:
+        with self.session_factory() as session:
+            result = (
+                session.query(DataSubmissionModel, DataProviderModel)
+                .join(
+                    DataProviderModel, DataProviderModel.id ==
+                    DataSubmissionModel.data_provider_id
+                )
+                .filter(DataSubmissionModel.file_name == file_name)
+                .first()
+            )
+
+            if result:
+                submission_model, provider_model = result
+                return submission_model.to_entity(provider_model.to_entity())
+            else:
+                return None
+
+    def get_by_provider(self, provider: DataProvider) -> List[DataSubmission]:
+        with self.session_factory() as session:
+            submission_models = (
+                session.query(DataSubmissionModel)
+                .filter(DataSubmissionModel.data_provider_id == provider.id)
+                .all()
+            )
+            submission_entities = (
+                [submission.to_entity(provider) for submission in submission_models]
+            )
+            return submission_entities

nad_ch/infrastructure/storage.py

@@ -0,0 +1,21 @@
+import os
+import shutil
+
+
+class LocalStorage:
+    def __init__(self, base_path: str):
+        self.base_path = base_path
+
+    def _full_path(self, path: str) -> str:
+        return os.path.join(self.base_path, path)
+
+    def upload(self, source: str, destination: str) -> None:
+        shutil.copy(source, self._full_path(destination))
+
+    def delete(self, file_path: str) -> None:
+        full_file_path = self._full_path(file_path)
+        if os.path.exists(full_file_path):
+            os.remove(full_file_path)
+
+    def get_file_url(self, file_name: str) -> str:
+        return file_name