Added first attempt entity class

Author: tristan-vt

.bandit

@@ -0,0 +1,2 @@
+[bandit]
+exclude: /tests

.flake8

@@ -0,0 +1,4 @@
+[flake8]
+ignore = E501,E722,E203,W503,E402
+max-line-length = 88
+exclude = tests/*

.pre-commit-config.yaml

@@ -0,0 +1,22 @@
+repos:
+-   repo: https://github.com/psf/black
+    rev: 19.10b0
+    hooks:
+    -   id: black
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.4.0
+    hooks:
+    -   id: end-of-file-fixer
+    -   id: trailing-whitespace
+    -   id: check-ast
+    -   id: check-json
+    -   id: check-merge-conflict
+    -   id: detect-private-key
+    -   id: flake8
+    -   id: pretty-format-json
+        args: [--autofix]
+-   repo: https://github.com/PyCQA/pydocstyle
+    rev: 5.0.2
+    hooks:
+    - id: pydocstyle
+      args: [--match, "(?!(tests/test_|docs/)).*\\.py"]

.pyup.yml

@@ -1,4 +1,4 @@
-# autogenerated pyup.io config file 
+# autogenerated pyup.io config file
 # see https://pyup.io/docs/configuration/ for all available options
 
 schedule: ''

README.rst

@@ -30,7 +30,7 @@ Flycs SDK contains the Python SDK to interact with the Flycs data framework.
 Features
 --------
 
-* TODO
+* Allows you to create the data objects needed to interact with Flycs.
 
 Credits
 -------

docs/conf.py

@@ -19,7 +19,8 @@
 #
 import os
 import sys
-sys.path.insert(0, os.path.abspath('..'))
+
+sys.path.insert(0, os.path.abspath(".."))
 
 import flycs_sdk
 
@@ -32,25 +33,25 @@
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
 extensions = [
-    'sphinx.ext.autodoc',
-    'sphinx.ext.viewcode',
-    'sphinx.ext.napoleon',
+    "sphinx.ext.autodoc",
+    "sphinx.ext.viewcode",
+    "sphinx.ext.napoleon",
 ]
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
+templates_path = ["_templates"]
 
 # The suffix(es) of source filenames.
 # You can specify multiple suffix as a list of string:
 #
 # source_suffix = ['.rst', '.md']
-source_suffix = '.rst'
+source_suffix = ".rst"
 
 # The master toctree document.
-master_doc = 'index'
+master_doc = "index"
 
 # General information about the project.
-project = 'Flycs SDK'
+project = "Flycs SDK"
 copyright = "2020, Tristan Van Thielen"
 author = "Tristan Van Thielen"
 
@@ -73,21 +74,20 @@
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
 # This patterns also effect to html_static_path and html_extra_path
-exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
 
 # The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
+pygments_style = "sphinx"
 
 # If true, `todo` and `todoList` produce output, else they produce nothing.
 todo_include_todos = False
 
-
 # -- Options for HTML output -------------------------------------------
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
-html_theme = 'alabaster'
+html_theme = "alabaster"
 
 # Theme options are theme-specific and customize the look and feel of a
 # theme further.  For a list of options available for each theme, see the
@@ -98,30 +98,25 @@
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
-
+html_static_path = ["_static"]
 
 # -- Options for HTMLHelp output ---------------------------------------
 
 # Output file base name for HTML help builder.
-htmlhelp_basename = 'flycs_sdkdoc'
-
+htmlhelp_basename = "flycs_sdkdoc"
 
 # -- Options for LaTeX output ------------------------------------------
 
 latex_elements = {
     # The paper size ('letterpaper' or 'a4paper').
     #
     # 'papersize': 'letterpaper',
-
     # The font size ('10pt', '11pt' or '12pt').
     #
     # 'pointsize': '10pt',
-
     # Additional stuff for the LaTeX preamble.
     #
     # 'preamble': '',
-
     # Latex figure (float) alignment
     #
     # 'figure_align': 'htbp',
@@ -131,36 +126,34 @@
 # (source start file, target name, title, author, documentclass
 # [howto, manual, or own class]).
 latex_documents = [
-    (master_doc, 'flycs_sdk.tex',
-     'Flycs SDK Documentation',
-     'Tristan Van Thielen', 'manual'),
+    (
+        master_doc,
+        "flycs_sdk.tex",
+        "Flycs SDK Documentation",
+        "Tristan Van Thielen",
+        "manual",
+    ),
 ]
 
-
 # -- Options for manual page output ------------------------------------
 
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
-man_pages = [
-    (master_doc, 'flycs_sdk',
-     'Flycs SDK Documentation',
-     [author], 1)
-]
-
+man_pages = [(master_doc, "flycs_sdk", "Flycs SDK Documentation", [author], 1)]
 
 # -- Options for Texinfo output ----------------------------------------
 
 # Grouping the document tree into Texinfo files. List of tuples
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-    (master_doc, 'flycs_sdk',
-     'Flycs SDK Documentation',
-     author,
-     'flycs_sdk',
-     'One line description of project.',
-     'Miscellaneous'),
+    (
+        master_doc,
+        "flycs_sdk",
+        "Flycs SDK Documentation",
+        author,
+        "flycs_sdk",
+        "One line description of project.",
+        "Miscellaneous",
+    ),
 ]
-
-
-

flycs_sdk/__init__.py

@@ -1,5 +1,5 @@
 """Top-level package for Flycs SDK."""
 
 __author__ = """Tristan Van Thielen"""
-__email__ = 'tristan.van.thielen@devoteam.com'
-__version__ = '0.4.0'
+__email__ = "tristan.van.thielen@devoteam.com"
+__version__ = "0.4.0"

flycs_sdk/cli.py

@@ -0,0 +1,159 @@
+"""Module containing entity classes."""
+
+from typing import Dict
+
+
+class Entity:
+    """Class that is used for converting an entity class to an Airflow DAG."""
+
+    def __init__(
+        self,
+        name: str,
+        version: str,
+        parameters: Dict[str, str] = None,
+        stage_config: Dict[str, Dict[str, str]] = None,
+    ):
+        """
+        Create an Entity object.
+
+        :param name: the name of the entity
+        :param version: the version of the entity, this can be used for table naming
+        :param parameters: the parameters as set in the running environment. These are inherited from the pipeline
+        this entity belongs to.
+        :param stage_config: a dictionary with the name of the stage as key and a dictionary of query names
+        and their versions as value.
+        """
+        self.name = name
+        self.version = version
+        self.parameters = parameters
+        self.stage_config = stage_config
+
+    def get_stage_versions(self, stage: str) -> Dict[str, str]:
+        """
+        Get the versions of the queries in the given stage.
+
+        :return: the versions of the queries in the given stage
+        """
+        return self.stage_config[stage]
+
+    def to_dict(self) -> Dict:
+        """
+        Serialize the entity to a dictionary object.
+
+        :return: the entity as a dictionary object.
+        """
+        return {
+            "name": self.name,
+            "version": self.version,
+            "stage_config": [
+                {"name": stage, "versions": self.get_stage_versions(stage)}
+                for stage in self.stage_config.keys()
+            ],
+        }
+
+
+class BaseLayerEntity(Entity):
+    """Class that is used for converting an entity class to an Airflow DAG."""
+
+    def __init__(
+        self,
+        name: str,
+        version: str,
+        parameters: Dict[str, str],
+        datalake_versions: Dict[str, str] = None,
+        preamble_versions: Dict[str, str] = None,
+        staging_versions: Dict[str, str] = None,
+        data_warehouse_versions: Dict[str, str] = None,
+        data_mart_versions: Dict[str, str] = None,
+    ):
+        """
+        Create an BaseLayerEntity object.
+
+        A BaseLayerEntity should be used in the case when the normal layer configuration is being used.
+        This means there are 5 layers: datalake, preamble, staging, data_warehouse and data_mart.
+
+        :param name: the name of the entity
+        :param version: the version of the entity, this can be used for table naming
+        :param parameters: the parameters as set in the running environment. These are inherited from the pipeline
+        this entity belongs to.
+        :param stage_config: a dictionary with the name of the stage as key and a dictionary of query names
+        and their versions as value.
+        """
+        self.datalake_versions = datalake_versions
+        self.preamble_versions = preamble_versions
+        self.staging_versions = staging_versions
+        self.data_warehouse_versions = data_warehouse_versions
+        self.data_mart_versions = data_mart_versions
+        super().__init__(name, version, parameters)
+
+    def get_stage_versions(self, stage: str) -> Dict[str, str]:
+        """
+        Get the versions of the queries in the given stage.
+
+        :return: the versions of the queries in the given stage
+        """
+        if stage == "datalake":
+            return self.get_datalake_versions()
+        elif stage == "preamble":
+            return self.get_preamble_versions()
+        if stage == "staging":
+            return self.get_staging_versions()
+        if stage == "data_warehouse":
+            return self.get_data_warehouse_versions()
+        if stage == "data_mart":
+            return self.get_data_mart_versions()
+
+    def get_datalake_versions(self) -> Dict[str, str]:
+        """
+        Get the versions of the queries in the datalake stage.
+
+        :return: the versions of the queries in the datalake stage
+        """
+        if self.datalake_versions is None:
+            return {}
+        else:
+            return self.datalake_versions
+
+    def get_preamble_versions(self) -> Dict[str, str]:
+        """
+        Get the versions of the queries in the preamble stage.
+
+        :return: the versions of the queries in the preamble stage
+        """
+        if self.preamble_versions is None:
+            return {}
+        else:
+            return self.preamble_versions
+
+    def get_staging_versions(self) -> Dict[str, str]:
+        """
+        Get the versions of the queries in the staging stage.
+
+        :return: the versions of the queries in the staging stage
+        """
+        if self.staging_versions is None:
+            return {}
+        else:
+            return self.staging_versions
+
+    def get_data_warehouse_versions(self) -> Dict[str, str]:
+        """
+        Get the versions of the queries in the data warehouse stage.
+
+        :return: the versions of the queries in the data warehouse stage
+        """
+        if self.data_warehouse_versions is None:
+            return {}
+        else:
+            return self.data_warehouse_versions
+
+    def get_data_mart_versions(self) -> Dict[str, str]:
+        """
+        Get the versions of the queries in the data mart stage.
+
+        :return: the versions of the queries in the data mart stage
+        """
+        if self.data_mart_versions is None:
+            return {}
+        else:
+            return self.data_mart_versions

flycs_sdk/entities.py

@@ -0,0 +1 @@
+"""Module containing all exceptions."""