Feature/optimize package

.circleci/config.yml

@@ -1,75 +1,130 @@
 version: 2.1
 executors:
-  docker-conda-executor:
+  docker-python-executor:
     docker:
-      - image: continuumio/miniconda3:latest
+      - image: circleci/python:3.9.5
 
 commands:
-  activate_conda:
+  update:
+    description: "Installing ubuntu prerequisites"
     steps:
       - run: |
-          conda init bash
-          source ~/.bashrc
-          conda env create -f environment.yml
-          conda activate houses
-  run_tests:
+          sudo apt-get update -y
+          sudo apt-get install make
+          sudo apt-get install unzip
+
+  prepare_pipeline_virtual_environment:
+    description: "Preparing the pipeline virtual environment"
     steps:
       - run: |
-          conda init bash
-          source ~/.bashrc
-          conda activate houses
-          pytest houses_pipeline
-          pytest houses_api
+          python3 -m venv venv
+          source venv/bin/activate
+          pip install --upgrade pip
 
-  train_lasso_model:
+  install_pipeline_requirements:
+    description: "Installing pipeline requirements"
     steps:
-      - checkout
       - run: |
-          conda init bash
-          source ~/.bashrc
-          conda activate houses
-          python -m houses_pipeline.modelling.train_lasso
+          source venv/bin/activate
+          . venv/bin/activate
+          pip install -r requirements.txt
 
-  fetch_dataset:
+  run_pipeline_lint:
+    description: "Run pipeline linting"
     steps:
-      - checkout
       - run: |
-          conda init bash
-          source ~/.bashrc
-          conda activate houses
-          chmod +x ./houses_pipeline/fetch/fetch_dataset.sh
-          ./houses_pipeline/fetch/fetch_dataset.sh data/raw
+          source venv/bin/activate
+          . venv/bin/activate
+          pylint houses_pipeline
+
+  run_pipeline_tests:
+    description: "Run pipeline tests"
+    steps:
+      - run: |
+          source venv/bin/activate
+          . venv/bin/activate
+          make test
 
-  preprocess_data:
+  train_and_publish_houses_model:
+    description: "Train the houses model"
+    steps:
+      - run: |
+          source venv/bin/activate
+          . venv/bin/activate
+          make publish
+
+  prepare_api_virtual_environment:
+    description: "Preparing the api virtual environment"
+    steps:
+      - run: |
+          python3 -m venv venv
+          source venv/bin/activate
+          pip install --upgrade pip
+
+  install_api_requirements:
+    description: "Installing API requirements"
+    steps:
+      - run: |
+          . venv/bin/activate
+          pip install -r requirements.txt
+
+  run_api_tests:
+    description: "Run api tests"
     steps:
-      - checkout
       - run: |
-          conda init bash
-          source ~/.bashrc
-          conda activate houses
-          python -m houses_pipeline.preprocess
+          . venv/bin/activate
+          pytest .
+
+  prepare_tox:
+    steps:
+      - run: |
+          sudo pip install --upgrade pip
+          pip install --user tox
 
 jobs:
-  setup_and_run_tests:
-    executor: docker-conda-executor
-    working_directory: ~/project
-    docker:
-      - image: continuumio/miniconda3:latest
+  setup_and_test_houses_pipeline:
+    executor: docker-python-executor
+    working_directory: ~/project/pipeline
+    steps:
+      - checkout:
+          path: ~/project
+      - update
+      - prepare_pipeline_virtual_environment
+      - install_pipeline_requirements
+      - run_pipeline_lint
+      - run_pipeline_tests
+      - train_and_publish_houses_model
+
+  setup_and_test_houses_api:
+    executor: docker-python-executor
+    working_directory: ~/project/api
+    steps:
+      - checkout:
+          path: ~/project
+      - update
+      - prepare_api_virtual_environment
+      - install_api_requirements
+      - run_api_tests
+
+  train_and_upload_houses_model:
+    executor: docker-python-executor
+    working_directory: ~/project/pipeline
     steps:
       - checkout
-      - activate_conda
-      # - TODO: ADD this fetch_dataset
-      - fetch_dataset
-      - preprocess_data
-      - train_lasso_model
-      - run_tests
+      - update
+      - prepare_houses_virtual_environment
+      - install_houses_requirements
+      - train_houses_model
 
-# TODO: add build workflow
 workflows:
   version: 2
   houses_workflow:
     jobs:
-      - setup_and_run_tests:
+      - setup_and_test_houses_pipeline:
           context:
             - kaggling
-
+            - gemfury
+      - setup_and_test_houses_api:
+          context:
+            - kaggling
+            - gemfury

.gitignore

@@ -1,4 +1,3 @@
-houses.egg-info
 mlruns
 build
 dist
@@ -21,3 +20,7 @@ models/*.pkl
 
 logs/*.log*
 !logs/.gitkeep
+# virtual environments
+venv
+pipeline_venv
+api_venv

Makefile

@@ -0,0 +1,53 @@
+all: lint, test, train, publish
+
+test:
+	@python -m pytest
+
+lint:
+	echo "Linting the entire project"
+	@pylint houses_pipeline
+	@pylint houses_api
+
+install_pipeline_develop:
+	pip install -e ./houses_pipeline
+
+clean:
+	@echo "Cleaning up $(project_name)"
+	@rm -rf *.egg-info
+	@rm -rf $(raws_to_remove)
+	@rm -rf data/**/*.csv
+	@rm -rf logs/*.log*
+
+# functionality
+train: clean preprocess
+	@echo "Training a model for the $(project_name) project"
+	@python -m $(pipeline_dir).modelling.lasso.py
+
+preprocess: fetch
+	@echo "Preprocessing the data for the $(project_name) project"
+	@mkdir -p $(pipeline_dir)/data/raw data/interim data/proccessed
+	@python -m $(pipeline_dir).preprocess data/raw/train.csv data/interim/train.csv
+	@python -m $(pipeline_dir).preprocess data/raw/test.csv data/interim/test.csv
+
+fetch:
+	@mkdir -p data/raw data/interim data/proccessed
+	@chmod 701 ./$(pipeline_dir)/fetch/fetch_dataset.sh
+	@./$(pipeline_dir)/fetch/fetch_dataset.sh data/raw
+
+clean_pipeline:
+	@echo "Cleaning up generated data, logs and models"
+	@rm -rf data/**/*.csv
+	@rm -rf models/**/*.pkl
+	# @rm -rf logs/*.log*
+
+publish: train
+	.$(houses_pipeline)/scripts/publish_model.sh .
+
+
+# ignore the following
+# .PHONY: preprocess, test, fetch
+
+# variables
+project_name = houses_pipeline
+pipeline_dir = houses_pipeline
+raws_to_remove = data/raw/*.csv, data/raw/*.zip, data/raw/*.txt

api/requirements.txt

@@ -0,0 +1,14 @@
+--index-url https://${FURY_AUTH}:@pypi.fury.io/nshahpazov/
+--extra-index-url https://pypi.org/simple houses_pipeline
+
+# api
+flask==2.2.2
+# schema validation
+marshmallow==3.17.1
+
+# # tests
+pytest==7.1.2
+
+# gemfury stored module
+houses_pipeline==0.0.31
+joblib==1.1.0

houses_api/tests/test_controller.py → api/tests/test_controller.py

@@ -1,16 +1,20 @@
 """Controller setup for the testing context"""
-import os
 import json
 import math
 import pandas as pd
 from houses_pipeline import __version__ as _model_version
+from houses_pipeline.fetch import fetch_houses_dataset
+from houses_pipeline.preprocess.core import preprocess
+from houses_pipeline.modelling import lasso
+
+# from houses_pipeline.fetch import fetch_houses_dataset
+# from houses_pipeline.preprocess import preproces
 from .. import __version__ as _api_version
 
 from ..api.config import get_logger
 from ..api.config import TEST_DATASET_PATH
 
 
-
 _logger = get_logger(logger_name=__name__)
 
 def test_health_endpoint_returns_ok_status(test_client):
@@ -48,28 +52,25 @@ def test_prediction_endpoint_returns_prediction(test_client):
     across packages.
     """
 
-    # fetch data
-    os.system("./houses_pipeline/fetch/fetch_dataset.sh data/raw")
+    fetch_houses_dataset()
 
     # preprocess
-    os.system("python -m houses_pipeline.preprocess")
-    os.system(
-        "python -m houses_pipeline.preprocess data/raw/test.csv data/interim/test.csv"
-    )
+    preprocess("data/raw/train.csv", "data/interim/train.csv")
+    preprocess("data/raw/test.csv", "data/interim/test.csv")
 
-    # train a model
-    os.system("python -m houses_pipeline.modelling.train_lasso")
+    # train a model and save it
+    lasso.train()
 
     test_data = pd.read_csv(TEST_DATASET_PATH)
     post_json = test_data[0:1].to_json(orient='records')
     # when
+
     response = test_client.post('/predict/lasso', json=json.loads(post_json))
 
     # then
     assert response.status_code == 200
     response_json = json.loads(response.data)
     prediction = response_json['predictions']
-    print(response_json)
     response_version = response_json['version']
     assert math.ceil(prediction[0]) == 117205
     assert response_version == _model_version