-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathMakefile
106 lines (83 loc) · 4.3 KB
/
Makefile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
.DEFAULT_GOAL := default
#################### PACKAGE ACTIONS ###################
reinstall_package:
@pip uninstall -y taxifare || :
@pip install -e .
run_preprocess:
python -c 'from taxifare.interface.main import preprocess; preprocess()'
run_train:
python -c 'from taxifare.interface.main import train; train()'
run_pred:
python -c 'from taxifare.interface.main import pred; pred()'
run_evaluate:
python -c 'from taxifare.interface.main import evaluate; evaluate()'
run_all: run_preprocess run_train run_pred run_evaluate
run_workflow:
PREFECT__LOGGING__LEVEL=${PREFECT_LOG_LEVEL} python -m taxifare.interface.workflow
run_api:
uvicorn taxifare.api.fast:app --reload
##################### TESTS #####################
test_gcp_setup:
@pytest \
tests/all/test_gcp_setup.py::TestGcpSetup::test_setup_key_env \
tests/all/test_gcp_setup.py::TestGcpSetup::test_setup_key_path \
tests/all/test_gcp_setup.py::TestGcpSetup::test_code_get_project \
tests/all/test_gcp_setup.py::TestGcpSetup::test_code_get_wagon_project
default:
cat tests/api/test_output.txt
test_kitt:
@echo "\n 🧪 computing and saving your progress at 'tests/api/test_output.txt'..."
@pytest tests/api -c "./tests/pytest_kitt.ini" 2>&1 > tests/api/test_output.txt || true
@echo "\n 🙏 Please: \n git add tests \n git commit -m 'checkpoint' \n ggpush"
test_api_root:
pytest \
tests/api/test_endpoints.py::test_root_is_up --asyncio-mode=strict -W "ignore" \
tests/api/test_endpoints.py::test_root_returns_greeting --asyncio-mode=strict -W "ignore"
test_api_predict:
pytest \
tests/api/test_endpoints.py::test_predict_is_up --asyncio-mode=strict -W "ignore" \
tests/api/test_endpoints.py::test_predict_is_dict --asyncio-mode=strict -W "ignore" \
tests/api/test_endpoints.py::test_predict_has_key --asyncio-mode=strict -W "ignore" \
tests/api/test_endpoints.py::test_predict_val_is_float --asyncio-mode=strict -W "ignore"
test_api_on_prod:
pytest \
tests/api/test_cloud_endpoints.py --asyncio-mode=strict -W "ignore"
################### DATA SOURCES ACTIONS ################
# Data sources: targets for monthly data imports
ML_DIR=~/.lewagon/mlops
HTTPS_DIR=https://storage.googleapis.com/datascience-mlops/taxi-fare-ny/
GS_DIR=gs://datascience-mlops/taxi-fare-ny
show_sources_all:
-ls -laR ~/.lewagon/mlops/data
-bq ls ${BQ_DATASET}
-bq show ${BQ_DATASET}.processed_1k
-bq show ${BQ_DATASET}.processed_200k
-bq show ${BQ_DATASET}.processed_all
-gsutil ls gs://${BUCKET_NAME}
reset_local_files:
rm -rf ${ML_DIR}
mkdir -p ~/.lewagon/mlops/data/
mkdir ~/.lewagon/mlops/data/raw
mkdir ~/.lewagon/mlops/data/processed
mkdir ~/.lewagon/mlops/training_outputs
mkdir ~/.lewagon/mlops/training_outputs/metrics
mkdir ~/.lewagon/mlops/training_outputs/models
mkdir ~/.lewagon/mlops/training_outputs/params
reset_local_files_with_csv_solutions: reset_local_files
-curl ${HTTPS_DIR}solutions/data_query_fixture_2009-01-01_2015-01-01_1k.csv > ${ML_DIR}/data/raw/query_2009-01-01_2015-01-01_1k.csv
-curl ${HTTPS_DIR}solutions/data_query_fixture_2009-01-01_2015-01-01_200k.csv > ${ML_DIR}/data/raw/query_2009-01-01_2015-01-01_200k.csv
-curl ${HTTPS_DIR}solutions/data_query_fixture_2009-01-01_2015-01-01_all.csv > ${ML_DIR}/data/raw/query_2009-01-01_2015-01-01_all.csv
-curl ${HTTPS_DIR}solutions/data_processed_fixture_2009-01-01_2015-01-01_1k.csv > ${ML_DIR}/data/processed/processed_2009-01-01_2015-01-01_1k.csv
-curl ${HTTPS_DIR}solutions/data_processed_fixture_2009-01-01_2015-01-01_200k.csv > ${ML_DIR}/data/processed/processed_2009-01-01_2015-01-01_200k.csv
-curl ${HTTPS_DIR}solutions/data_processed_fixture_2009-01-01_2015-01-01_all.csv > ${ML_DIR}/data/processed/processed_2009-01-01_2015-01-01_all.csv
reset_bq_files:
-bq rm --project_id ${GCP_PROJECT} ${BQ_DATASET}.processed_1k
-bq rm --project_id ${GCP_PROJECT} ${BQ_DATASET}.processed_200k
-bq rm --project_id ${GCP_PROJECT} ${BQ_DATASET}.processed_all
-bq mk --sync --project_id ${GCP_PROJECT} --location=${BQ_REGION} ${BQ_DATASET}.processed_1k
-bq mk --sync --project_id ${GCP_PROJECT} --location=${BQ_REGION} ${BQ_DATASET}.processed_200k
-bq mk --sync --project_id ${GCP_PROJECT} --location=${BQ_REGION} ${BQ_DATASET}.processed_all
reset_gcs_files:
-gsutil rm -r gs://${BUCKET_NAME}
-gsutil mb -p ${GCP_PROJECT} -l ${GCP_REGION} gs://${BUCKET_NAME}
reset_all_files: reset_local_files reset_bq_files reset_gcs_files