[MAINTENANCE] moved tests to lambda directories (#101)

* feat: moved tests to lambda dirs
provectus · Jun 30, 2023 · ac6b4b4 · ac6b4b4
1 parent 39dabdf
commit ac6b4b4
Show file tree

Hide file tree

Showing 35 changed files with 53 additions and 30 deletions.
diff --git a/Makefile b/Makefile
@@ -22,16 +22,13 @@ build-integration-tests-img: build-lambda-img
 	-t "$(test)_integration_tests" .
 
 run-integration-tests: build-integration-tests-img
-	cd ./tests/integration_tests/$(test) && \
 	docker run --env BUCKET=$(QA_BUCKET) \
 	--env S3_HOST=$(HOST) --env S3_PORT=$(PORT) $(test)_integration_tests
 
-build-unit-tests-img: build-lambda-img
-	cd ./tests/unit_tests/$(test) && \
-	docker build --build-arg="IMAGE_NAME=$(test)" \
-	--build-arg="VERSION=$(IMAGE_VERSION)" \
+build-unit-tests-img:
+	cd ./functions/$(test) && \
+	docker build --target=unit-tests \
 	-t $(test)_unit_tests .
 
 run-unit-tests: build-unit-tests-img
-	cd ./tests/unit_tests/$(test) && \
 	docker run $(test)_unit_tests
diff --git a/functions/allure_report/Dockerfile b/functions/allure_report/Dockerfile
@@ -1,4 +1,4 @@
-FROM public.ecr.aws/lambda/python:3.9
+FROM public.ecr.aws/lambda/python:3.9 as base
 RUN yum install -y unzip java-1.8.0-openjdk
 ENV JAVA_HOME=/usr/
 ADD ./requirements.txt ./requirements.txt
@@ -8,6 +8,14 @@ RUN mkdir allure && curl https://github.com/allure-framework/allure2/releases/do
 && curl https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip -O -L -C -\
 && unzip awscli-exe-linux-x86_64.zip && rm awscli-exe-linux-x86_64.zip \
 && ./aws/install
-COPY ./*.py ./
-COPY ./*.sh ./
+COPY ./allure_report/*.py ./
+COPY ./allure_report/*.sh ./
+
+FROM base as unit-tests
+RUN pip install pytest==7.3.1
+ENV BUCKET=test-bucket
+COPY ./tests/test*.py ./
+ENTRYPOINT ["pytest", "-qvs"]
+
+FROM base as production
 CMD ["make_allure_report.handler"]
diff --git a/functions/allure_report/generate_report.sh → ...e_report/allure_report/generate_report.sh b/functions/allure_report/generate_report.sh → ...e_report/allure_report/generate_report.sh
diff --git a/...tions/allure_report/make_allure_report.py → ...eport/allure_report/make_allure_report.py b/...tions/allure_report/make_allure_report.py → ...eport/allure_report/make_allure_report.py
diff --git a/functions/allure_report/mapper.py → ...ons/allure_report/allure_report/mapper.py b/functions/allure_report/mapper.py → ...ons/allure_report/allure_report/mapper.py
diff --git a/tests/unit_tests/allure_report/Dockerfile → functions/allure_report/tests/Dockerfile b/tests/unit_tests/allure_report/Dockerfile → functions/allure_report/tests/Dockerfile
diff --git a/...s/unit_tests/allure_report/test_mapper.py → functions/allure_report/tests/test_mapper.py b/...s/unit_tests/allure_report/test_mapper.py → functions/allure_report/tests/test_mapper.py
diff --git a/functions/data_test/Dockerfile b/functions/data_test/Dockerfile
@@ -1,8 +1,25 @@
-FROM public.ecr.aws/lambda/python:3.9
+FROM public.ecr.aws/lambda/python:3.9 as base
 COPY requirements.txt ./
 RUN pip3 install -r requirements.txt --no-cache-dir
-COPY ./*.py ./
-COPY great_expectations ./great_expectations
+COPY ./data_test/*.py ./
+COPY ./data_test/great_expectations ./great_expectations
 RUN rm -f  /etc/timezone
 RUN python3 -c "import tzlocal; tzlocal.get_localzone()"
+
+FROM base as unit-tests
+RUN pip install pytest==7.3.1 moto==4.1.6
+ENV ENVIRONMENT=local \
+    REPORTS_WEB=test \
+    AWS_ACCESS_KEY_ID=test \
+    AWS_SECRET_ACCESS_KEY=test \
+    AWS_DEFAULT_REGION=us-east-1 \
+    REDSHIFT_DB=test \
+    REDSHIFT_SECRET=test \
+    S3_HOST=localhost \
+    S3_PORT=9000 \
+    BUCKET=test-bucket
+COPY ./tests/test*.py ./
+ENTRYPOINT ["pytest","-qvs"]
+
+FROM base as production
 CMD ["data_test.handler"]
diff --git a/...tions/data_test/Expectation_report_new.py → ..._test/data_test/Expectation_report_new.py b/...tions/data_test/Expectation_report_new.py → ..._test/data_test/Expectation_report_new.py
diff --git a/functions/data_test/data_source_factory.py → ...ata_test/data_test/data_source_factory.py b/functions/data_test/data_source_factory.py → ...ata_test/data_test/data_source_factory.py
diff --git a/functions/data_test/data_test.py → functions/data_test/data_test/data_test.py b/functions/data_test/data_test.py → functions/data_test/data_test/data_test.py
diff --git a/functions/data_test/datasource.py → functions/data_test/data_test/datasource.py b/functions/data_test/datasource.py → functions/data_test/data_test/datasource.py
diff --git a/...s/data_test/great_expectations/.gitignore → ...t/data_test/great_expectations/.gitignore b/...s/data_test/great_expectations/.gitignore → ...t/data_test/great_expectations/.gitignore
diff --git a/...ata_test/great_expectations/__ init__ .py → ...ata_test/great_expectations/__ init__ .py b/...ata_test/great_expectations/__ init__ .py → ...ata_test/great_expectations/__ init__ .py
diff --git a/...tations/expectations/.ge_store_backend_id → ...tations/expectations/.ge_store_backend_id b/...tations/expectations/.ge_store_backend_id → ...tations/expectations/.ge_store_backend_id
diff --git a/...great_expectations/great_expectations.yml → ...great_expectations/great_expectations.yml b/...great_expectations/great_expectations.yml → ...great_expectations/great_expectations.yml
diff --git a/...ebooks/pandas/validation_playground.ipynb → ...ebooks/pandas/validation_playground.ipynb b/...ebooks/pandas/validation_playground.ipynb → ...ebooks/pandas/validation_playground.ipynb
diff --git a/...tebooks/spark/validation_playground.ipynb → ...tebooks/spark/validation_playground.ipynb b/...tebooks/spark/validation_playground.ipynb → ...tebooks/spark/validation_playground.ipynb
diff --git a/...notebooks/sql/validation_playground.ipynb → ...notebooks/sql/validation_playground.ipynb b/...notebooks/sql/validation_playground.ipynb → ...notebooks/sql/validation_playground.ipynb
diff --git a/...ns/custom_data_docs/renderers/__init__.py → ...ns/custom_data_docs/renderers/__init__.py b/...ns/custom_data_docs/renderers/__init__.py → ...ns/custom_data_docs/renderers/__init__.py
diff --git a/...a_docs/renderers/custom_table_renderer.py → ...a_docs/renderers/custom_table_renderer.py b/...a_docs/renderers/custom_table_renderer.py → ...a_docs/renderers/custom_table_renderer.py
diff --git a/...a_docs/styles/data_docs_custom_styles.css → ...a_docs/styles/data_docs_custom_styles.css b/...a_docs/styles/data_docs_custom_styles.css → ...a_docs/styles/data_docs_custom_styles.css
diff --git a/functions/data_test/profiling.py → functions/data_test/data_test/profiling.py b/functions/data_test/profiling.py → functions/data_test/data_test/profiling.py
diff --git a/functions/data_test/suite_run.py → functions/data_test/data_test/suite_run.py b/functions/data_test/suite_run.py → functions/data_test/data_test/suite_run.py
diff --git a/tests/unit_tests/data_test/Dockerfile → functions/data_test/tests/Dockerfile b/tests/unit_tests/data_test/Dockerfile → functions/data_test/tests/Dockerfile
diff --git a/...tests/data_test/test_athena_datasource.py → ...data_test/tests/test_athena_datasource.py b/...tests/data_test/test_athena_datasource.py → ...data_test/tests/test_athena_datasource.py
diff --git a/...s/unit_tests/data_test/test_datasource.py → functions/data_test/tests/test_datasource.py b/...s/unit_tests/data_test/test_datasource.py → functions/data_test/tests/test_datasource.py
diff --git a/...ests/data_test/test_datasource_factory.py → ...ata_test/tests/test_datasource_factory.py b/...ests/data_test/test_datasource_factory.py → ...ata_test/tests/test_datasource_factory.py
diff --git a/tests/unit_tests/data_test/test_profiling.py → functions/data_test/tests/test_profiling.py b/tests/unit_tests/data_test/test_profiling.py → functions/data_test/tests/test_profiling.py
diff --git a/...sts/data_test/test_redshift_datasource.py → ...ta_test/tests/test_redshift_datasource.py b/...sts/data_test/test_redshift_datasource.py → ...ta_test/tests/test_redshift_datasource.py
diff --git a/functions/report_push/Dockerfile b/functions/report_push/Dockerfile
@@ -1,5 +1,22 @@
-FROM public.ecr.aws/lambda/python:3.9
+FROM public.ecr.aws/lambda/python:3.9 as base
 ADD ./requirements.txt ./requirements.txt
 RUN pip3 install -r requirements.txt
-COPY ./*.py ./
+COPY ./report_push/*.py ./
+
+FROM base as unit-tests
+RUN pip install pytest==7.3.1 moto==4.1.6
+ENV ENVIRONMENT=local \
+    BUCKET=test-bucket \
+    DYNAMODB_TABLE=test-table \
+    AWS_ACCESS_KEY_ID=test \
+    AWS_SECRET_ACCESS_KEY=test \
+    AWS_DEFAULT_REGION=us-east-1 \
+    SNS_BUGS_TOPIC_ARN=test-topic-arn \
+    JIRA_URL=https://jira.localhost \
+    DATAQA_JIRA_USERNAME=qauser \
+    DATAQA_JIRA_PASSWORD=qapassword
+COPY ./tests/test*.py ./
+ENTRYPOINT ["pytest", "-qvs"]
+
+FROM base as production
 CMD ["push_data_report.handler"]
diff --git a/functions/report_push/jira_events.py → ...ns/report_push/report_push/jira_events.py b/functions/report_push/jira_events.py → ...ns/report_push/report_push/jira_events.py
diff --git a/functions/report_push/push_data_report.py → ...port_push/report_push/push_data_report.py b/functions/report_push/push_data_report.py → ...port_push/report_push/push_data_report.py
diff --git a/...ests/report_push/test_push_data_report.py → ...eport_push/tests/test_push_data_report.py b/...ests/report_push/test_push_data_report.py → ...eport_push/tests/test_push_data_report.py
diff --git a/tests/unit_tests/report_push/Dockerfile b/tests/unit_tests/report_push/Dockerfile