[MAINTENANCE] run unit tests in docker

.github/workflows/run-integration-tests.yml

@@ -31,13 +31,5 @@ jobs:
       - name: check localstack  
         run: |
          curl http://localhost:4566/_localstack/health -i
-      - name: Build lambda image
-        run: |
-          cd ./functions/data_test
-          docker build -t data-test:latest .
-      - name: Build test image
-        run: |
-          cd ./tests/integration_tests/test_data_tests
-          docker build -t integration-tests:latest .
       - name: Run tests
-        run: docker run --env BUCKET=dqg-settings-local --env S3_HOST=172.17.0.1 --env S3_PORT=4566 integration-tests
+        run: make run-integration-tests QA_BUCKET=dqg-settings-local HOST=172.17.0.1

.github/workflows/run-unit-tests.yml

@@ -10,12 +10,5 @@ jobs:
     steps:
       - name: checkout
         uses: actions/checkout@v3  
-      - name: Set up Python  
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.7'
-          cache: 'pip'
-      - name: install dependieces  
-        run: make prepare-unit-tests
-      - name: run tests
-        run: make run-unit-tests
+      - name: run data test unit tests
+        run: make run-unit-tests-in-docker

Makefile

@@ -1,3 +1,14 @@
+HOST := host.docker.internal
+PORT := 4566
+QA_BUCKET := integration-test-bucket
+
+INTEGRATION_TESTS_DIR := ./tests/integration_tests/test_data_tests
+DATA_TEST_UNIT_TESTS_DIR := ./tests/unit_tests/data_test
+DATA_TEST_UNIT_TESTS_IMG := data_test_unit_tests
+DATA_TEST_INTEGRATION_TESTS_IMG := data_test_integration_tests
+DATA_TEST_IMAGE_NAME := data_test
+DATA_TEST_IMAGE_VERSION := latest
+
 run-localstack:
 	docker run --rm -d -p 4566:4566 -p 4510-4559:4510-4559 localstack/localstack:1.3.1
 
@@ -8,40 +19,25 @@ deploy-qa-infra:
 
 build-data-test-img:
 	cd ./functions/data_test && \
-	docker build -t data-test:latest .
-
-integration_tests_dir := ./tests/integration_tests/test_data_tests
-unit_tests_dir := ./tests/unit_tests
+	docker build -t $(DATA_TEST_IMAGE_NAME):$(DATA_TEST_IMAGE_VERSION) .
 
 build-data-test-tests-img: build-data-test-img
-	cd $(integration_tests_dir) && \
-	docker build -t test_data_tests .
-
-build-unit-tests-img: build-data-test-img
-	cd $(unit_tests_dir) && \
-	docker build -t unit_tests .	
-
-host := host.docker.internal
-port:= 4566
-qa_bucket = dqg-settings-local
+	cd $(INTEGRATION_TESTS_DIR) && \
+	docker build --build-arg="IMAGE_NAME=$(DATA_TEST_IMAGE_NAME)" \
+	--build-arg="VERSION=$(DATA_TEST_IMAGE_VERSION)" \
+	-t $(DATA_TEST_INTEGRATION_TESTS_IMG) .
 
 run-integration-tests: build-data-test-img build-data-test-tests-img
-	cd $(integration_tests_dir)
-	docker run --env BUCKET=$(qa_bucket) --env S3_HOST=$(host) --env S3_PORT=$(port) test_data_tests
-
-prepare-unit-tests:
-	cd ./functions/data_test && \
-	pip install -r requirements.txt && \
-	pip install pytest==7.2.1 && \
-	pip install moto==4.1.6
-
-run-unit-tests:
-	export ENVIRONMENT='local' && \
-	export S3_HOST='localhost' && \
-	export S3_PORT='4566' && \
-	export BUCKET='test-bucket' && \
-	export AWS_DEFAULT_REGION='us-east-1' && \
-	export REDSHIFT_DB='titanic' && \
-	export REDSHIFT_SECRET='titanic' && \
-	cd ./functions/data_test && \
-	python -m pytest ../../tests/unit_tests/data_test/ -v
+	cd $(INTEGRATION_TESTS_DIR)
+	docker run --env BUCKET=$(QA_BUCKET) \
+	--env S3_HOST=$(HOST) --env S3_PORT=$(PORT) $(DATA_TEST_INTEGRATION_TESTS_IMG)
+
+build-data-test-unit-tests-img: build-data-test-img
+	cd $(DATA_TEST_UNIT_TESTS_DIR) && \
+	docker build --build-arg="IMAGE_NAME=$(DATA_TEST_IMAGE_NAME)" \
+	--build-arg="VERSION=$(DATA_TEST_IMAGE_VERSION)" \
+	-t $(DATA_TEST_UNIT_TESTS_IMG) .
+
+run-unit-tests-in-docker: build-data-test-unit-tests-img
+	cd $(DATA_TEST_UNIT_TESTS_DIR) && \
+	docker run $(DATA_TEST_UNIT_TESTS_IMG)

tests/integration_tests/test_data_tests/Dockerfile

@@ -1,5 +1,7 @@
-FROM data-test:latest
-RUN pip3 install pytest==7.3.1
+ARG IMAGE_NAME=data-test
+ARG VERSION=latest
+FROM ${IMAGE_NAME}:${VERSION}
+RUN pip install pytest==7.3.1
 ENV ENVIRONMENT local
 ENV REPORTS_WEB test
 ENV AWS_ACCESS_KEY_ID test

tests/unit_tests/data_test/Dockerfile

@@ -0,0 +1,16 @@
+ARG IMAGE_NAME=data-test
+ARG VERSION=latest
+FROM ${IMAGE_NAME}:${VERSION}
+RUN pip install pytest==7.3.1 moto==4.1.6
+ENV ENVIRONMENT=local \
+    REPORTS_WEB=test \
+    AWS_ACCESS_KEY_ID=test \
+    AWS_SECRET_ACCESS_KEY=test \
+    AWS_DEFAULT_REGION=us-east-1 \
+    REDSHIFT_DB=test \
+    REDSHIFT_SECRET=test \
+    S3_HOST=localhost \
+    S3_PORT=9000 \
+    BUCKET=test-bucket
+COPY ./test*.py ./
+ENTRYPOINT ["pytest", "-qvs"]

tests/unit_tests/data_test/test_athena_datasource.py

@@ -1,4 +1,4 @@
-from functions.data_test.data_source_factory import AthenaDataSource
+from data_source_factory import AthenaDataSource
 from moto import mock_athena
 import pandas as pd
 
@@ -7,7 +7,7 @@ class TestAthenaDataSource:
     @mock_athena
     def test_athena_datasource(self):
         athena_data_source = AthenaDataSource("test-bucket", "test_table")
-        final_df, source = athena_data_source.read("s3://test-bucket/test-path/test_file.csv")
+        final_df, source = athena_data_source.read(
+            "s3://test-bucket/test-path/test_file.csv")
         assert final_df.equals(pd.DataFrame())
         assert source == "s3://test-bucket/test-path/test_file.csv"
-

tests/unit_tests/data_test/test_datasource.py

@@ -1,5 +1,5 @@
 import pytest
-from functions.data_test.datasource import (
+from datasource import (
   get_file_extension,
   concat_source_list,
   get_source_name)

tests/unit_tests/data_test/test_datasource_factory.py

@@ -1,5 +1,5 @@
 import pytest
-from functions.data_test.data_source_factory import (
+from data_source_factory import (
     DataSourceFactory,
     S3DataSource,
     HudiDataSource,

tests/unit_tests/data_test/test_profiling.py

@@ -1,5 +1,5 @@
 import pytest
-from functions.data_test.profiling import (add_local_s3_to_stores,
+from profiling import (add_local_s3_to_stores,
                                            read_gx_config_file)
 
 ENDPOINT_URL = "http://localhost:4566"

tests/unit_tests/data_test/test_redshift_datasource.py

@@ -2,7 +2,7 @@
 import moto
 import boto3
 import pytest
-from functions.data_test.data_source_factory import RedshiftDataSource
+from data_source_factory import RedshiftDataSource
 import awswrangler as wr
 from unittest.mock import patch