From b62725a1bc84316d09cfc2c18a6e3be1798955d7 Mon Sep 17 00:00:00 2001 From: Irwan Fathurrahman Date: Wed, 2 Oct 2024 14:40:11 +0700 Subject: [PATCH] Add tio collector and save it to zipping file (#162) * Add tio collector and save it to zipping file * Fix tests * Add minio to tests * Autocreate bucket of minio on test * Fix flake * Add test for tio shortterm collector * Fix tests * Fix flake --- deployment/docker-compose.test.yml | 23 ++ django_project/core/settings/project.py | 12 +- django_project/core/settings/test.py | 14 - django_project/core/tests/runner.py | 2 + django_project/core/tests/test_s3.py | 42 +++ django_project/core/utils/__init__.py | 0 django_project/core/utils/s3.py | 82 +++++ django_project/gap/fixtures/7.attribute.json | 33 ++ .../gap/fixtures/8.dataset_attribute.json | 33 ++ django_project/gap/ingestor/tio_shortterm.py | 119 +++++++ ...collectorsession_ingestor_type_and_more.py | 35 ++ django_project/gap/models/dataset.py | 2 +- django_project/gap/models/farm_group.py | 4 +- django_project/gap/models/ingestor.py | 8 + django_project/gap/providers/tio.py | 29 +- .../data/tio_shorterm_collector/test.json | 193 ++++++++++ .../ingestor/test_tio_shortterm_collector.py | 330 ++++++++++++++++++ django_project/gap/tests/utils/test_netcdf.py | 5 +- django_project/gap/utils/reader.py | 8 +- .../spw/tests/test_crop_insight_generator.py | 2 +- 20 files changed, 949 insertions(+), 27 deletions(-) create mode 100644 django_project/core/tests/test_s3.py create mode 100644 django_project/core/utils/__init__.py create mode 100644 django_project/core/utils/s3.py create mode 100644 django_project/gap/ingestor/tio_shortterm.py create mode 100644 django_project/gap/migrations/0026_alter_collectorsession_ingestor_type_and_more.py create mode 100644 django_project/gap/tests/ingestor/data/tio_shorterm_collector/test.json create mode 100644 django_project/gap/tests/ingestor/test_tio_shortterm_collector.py diff --git a/deployment/docker-compose.test.yml b/deployment/docker-compose.test.yml index 8e8e223..5e66988 100644 --- a/deployment/docker-compose.test.yml +++ b/deployment/docker-compose.test.yml @@ -3,18 +3,34 @@ volumes: static-data: media-data: + minio-data: # Exactly the same as production but for dev env, we expose the port and uses # different port for the web. version: '3.4' services: + minio: + image: quay.io/minio/minio:RELEASE.2024-03-30T09-41-56Z.fips + command: minio server /data --console-address ":9001" + ports: + - "9010:9000" + - "9011:9001" + environment: + - MINIO_ROOT_USER=minio_user + - MINIO_ROOT_PASSWORD=minio_password + - MINIO_HTTP_TRACE + volumes: + - minio-data:/data + restart: always + dev: image: ${APP_IMAGE}:dev container_name: "dev" links: - db - redis + - minio volumes: - static-data:/home/web/static - media-data:/home/web/media @@ -41,6 +57,13 @@ services: - ADMIN_EMAIL=admin@example.com - SENTRY_DSN= - SENTRY_ENVIRONMENT=staging + + # Minio + - MINIO_AWS_ACCESS_KEY_ID=minio_user + - MINIO_AWS_SECRET_ACCESS_KEY=minio_password + - MINIO_AWS_ENDPOINT_URL=http://minio:9000/ + - MINIO_AWS_BUCKET_NAME=tomorrownow + - MINIO_AWS_DIR_PREFIX=dev/media entrypoint: [ ] ports: # for django test server diff --git a/django_project/core/settings/project.py b/django_project/core/settings/project.py index 64d26d7..9f9e7f8 100644 --- a/django_project/core/settings/project.py +++ b/django_project/core/settings/project.py @@ -47,17 +47,21 @@ use_threads=True, max_concurrency=10 ) +MINIO_AWS_ACCESS_KEY_ID = os.environ.get("MINIO_AWS_ACCESS_KEY_ID") +MINIO_AWS_SECRET_ACCESS_KEY = os.environ.get("MINIO_AWS_SECRET_ACCESS_KEY") +MINIO_AWS_BUCKET_NAME = os.environ.get("MINIO_AWS_BUCKET_NAME") +MINIO_AWS_ENDPOINT_URL = os.environ.get("MINIO_AWS_ENDPOINT_URL") STORAGES = { "default": { "BACKEND": "storages.backends.s3.S3Storage", "OPTIONS": { - "access_key": os.environ.get("MINIO_AWS_ACCESS_KEY_ID"), - "secret_key": os.environ.get("MINIO_AWS_SECRET_ACCESS_KEY"), - "bucket_name": os.environ.get("MINIO_AWS_BUCKET_NAME"), + "access_key": MINIO_AWS_ACCESS_KEY_ID, + "secret_key": MINIO_AWS_SECRET_ACCESS_KEY, + "bucket_name": MINIO_AWS_BUCKET_NAME, "file_overwrite": False, "max_memory_size": 300 * MB, # 300MB "transfer_config": AWS_TRANSFER_CONFIG, - "endpoint_url": os.environ.get("MINIO_AWS_ENDPOINT_URL") + "endpoint_url": MINIO_AWS_ENDPOINT_URL }, }, "staticfiles": { diff --git a/django_project/core/settings/test.py b/django_project/core/settings/test.py index 18799c7..8a78a62 100644 --- a/django_project/core/settings/test.py +++ b/django_project/core/settings/test.py @@ -16,18 +16,4 @@ 'BACKEND': 'django.core.cache.backends.dummy.DummyCache', } } - -STORAGES = { - "default": { - "BACKEND": "django.core.files.storage.FileSystemStorage", - "OPTIONS": { - "location": "/home/web/media/default_test", - }, - }, - "staticfiles": { - "BACKEND": ( - "django.contrib.staticfiles.storage.ManifestStaticFilesStorage" - ), - } -} EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' diff --git a/django_project/core/tests/runner.py b/django_project/core/tests/runner.py index 301497b..719cbad 100644 --- a/django_project/core/tests/runner.py +++ b/django_project/core/tests/runner.py @@ -9,6 +9,7 @@ from django.test.runner import DiscoverRunner from core.celery import app as celery_app +from core.utils.s3 import create_s3_bucket class CustomTestRunner(DiscoverRunner): @@ -31,4 +32,5 @@ def __disable_celery(): def setup_test_environment(self, **kwargs): """Prepare test env.""" CustomTestRunner.__disable_celery() + create_s3_bucket(settings.MINIO_AWS_BUCKET_NAME) super(CustomTestRunner, self).setup_test_environment(**kwargs) diff --git a/django_project/core/tests/test_s3.py b/django_project/core/tests/test_s3.py new file mode 100644 index 0000000..436052c --- /dev/null +++ b/django_project/core/tests/test_s3.py @@ -0,0 +1,42 @@ +# coding=utf-8 +""" +Tomorrow Now GAP. + +.. note:: Unit test for S3 utils. +""" +import os + +from django.conf import settings +from django.core.files.base import ContentFile +from django.core.files.storage import default_storage +from django.test import TestCase + +from core.utils.s3 import zip_folder_in_s3, remove_s3_folder, create_s3_bucket + + +class TestS3Utilities(TestCase): + """Test S3 utilities.""" + + def test_bucket_already_created(self): + """Test S3 bucket already created.""" + self.assertFalse(create_s3_bucket(settings.MINIO_AWS_BUCKET_NAME)) + + def test_zip_folder_in_s3(self): + """Test zip folder in S3.""" + folder = 'test_folder' + remove_s3_folder(default_storage, folder) + default_storage.save( + os.path.join(folder, 'test'), ContentFile(b"new content") + ) + default_storage.save( + os.path.join(folder, 'test_2'), ContentFile(b"new content") + ) + zip_folder_in_s3( + default_storage, folder, 'test_folder.zip' + ) + self.assertTrue( + default_storage.exists('test_folder.zip') + ) + self.assertFalse( + default_storage.exists(folder) + ) diff --git a/django_project/core/utils/__init__.py b/django_project/core/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/django_project/core/utils/s3.py b/django_project/core/utils/s3.py new file mode 100644 index 0000000..d632a26 --- /dev/null +++ b/django_project/core/utils/s3.py @@ -0,0 +1,82 @@ +# coding=utf-8 +""" +Tomorrow Now GAP. + +.. note:: Utilities for S3. +""" + +import io +import zipfile + +import boto3 +from botocore.exceptions import ClientError +from django.conf import settings +from django.core.files.base import ContentFile +from storages.backends.s3boto3 import S3Boto3Storage + + +def zip_folder_in_s3( + s3_storage: S3Boto3Storage, folder_path: str, zip_file_name: str +): + """Zip folder contents into a zip file on S3.""" + zip_buffer = io.BytesIO() + + if s3_storage.exists(zip_file_name): + s3_storage.delete(zip_file_name) + + # Create buffer zip file + with zipfile.ZipFile(zip_buffer, 'w') as zip_file: + # Get file list + files_in_folder = s3_storage.bucket.objects.filter( + Prefix=folder_path + ) + + for s3_file in files_in_folder: + file_name = s3_file.key.split('/')[-1] + if not file_name: + continue + + # Read the file and add to zip file + file_content = s3_file.get()['Body'].read() + zip_file.writestr(file_name, file_content) + + # Save it to S3 + zip_buffer.seek(0) + s3_storage.save(zip_file_name, ContentFile(zip_buffer.read())) + remove_s3_folder(s3_storage, folder_path) + + +def remove_s3_folder(s3_storage: S3Boto3Storage, folder_path: str): + """Remove folder from S3 storage.""" + if not folder_path.endswith('/'): + folder_path += '/' + + # Get all file in the folder and remove one by one + bucket = s3_storage.bucket + objects_to_delete = bucket.objects.filter(Prefix=folder_path) + for obj in objects_to_delete: + obj.delete() + + +def create_s3_bucket(bucket_name, region=None): + """Create an S3 bucket in a specified region.""" + # Create bucket + try: + s3_client = boto3.client( + 's3', + region_name=region, + endpoint_url=settings.MINIO_AWS_ENDPOINT_URL, + aws_access_key_id=settings.MINIO_AWS_ACCESS_KEY_ID, + aws_secret_access_key=settings.MINIO_AWS_SECRET_ACCESS_KEY + ) + if region is None: + s3_client.create_bucket(Bucket=bucket_name) + else: + location = {'LocationConstraint': region} + s3_client.create_bucket( + Bucket=bucket_name, + CreateBucketConfiguration=location + ) + except ClientError: + return False + return True diff --git a/django_project/gap/fixtures/7.attribute.json b/django_project/gap/fixtures/7.attribute.json index 12ce3be..5a4b3a7 100755 --- a/django_project/gap/fixtures/7.attribute.json +++ b/django_project/gap/fixtures/7.attribute.json @@ -581,5 +581,38 @@ "unit": 8, "is_active": true } + }, + { + "model": "gap.attribute", + "pk": 54, + "fields": { + "name": "Humidity Maximum", + "description": "The concentration of water vapor present in the air", + "variable_name": "humidity_maximum", + "unit": 6, + "is_active": true + } + }, + { + "model": "gap.attribute", + "pk": 55, + "fields": { + "name": "Humidity Minimum", + "description": "The total amount of shortwave radiation received from above by a surface horizontal to the ground", + "variable_name": "humidity_minimum", + "unit": 6, + "is_active": true + } + }, + { + "model": "gap.attribute", + "pk": 56, + "fields": { + "name": "Wind speed average", + "description": "The fundamental atmospheric quantity caused by air moving from high to low pressure, usually due to changes in temperature (at 10m)", + "variable_name": "wind_speed_avg", + "unit": 8, + "is_active": true + } } ] \ No newline at end of file diff --git a/django_project/gap/fixtures/8.dataset_attribute.json b/django_project/gap/fixtures/8.dataset_attribute.json index d219ea0..0edb0d1 100755 --- a/django_project/gap/fixtures/8.dataset_attribute.json +++ b/django_project/gap/fixtures/8.dataset_attribute.json @@ -878,5 +878,38 @@ "source_unit": 11, "ensembles": false } + }, + { + "model": "gap.datasetattribute", + "pk": 81, + "fields": { + "dataset": 6, + "attribute": 54, + "source": "humidityMax", + "source_unit": 6, + "ensembles": false + } + }, + { + "model": "gap.datasetattribute", + "pk": 82, + "fields": { + "dataset": 6, + "attribute": 55, + "source": "humidityMin", + "source_unit": 6, + "ensembles": false + } + }, + { + "model": "gap.datasetattribute", + "pk": 83, + "fields": { + "dataset": 6, + "attribute": 56, + "source": "windSpeedAvg", + "source_unit": 8, + "ensembles": false + } } ] \ No newline at end of file diff --git a/django_project/gap/ingestor/tio_shortterm.py b/django_project/gap/ingestor/tio_shortterm.py new file mode 100644 index 0000000..182b754 --- /dev/null +++ b/django_project/gap/ingestor/tio_shortterm.py @@ -0,0 +1,119 @@ +# coding=utf-8 +""" +Tomorrow Now GAP. + +.. note:: Tio Short Tem ingestor. +""" + +import json +import logging +import os +import traceback +import uuid +from datetime import timedelta + +from django.conf import settings +from django.core.files.base import ContentFile +from django.core.files.storage import default_storage +from django.utils import timezone + +from core.utils.s3 import zip_folder_in_s3 +from gap.ingestor.base import BaseIngestor +from gap.models import ( + CastType, CollectorSession, DataSourceFile, DatasetStore, Grid +) +from gap.providers import TomorrowIODatasetReader +from gap.providers.tio import tomorrowio_shortterm_forecast_dataset +from gap.utils.reader import DatasetReaderInput + +logger = logging.getLogger(__name__) + + +def path(filename): + """Return upload path for Ingestor files.""" + return f'{settings.STORAGE_DIR_PREFIX}tio-short-term-collector/{filename}' + + +class TioShortTermCollector(BaseIngestor): + """Collector for Tio Short Term data.""" + + def __init__(self, session: CollectorSession, working_dir: str = '/tmp'): + """Initialize TioShortTermCollector.""" + super().__init__(session, working_dir) + self.dataset = tomorrowio_shortterm_forecast_dataset() + today = timezone.now().replace( + hour=0, minute=0, second=0, microsecond=0 + ) + self.start_dt = today + self.end_dt = today + timedelta(days=14) + + def _run(self): + """Run Salient ingestor.""" + s3_storage = default_storage + zip_file = path(f"{uuid.uuid4()}.zip") + dataset = self.dataset + start_dt = self.start_dt + end_dt = self.end_dt + data_source_file, _ = DataSourceFile.objects.get_or_create( + dataset=dataset, + start_date_time=start_dt, + end_date_time=end_dt, + format=DatasetStore.ZIP_FILE, + defaults={ + 'name': zip_file, + 'created_on': timezone.now() + } + ) + filename = data_source_file.name.split('/')[-1] + _uuid = os.path.splitext(filename)[0] + zip_file = path(f"{_uuid}.zip") + folder = path(_uuid) + + # If it is already have zip file, skip the process + if s3_storage.exists(zip_file): + return + + TomorrowIODatasetReader.init_provider() + for grid in Grid.objects.all(): + file_name = f"grid-{grid.id}.json" + bbox_filename = os.path.join(folder, file_name) + + # If the json file is exist, skip it + if s3_storage.exists(bbox_filename): + continue + + # Get the data + location_input = DatasetReaderInput.from_polygon( + grid.geometry + ) + forecast_attrs = dataset.datasetattribute_set.filter( + dataset__type__type=CastType.FORECAST + ) + reader = TomorrowIODatasetReader( + dataset, + forecast_attrs, + location_input, start_dt, end_dt + ) + reader.read() + values = reader.get_data_values() + + # Save the reasult to file + content = ContentFile(json.dumps(values.to_json(), indent=4)) + s3_storage.save(bbox_filename, content) + + # Zip the folder + zip_folder_in_s3( + s3_storage, folder_path=folder, zip_file_name=zip_file + ) + + def run(self): + """Run Tio Short Term Ingestor.""" + # Run the ingestion + try: + self._run() + except Exception as e: + logger.error('Ingestor Tio Short Term failed!', e) + logger.error(traceback.format_exc()) + raise Exception(e) + finally: + pass diff --git a/django_project/gap/migrations/0026_alter_collectorsession_ingestor_type_and_more.py b/django_project/gap/migrations/0026_alter_collectorsession_ingestor_type_and_more.py new file mode 100644 index 0000000..166ee0c --- /dev/null +++ b/django_project/gap/migrations/0026_alter_collectorsession_ingestor_type_and_more.py @@ -0,0 +1,35 @@ +# Generated by Django 4.2.7 on 2024-10-01 04:10 + +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('gap', '0025_farmsuitableplantingwindowsignal_last_2_days_and_more'), + ] + + operations = [ + migrations.AlterField( + model_name='collectorsession', + name='ingestor_type', + field=models.CharField(choices=[('Tahmo', 'Tahmo'), ('Farm', 'Farm'), ('CBAM', 'CBAM'), ('Salient', 'Salient'), ('Tomorrow.io', 'Tomorrow.io'), ('Arable', 'Arable'), ('Grid', 'Grid'), ('Tahmo API', 'Tahmo API'), ('Tio Forecast Collector', 'Tio Forecast Collector')], default='Tahmo', max_length=512), + ), + migrations.AlterField( + model_name='farmgroup', + name='farms', + field=models.ManyToManyField(blank=True, to='gap.farm'), + ), + migrations.AlterField( + model_name='farmgroup', + name='users', + field=models.ManyToManyField(blank=True, to=settings.AUTH_USER_MODEL), + ), + migrations.AlterField( + model_name='ingestorsession', + name='ingestor_type', + field=models.CharField(choices=[('Tahmo', 'Tahmo'), ('Farm', 'Farm'), ('CBAM', 'CBAM'), ('Salient', 'Salient'), ('Tomorrow.io', 'Tomorrow.io'), ('Arable', 'Arable'), ('Grid', 'Grid'), ('Tahmo API', 'Tahmo API'), ('Tio Forecast Collector', 'Tio Forecast Collector')], default='Tahmo', max_length=512), + ), + ] diff --git a/django_project/gap/models/dataset.py b/django_project/gap/models/dataset.py index 5541003..4bb0a39 100644 --- a/django_project/gap/models/dataset.py +++ b/django_project/gap/models/dataset.py @@ -51,7 +51,7 @@ class DatasetTimeStep: class Dataset(Definition): - """Model representing dataset of measument collection.""" + """Model representing dataset of measurement collection.""" provider = models.ForeignKey( Provider, on_delete=models.CASCADE diff --git a/django_project/gap/models/farm_group.py b/django_project/gap/models/farm_group.py index fbc68d3..ffdbf72 100644 --- a/django_project/gap/models/farm_group.py +++ b/django_project/gap/models/farm_group.py @@ -19,10 +19,10 @@ class FarmGroup(Definition): """Model representing group of farms.""" farms = models.ManyToManyField( - Farm, blank=True, null=True + Farm, blank=True ) users = models.ManyToManyField( - User, blank=True, null=True + User, blank=True ) def email_recipients(self) -> list: diff --git a/django_project/gap/models/ingestor.py b/django_project/gap/models/ingestor.py index 3a4451a..5a37ea9 100644 --- a/django_project/gap/models/ingestor.py +++ b/django_project/gap/models/ingestor.py @@ -33,6 +33,7 @@ class IngestorType: ARABLE = 'Arable' GRID = 'Grid' TAHMO_API = 'Tahmo API' + TIO_FORECAST_COLLECTOR = 'Tio Forecast Collector' class IngestorSessionStatus: @@ -63,6 +64,10 @@ class Meta: # noqa: D106 (IngestorType.ARABLE, IngestorType.ARABLE), (IngestorType.GRID, IngestorType.GRID), (IngestorType.TAHMO_API, IngestorType.TAHMO_API), + ( + IngestorType.TIO_FORECAST_COLLECTOR, + IngestorType.TIO_FORECAST_COLLECTOR + ), ), max_length=512 ) @@ -109,12 +114,15 @@ def _run(self, working_dir): """Run the collector session.""" from gap.ingestor.cbam import CBAMCollector from gap.ingestor.salient import SalientCollector + from gap.ingestor.tio_shortterm import TioShortTermCollector ingestor = None if self.ingestor_type == IngestorType.CBAM: ingestor = CBAMCollector(self, working_dir) elif self.ingestor_type == IngestorType.SALIENT: ingestor = SalientCollector(self, working_dir) + elif self.ingestor_type == IngestorType.TIO_FORECAST_COLLECTOR: + ingestor = TioShortTermCollector(self, working_dir) if ingestor: ingestor.run() diff --git a/django_project/gap/providers/tio.py b/django_project/gap/providers/tio.py index daed5c7..ee9c314 100644 --- a/django_project/gap/providers/tio.py +++ b/django_project/gap/providers/tio.py @@ -60,8 +60,35 @@ TIO_SHORT_TERM_FORCAST_VARIABLES = { 'precipitationProbability': DatasetVariable( 'Precipitation Probability', - '', + + ( + 'Probability of precipitation represents the chance of >0.0254 cm ' + '(0.01 in.) of liquid equivalent precipitation at a radius ' + 'surrounding a point location over a specific period of time.' + ), '%', 'precipitation_probability' + ), + 'humidityMax': DatasetVariable( + 'Humidity Maximum', + 'The concentration of water vapor present in the air', + '%', 'humidity_maximum' + ), + 'humidityMin': DatasetVariable( + 'Humidity Minimum', + ( + 'The total amount of shortwave radiation received ' + 'from above by a surface horizontal to the ground' + ), + '%', 'humidity_minimum' + ), + 'windSpeedAvg': DatasetVariable( + 'Wind speed average', + ( + 'The fundamental atmospheric quantity caused by air moving from ' + 'high to low pressure, usually due to changes in temperature ' + '(at 10m)' + ), + 'm/s', 'wind_speed_avg' ) } diff --git a/django_project/gap/tests/ingestor/data/tio_shorterm_collector/test.json b/django_project/gap/tests/ingestor/data/tio_shorterm_collector/test.json new file mode 100644 index 0000000..fea6cee --- /dev/null +++ b/django_project/gap/tests/ingestor/data/tio_shorterm_collector/test.json @@ -0,0 +1,193 @@ +{ + "data": { + "timelines": [ + { + "timestep": "1d", + "endTime": "2024-10-15T06:00:00Z", + "startTime": "2024-10-01T06:00:00Z", + "intervals": [ + { + "startTime": "2024-10-01T06:00:00Z", + "values": { + "humidityMax": 84, + "humidityMin": 69, + "precipitationProbability": 0, + "rainAccumulationSum": 0, + "temperatureMax": 28, + "temperatureMin": 24.38, + "windSpeedAvg": 4.77 + } + }, + { + "startTime": "2024-10-02T06:00:00Z", + "values": { + "humidityMax": 80, + "humidityMin": 71, + "precipitationProbability": 5, + "rainAccumulationSum": 0, + "temperatureMax": 28, + "temperatureMin": 27, + "windSpeedAvg": 4.35 + } + }, + { + "startTime": "2024-10-03T06:00:00Z", + "values": { + "humidityMax": 79, + "humidityMin": 73, + "precipitationProbability": 5, + "rainAccumulationSum": 0, + "temperatureMax": 28, + "temperatureMin": 27, + "windSpeedAvg": 5.58 + } + }, + { + "startTime": "2024-10-04T06:00:00Z", + "values": { + "humidityMax": 78, + "humidityMin": 72, + "precipitationProbability": 5, + "rainAccumulationSum": 0, + "temperatureMax": 28, + "temperatureMin": 27, + "windSpeedAvg": 5.74 + } + }, + { + "startTime": "2024-10-05T06:00:00Z", + "values": { + "humidityMax": 76, + "humidityMin": 70, + "precipitationProbability": 5, + "rainAccumulationSum": 0, + "temperatureMax": 28, + "temperatureMin": 27, + "windSpeedAvg": 5.09 + } + }, + { + "startTime": "2024-10-06T06:00:00Z", + "values": { + "humidityMax": 76, + "humidityMin": 72, + "precipitationProbability": 5, + "rainAccumulationSum": 0, + "temperatureMax": 28, + "temperatureMin": 27, + "windSpeedAvg": 4.01 + } + }, + { + "startTime": "2024-10-07T06:00:00Z", + "values": { + "humidityMax": 76, + "humidityMin": 70, + "precipitationProbability": 0, + "rainAccumulationSum": 0, + "temperatureMax": 28.5, + "temperatureMin": 27, + "windSpeedAvg": 3.82 + } + }, + { + "startTime": "2024-10-08T06:00:00Z", + "values": { + "humidityMax": 78, + "humidityMin": 72, + "precipitationProbability": 5, + "rainAccumulationSum": 0, + "temperatureMax": 28, + "temperatureMin": 27.5, + "windSpeedAvg": 4.12 + } + }, + { + "startTime": "2024-10-09T06:00:00Z", + "values": { + "humidityMax": 80, + "humidityMin": 74, + "precipitationProbability": 5, + "rainAccumulationSum": 0, + "temperatureMax": 28, + "temperatureMin": 27.5, + "windSpeedAvg": 5.29 + } + }, + { + "startTime": "2024-10-10T06:00:00Z", + "values": { + "humidityMax": 80, + "humidityMin": 73, + "precipitationProbability": 5, + "rainAccumulationSum": 0, + "temperatureMax": 28, + "temperatureMin": 27.5, + "windSpeedAvg": 4.96 + } + }, + { + "startTime": "2024-10-11T06:00:00Z", + "values": { + "humidityMax": 77, + "humidityMin": 68, + "precipitationProbability": 5, + "rainAccumulationSum": 0, + "temperatureMax": 29, + "temperatureMin": 27, + "windSpeedAvg": 4.1 + } + }, + { + "startTime": "2024-10-12T06:00:00Z", + "values": { + "humidityMax": 78, + "humidityMin": 70, + "precipitationProbability": 5, + "rainAccumulationSum": 0, + "temperatureMax": 28.5, + "temperatureMin": 27, + "windSpeedAvg": 4.42 + } + }, + { + "startTime": "2024-10-13T06:00:00Z", + "values": { + "humidityMax": 78, + "humidityMin": 72, + "precipitationProbability": 5, + "rainAccumulationSum": 0, + "temperatureMax": 28, + "temperatureMin": 27.5, + "windSpeedAvg": 4.52 + } + }, + { + "startTime": "2024-10-14T06:00:00Z", + "values": { + "humidityMax": 78, + "humidityMin": 72, + "precipitationProbability": 5, + "rainAccumulationSum": 0, + "temperatureMax": 28, + "temperatureMin": 24.12, + "windSpeedAvg": 4.74 + } + }, + { + "startTime": "2024-10-15T06:00:00Z", + "values": { + "humidityMax": 77.83, + "humidityMin": 72.77, + "precipitationProbability": 5, + "rainAccumulationSum": 0, + "temperatureMax": 24.9, + "temperatureMin": 24.12, + "windSpeedAvg": 3.17 + } + } + ] + } + ] + } +} \ No newline at end of file diff --git a/django_project/gap/tests/ingestor/test_tio_shortterm_collector.py b/django_project/gap/tests/ingestor/test_tio_shortterm_collector.py new file mode 100644 index 0000000..18a7d78 --- /dev/null +++ b/django_project/gap/tests/ingestor/test_tio_shortterm_collector.py @@ -0,0 +1,330 @@ +# coding=utf-8 +""" +Tomorrow Now GAP. + +.. note:: Unit tests for Tio Collector. +""" +import json +import os +import zipfile +from datetime import datetime +from unittest.mock import patch + +import responses +from django.conf import settings +from django.contrib.gis.gdal import DataSource +from django.contrib.gis.geos import Polygon, GEOSGeometry +from django.core.files.storage import default_storage +from django.test import TestCase +from django.utils import timezone + +from core.settings.utils import absolute_path +from gap.factories.grid import GridFactory +from gap.ingestor.tio_shortterm import path +from gap.models import ( + Country, IngestorSessionStatus, IngestorType +) +from gap.models.dataset import DataSourceFile +from gap.models.ingestor import CollectorSession +from gap.tests.mock_response import BaseTestWithPatchResponses, PatchReqeust + + +class TioShortTermCollectorTest(BaseTestWithPatchResponses, TestCase): + """Tio Collector test case.""" + + fixtures = [ + '2.provider.json', + '3.station_type.json', + '4.dataset_type.json', + '5.dataset.json', + '6.unit.json', + '7.attribute.json', + '8.dataset_attribute.json' + ] + ingestor_type = IngestorType.TIO_FORECAST_COLLECTOR + responses_folder = absolute_path( + 'gap', 'tests', 'ingestor', 'data', 'tio_shorterm_collector' + ) + api_key = 'tomorrow_api_key' + + def setUp(self): + """Init test case.""" + os.environ['TOMORROW_IO_API_KEY'] = self.api_key + # Init kenya Country + shp_path = os.path.join( + os.path.dirname(os.path.abspath(__file__)), + 'data', + 'Kenya.geojson' + ) + data_source = DataSource(shp_path) + layer = data_source[0] + for feature in layer: + geometry = GEOSGeometry(feature.geom.wkt, srid=4326) + Country.objects.create( + name=feature['name'], + iso_a3=feature['iso_a3'], + geometry=geometry + ) + + @property + def mock_requests(self): + """Mock requests.""" + return [ + # Devices API + PatchReqeust( + f'https://api.tomorrow.io/v4/timelines?apikey={self.api_key}', + file_response=os.path.join( + self.responses_folder, 'test.json' + ), + request_method='POST' + ) + ] + + def test_path(self): + """Test path.""" + self.assertEqual( + path('test'), + f'{settings.STORAGE_DIR_PREFIX}tio-short-term-collector/test' + ) + + def test_collector_empty_grid(self): + """Testing collector.""" + session = CollectorSession.objects.create( + ingestor_type=self.ingestor_type + ) + session.run() + session.refresh_from_db() + self.assertEqual(session.status, IngestorSessionStatus.SUCCESS) + self.assertEqual(DataSourceFile.objects.count(), 1) + _file = default_storage.open(DataSourceFile.objects.first().name) + with zipfile.ZipFile(_file, 'r') as zip_file: + self.assertEqual(len(zip_file.filelist), 0) + + @patch('gap.ingestor.tio_shortterm.timezone') + @responses.activate + def test_collector_one_grid(self, mock_timezone): + """Testing collector.""" + self.init_mock_requests() + today = datetime( + 2024, 10, 1, 6, 0, 0 + ) + today = timezone.make_aware( + today, timezone.get_default_timezone() + ) + mock_timezone.now.return_value = today + grid = GridFactory( + geometry=Polygon( + ( + (0, 0), (0, 0.01), (0.01, 0.01), (0.01, 0), (0, 0) + ) + ) + ) + session = CollectorSession.objects.create( + ingestor_type=self.ingestor_type + ) + session.run() + session.refresh_from_db() + self.assertEqual(session.status, IngestorSessionStatus.SUCCESS) + self.assertEqual(DataSourceFile.objects.count(), 1) + _file = default_storage.open(DataSourceFile.objects.first().name) + with zipfile.ZipFile(_file, 'r') as zip_file: + self.assertEqual(len(zip_file.filelist), 1) + _file = zip_file.open(f'grid-{grid.id}.json') + _data = json.loads(_file.read().decode('utf-8')) + self.assertEqual( + _data, + { + 'geometry': { + 'type': 'Polygon', + 'coordinates': [ + [[0.0, 0.0], [0.0, 0.01], [0.01, 0.01], + [0.01, 0.0], [0.0, 0.0]] + ] + }, + 'data': [ + { + 'datetime': '2024-10-01T06:00:00+00:00', + 'values': { + 'total_rainfall': 0, + 'total_evapotranspiration_flux': None, + 'max_temperature': 28, + 'min_temperature': 24.38, + 'precipitation_probability': 0, + 'humidity_maximum': 84, + 'humidity_minimum': 69, + 'wind_speed_avg': 4.77 + } + }, + { + 'datetime': '2024-10-02T06:00:00+00:00', + 'values': { + 'total_rainfall': 0, + 'total_evapotranspiration_flux': None, + 'max_temperature': 28, + 'min_temperature': 27, + 'precipitation_probability': 5, + 'humidity_maximum': 80, + 'humidity_minimum': 71, + 'wind_speed_avg': 4.35 + } + }, + { + 'datetime': '2024-10-03T06:00:00+00:00', + 'values': { + 'total_rainfall': 0, + 'total_evapotranspiration_flux': None, + 'max_temperature': 28, + 'min_temperature': 27, + 'precipitation_probability': 5, + 'humidity_maximum': 79, + 'humidity_minimum': 73, + 'wind_speed_avg': 5.58 + } + }, + { + 'datetime': '2024-10-04T06:00:00+00:00', + 'values': { + 'total_rainfall': 0, + 'total_evapotranspiration_flux': None, + 'max_temperature': 28, + 'min_temperature': 27, + 'precipitation_probability': 5, + 'humidity_maximum': 78, + 'humidity_minimum': 72, + 'wind_speed_avg': 5.74 + } + }, + { + 'datetime': '2024-10-05T06:00:00+00:00', + 'values': { + 'total_rainfall': 0, + 'total_evapotranspiration_flux': None, + 'max_temperature': 28, + 'min_temperature': 27, + 'precipitation_probability': 5, + 'humidity_maximum': 76, + 'humidity_minimum': 70, + 'wind_speed_avg': 5.09 + } + }, + { + 'datetime': '2024-10-06T06:00:00+00:00', + 'values': { + 'total_rainfall': 0, + 'total_evapotranspiration_flux': None, + 'max_temperature': 28, + 'min_temperature': 27, + 'precipitation_probability': 5, + 'humidity_maximum': 76, + 'humidity_minimum': 72, + 'wind_speed_avg': 4.01 + } + }, + { + 'datetime': '2024-10-07T06:00:00+00:00', + 'values': { + 'total_rainfall': 0, + 'total_evapotranspiration_flux': None, + 'max_temperature': 28.5, + 'min_temperature': 27, + 'precipitation_probability': 0, + 'humidity_maximum': 76, + 'humidity_minimum': 70, + 'wind_speed_avg': 3.82 + } + }, + { + 'datetime': '2024-10-08T06:00:00+00:00', + 'values': { + 'total_rainfall': 0, + 'total_evapotranspiration_flux': None, + 'max_temperature': 28, + 'min_temperature': 27.5, + 'precipitation_probability': 5, + 'humidity_maximum': 78, + 'humidity_minimum': 72, + 'wind_speed_avg': 4.12 + } + }, + { + 'datetime': '2024-10-09T06:00:00+00:00', + 'values': { + 'total_rainfall': 0, + 'total_evapotranspiration_flux': None, + 'max_temperature': 28, + 'min_temperature': 27.5, + 'precipitation_probability': 5, + 'humidity_maximum': 80, + 'humidity_minimum': 74, + 'wind_speed_avg': 5.29 + } + }, + { + 'datetime': '2024-10-10T06:00:00+00:00', + 'values': { + 'total_rainfall': 0, + 'total_evapotranspiration_flux': None, + 'max_temperature': 28, + 'min_temperature': 27.5, + 'precipitation_probability': 5, + 'humidity_maximum': 80, + 'humidity_minimum': 73, + 'wind_speed_avg': 4.96 + } + }, + { + 'datetime': '2024-10-11T06:00:00+00:00', + 'values': { + 'total_rainfall': 0, + 'total_evapotranspiration_flux': None, + 'max_temperature': 29, + 'min_temperature': 27, + 'precipitation_probability': 5, + 'humidity_maximum': 77, + 'humidity_minimum': 68, + 'wind_speed_avg': 4.1 + } + }, + { + 'datetime': '2024-10-12T06:00:00+00:00', + 'values': { + 'total_rainfall': 0, + 'total_evapotranspiration_flux': None, + 'max_temperature': 28.5, + 'min_temperature': 27, + 'precipitation_probability': 5, + 'humidity_maximum': 78, + 'humidity_minimum': 70, + 'wind_speed_avg': 4.42 + } + }, + { + 'datetime': '2024-10-13T06:00:00+00:00', + 'values': { + 'total_rainfall': 0, + 'total_evapotranspiration_flux': None, + 'max_temperature': 28, + 'min_temperature': 27.5, + 'precipitation_probability': 5, + 'humidity_maximum': 78, + 'humidity_minimum': 72, + 'wind_speed_avg': 4.52 + } + }, + { + 'datetime': '2024-10-14T06:00:00+00:00', + 'values': { + 'total_rainfall': 0, + 'total_evapotranspiration_flux': None, + 'max_temperature': 28, + 'min_temperature': 24.12, + 'precipitation_probability': 5, + 'humidity_maximum': 78, + 'humidity_minimum': 72, + 'wind_speed_avg': 4.74 + } + } + ] + } + + ) diff --git a/django_project/gap/tests/utils/test_netcdf.py b/django_project/gap/tests/utils/test_netcdf.py index 1610252..375bfb2 100644 --- a/django_project/gap/tests/utils/test_netcdf.py +++ b/django_project/gap/tests/utils/test_netcdf.py @@ -247,9 +247,12 @@ def test_to_json_with_point_type(self): result = self.dataset_reader_value_list.to_json() self.assertEqual(result, {}) - def test_to_json_with_non_point_type(self): + def test_to_json_with_non_point_polygon_type(self): """Test invalid convert to json.""" self.mock_location_input.type = 'polygon' + self.dataset_reader_value_list.to_json() + + self.mock_location_input.type = 'bbox' with self.assertRaises(TypeError): self.dataset_reader_value_list.to_json() diff --git a/django_project/gap/utils/reader.py b/django_project/gap/utils/reader.py index 04eb7fe..8487018 100644 --- a/django_project/gap/utils/reader.py +++ b/django_project/gap/utils/reader.py @@ -308,7 +308,7 @@ def _to_dict(self) -> dict: return {} return { - 'geometry': json.loads(self.location_input.point.json), + 'geometry': json.loads(self.location_input.geometry.json), 'data': [result.to_dict() for result in self.values] } @@ -328,8 +328,10 @@ def to_json(self) -> dict: :return: data dictionary :rtype: dict """ - if self.location_input.type != LocationInputType.POINT: - raise TypeError('Location input type is not point!') + if self.location_input.type not in [ + LocationInputType.POINT, LocationInputType.POLYGON + ]: + raise TypeError('Location input type is not point or polygon!') if self._is_xr_dataset: return self._xr_dataset_to_dict() return self._to_dict() diff --git a/django_project/spw/tests/test_crop_insight_generator.py b/django_project/spw/tests/test_crop_insight_generator.py index 5968abf..4f0f551 100644 --- a/django_project/spw/tests/test_crop_insight_generator.py +++ b/django_project/spw/tests/test_crop_insight_generator.py @@ -309,7 +309,7 @@ def mock_send_fn(self, fail_silently=False): self.request.refresh_from_db() # Check the if of farm group in the path - self.assertTrue(f'{self.farm_group.id}/' in self.request.file.path) + self.assertTrue(f'{self.farm_group.id}/' in self.request.file.name) # Check the attachment on email self.assertEqual(attachments[0][0], self.request.filename)