From 4fd6bd6b9e37799410da9c22dc861d2debdac690 Mon Sep 17 00:00:00 2001 From: Danang Date: Fri, 4 Oct 2024 09:31:59 +0100 Subject: [PATCH] Change date range Tio Short-term Forecast Collector (#174) * change date range to D-6 to D+14 * fix tests --- django_project/gap/ingestor/tio_shortterm.py | 9 ++++++--- django_project/gap/providers/tio.py | 10 ++++++++-- .../gap/tests/ingestor/test_arable.py | 10 +++++----- .../gap/tests/ingestor/test_tahmo_api.py | 18 +++++++++--------- .../ingestor/test_tio_shortterm_collector.py | 17 +++++++++++++++-- .../tests/ingestor/test_wind_borne_sistems.py | 8 ++++---- django_project/gap/tests/mock_response.py | 6 +++--- django_project/gap/tests/providers/test_tio.py | 6 ++++-- 8 files changed, 54 insertions(+), 30 deletions(-) diff --git a/django_project/gap/ingestor/tio_shortterm.py b/django_project/gap/ingestor/tio_shortterm.py index 182b754..cf38f61 100644 --- a/django_project/gap/ingestor/tio_shortterm.py +++ b/django_project/gap/ingestor/tio_shortterm.py @@ -44,8 +44,10 @@ def __init__(self, session: CollectorSession, working_dir: str = '/tmp'): today = timezone.now().replace( hour=0, minute=0, second=0, microsecond=0 ) - self.start_dt = today - self.end_dt = today + timedelta(days=14) + # Retrieve D-6 to D+14 + # Total days: 21 + self.start_dt = today - timedelta(days=6) + self.end_dt = today + timedelta(days=15) def _run(self): """Run Salient ingestor.""" @@ -98,7 +100,8 @@ def _run(self): values = reader.get_data_values() # Save the reasult to file - content = ContentFile(json.dumps(values.to_json(), indent=4)) + content = ContentFile( + json.dumps(values.to_json(), separators=(',', ':'))) s3_storage.save(bbox_filename, content) # Zip the folder diff --git a/django_project/gap/providers/tio.py b/django_project/gap/providers/tio.py index 215aa24..43c9a73 100644 --- a/django_project/gap/providers/tio.py +++ b/django_project/gap/providers/tio.py @@ -125,13 +125,14 @@ class TomorrowIODatasetReader(BaseDatasetReader): def __init__( self, dataset: Dataset, attributes: List[DatasetAttribute], location_input: DatasetReaderInput, start_date: datetime, - end_date: datetime) -> None: + end_date: datetime, verbose = False) -> None: """Initialize Dataset Reader.""" super().__init__( dataset, attributes, location_input, start_date, end_date) self.errors = None self.warnings = None self.results = [] + self.verbose = verbose @classmethod def init_provider(cls): @@ -276,8 +277,11 @@ def read(self): self.end_date ) else: + min_available_timelines = today - timedelta(days=6) self.read_forecast_data( - self.start_date if self.start_date >= today else today, + self.start_date if + self.start_date >= min_available_timelines else + min_available_timelines, self.end_date ) @@ -288,6 +292,8 @@ def _log_errors(self): def _log_warnings(self): """Log any warnings from the API.""" + if not self.verbose: + return logger.warning(f'Tomorrow.io API warnings: {len(self.warnings)}') logger.warning(json.dumps(self.warnings)) diff --git a/django_project/gap/tests/ingestor/test_arable.py b/django_project/gap/tests/ingestor/test_arable.py index 044aa65..821227f 100644 --- a/django_project/gap/tests/ingestor/test_arable.py +++ b/django_project/gap/tests/ingestor/test_arable.py @@ -18,7 +18,7 @@ Country, Station, Measurement, IngestorSession, IngestorSessionStatus, IngestorType ) -from gap.tests.mock_response import BaseTestWithPatchResponses, PatchReqeust +from gap.tests.mock_response import BaseTestWithPatchResponses, PatchRequest class ArableIngestorTest(BaseTestWithPatchResponses, TestCase): @@ -44,20 +44,20 @@ def mock_requests(self): arable_api = ArableAPI() return [ # Devices API - PatchReqeust( + PatchRequest( arable_api.DEVICES + '?page=1', file_response=os.path.join( self.responses_folder, 'devices.json' ) ), # Devices API - PatchReqeust( + PatchRequest( arable_api.DEVICES + '?page=2', response={}, status_code=404 ), # Data API - PatchReqeust( + PatchRequest( ( f'{arable_api.DATA}?device=A00&' 'select=et,max_rh,maxt,mean_rh,meant,min_rh,mint,prate,' @@ -69,7 +69,7 @@ def mock_requests(self): ) ), # Data API - PatchReqeust( + PatchRequest( ( f'{arable_api.DATA}?device=A01&' 'select=et,max_rh,maxt,mean_rh,meant,min_rh,mint,prate,' diff --git a/django_project/gap/tests/ingestor/test_tahmo_api.py b/django_project/gap/tests/ingestor/test_tahmo_api.py index ffb9635..b331aa5 100644 --- a/django_project/gap/tests/ingestor/test_tahmo_api.py +++ b/django_project/gap/tests/ingestor/test_tahmo_api.py @@ -23,7 +23,7 @@ Country, Station, IngestorSession, IngestorSessionStatus, IngestorType ) from gap.models.preferences import Preferences -from gap.tests.mock_response import BaseTestWithPatchResponses, PatchReqeust +from gap.tests.mock_response import BaseTestWithPatchResponses, PatchRequest class TahmoAPIIngestorTest(BaseTestWithPatchResponses, TestCase): @@ -54,7 +54,7 @@ def mock_requests(self): continue for station_code in ['TD00001', 'TD00002']: patches.append( - PatchReqeust( + PatchRequest( ( f'{preferences.tahmo_api_url}' f'/services/measurements/v2/' @@ -66,7 +66,7 @@ def mock_requests(self): ) ) patches.append( - PatchReqeust( + PatchRequest( ( f'{preferences.tahmo_api_url}' f'/services/measurements/v2/' @@ -80,13 +80,13 @@ def mock_requests(self): return patches + [ # Devices API - PatchReqeust( + PatchRequest( f'{preferences.tahmo_api_url}/services/assets/v2/stations', file_response=os.path.join( self.responses_folder, 'devices.json' ) ), - PatchReqeust( + PatchRequest( ( f'{preferences.tahmo_api_url}/services/measurements/v2/' f'stations/TD00001/measurements/raw?' @@ -97,7 +97,7 @@ def mock_requests(self): self.responses_folder, 'TD00001.1.json' ) ), - PatchReqeust( + PatchRequest( ( f'{preferences.tahmo_api_url}/services/measurements/v2/' f'stations/TD00002/measurements/raw?' @@ -108,7 +108,7 @@ def mock_requests(self): self.responses_folder, 'TD00001.1.json' ) ), - PatchReqeust( + PatchRequest( ( f'{preferences.tahmo_api_url}/services/measurements/v2/' f'stations/TD00001/measurements/raw?' @@ -119,7 +119,7 @@ def mock_requests(self): self.responses_folder, 'TD00001.2.json' ) ), - PatchReqeust( + PatchRequest( ( f'{preferences.tahmo_api_url}/services/measurements/v2/' f'stations/TD00002/measurements/raw?' @@ -130,7 +130,7 @@ def mock_requests(self): self.responses_folder, 'TD00001.1.json' ) ), - PatchReqeust( + PatchRequest( ( f'{preferences.tahmo_api_url}/services/measurements/v2/' f'stations/TD00002/measurements/raw?' diff --git a/django_project/gap/tests/ingestor/test_tio_shortterm_collector.py b/django_project/gap/tests/ingestor/test_tio_shortterm_collector.py index 18a7d78..d1d82d3 100644 --- a/django_project/gap/tests/ingestor/test_tio_shortterm_collector.py +++ b/django_project/gap/tests/ingestor/test_tio_shortterm_collector.py @@ -26,7 +26,7 @@ ) from gap.models.dataset import DataSourceFile from gap.models.ingestor import CollectorSession -from gap.tests.mock_response import BaseTestWithPatchResponses, PatchReqeust +from gap.tests.mock_response import BaseTestWithPatchResponses, PatchRequest class TioShortTermCollectorTest(BaseTestWithPatchResponses, TestCase): @@ -71,7 +71,7 @@ def mock_requests(self): """Mock requests.""" return [ # Devices API - PatchReqeust( + PatchRequest( f'https://api.tomorrow.io/v4/timelines?apikey={self.api_key}', file_response=os.path.join( self.responses_folder, 'test.json' @@ -323,6 +323,19 @@ def test_collector_one_grid(self, mock_timezone): 'humidity_minimum': 72, 'wind_speed_avg': 4.74 } + }, + { + "datetime": "2024-10-15T06:00:00+00:00", + "values": { + 'total_rainfall': 0, + 'total_evapotranspiration_flux': None, + 'max_temperature': 24.9, + 'min_temperature': 24.12, + 'precipitation_probability': 5, + 'humidity_maximum': 77.83, + 'humidity_minimum': 72.77, + 'wind_speed_avg': 3.17 + } } ] } diff --git a/django_project/gap/tests/ingestor/test_wind_borne_sistems.py b/django_project/gap/tests/ingestor/test_wind_borne_sistems.py index 3fe103b..425d339 100644 --- a/django_project/gap/tests/ingestor/test_wind_borne_sistems.py +++ b/django_project/gap/tests/ingestor/test_wind_borne_sistems.py @@ -20,7 +20,7 @@ from gap.models import ( Country, Station, IngestorSession, IngestorSessionStatus, IngestorType ) -from gap.tests.mock_response import BaseTestWithPatchResponses, PatchReqeust +from gap.tests.mock_response import BaseTestWithPatchResponses, PatchRequest class WindBorneSystemsAPIIngestorTest(BaseTestWithPatchResponses, TestCase): @@ -45,7 +45,7 @@ def mock_requests(self): """Mock requests.""" base_url = WindBorneSystemsAPI.base_url return [ - PatchReqeust( + PatchRequest( ( f'{base_url}/observations.json?' f'include_ids=True&include_mission_name=True' @@ -54,7 +54,7 @@ def mock_requests(self): self.responses_folder, 'since_1.json' ) ), - PatchReqeust( + PatchRequest( ( f'{base_url}/observations.json?' f'include_ids=True&include_mission_name=True&' @@ -64,7 +64,7 @@ def mock_requests(self): self.responses_folder, 'since_2.json' ) ), - PatchReqeust( + PatchRequest( ( f'{base_url}/observations.json?' f'include_ids=True&include_mission_name=True&' diff --git a/django_project/gap/tests/mock_response.py b/django_project/gap/tests/mock_response.py index 3ca7413..0e954f9 100644 --- a/django_project/gap/tests/mock_response.py +++ b/django_project/gap/tests/mock_response.py @@ -9,14 +9,14 @@ import responses -class PatchReqeust: +class PatchRequest: """Request object for patch.""" def __init__( self, url: str, response: dict = None, file_response: str = None, request_method='GET', status_code=200 ): - """Initialize the PatchReqeust. + """Initialize the PatchRequest. :param url: URL to mock :param response: Response to be used as responses. @@ -37,7 +37,7 @@ class BaseTestWithPatchResponses: mock_requests = [] - def _mock_request(self, patch_request: PatchReqeust): + def _mock_request(self, patch_request: PatchRequest): """Mock response with file.""" request_method = patch_request.request_method response = {} diff --git a/django_project/gap/tests/providers/test_tio.py b/django_project/gap/tests/providers/test_tio.py index 540468f..45a0fa8 100644 --- a/django_project/gap/tests/providers/test_tio.py +++ b/django_project/gap/tests/providers/test_tio.py @@ -281,7 +281,7 @@ def test_read_forecast_data_only( self.reader.end_date = dt_now self.reader.dataset.type = DatasetType( type=CastType.FORECAST, name='TestForecast') - self.reader.start_date = dt_now - timedelta(days=1) + self.reader.start_date = dt_now - timedelta(days=7) # Call the read method with patch('gap.providers.tio.datetime') as mock_datetime: @@ -290,7 +290,9 @@ def test_read_forecast_data_only( # Check that the correct method is called mock_read_forecast_data.assert_called_once() - self.assertEqual(mock_read_forecast_data.call_args[0][0], dt_now) + self.assertEqual( + mock_read_forecast_data.call_args[0][0], + dt_now - timedelta(days=6)) self.assertEqual( mock_read_forecast_data.call_args[0][1], self.reader.end_date) mock_read_historical_data.assert_not_called()