Change date range Tio Short-term Forecast Collector (#174)

* change date range to D-6 to D+14

* fix tests

django_project/gap/ingestor/tio_shortterm.py

@@ -44,8 +44,10 @@ def __init__(self, session: CollectorSession, working_dir: str = '/tmp'):
  today = timezone.now().replace(
  hour=0, minute=0, second=0, microsecond=0
  )
- self.start_dt = today
- self.end_dt = today + timedelta(days=14)
+ # Retrieve D-6 to D+14
+ # Total days: 21
+ self.start_dt = today - timedelta(days=6)
+ self.end_dt = today + timedelta(days=15)
 
  def _run(self):
  """Run Salient ingestor."""
@@ -98,7 +100,8 @@ def _run(self):
  values = reader.get_data_values()
 
  # Save the reasult to file
- content = ContentFile(json.dumps(values.to_json(), indent=4))
+ content = ContentFile(
+ json.dumps(values.to_json(), separators=(',', ':')))
  s3_storage.save(bbox_filename, content)
 
  # Zip the folder

django_project/gap/providers/tio.py

@@ -125,13 +125,14 @@ class TomorrowIODatasetReader(BaseDatasetReader):
  def __init__(
  self, dataset: Dataset, attributes: List[DatasetAttribute],
  location_input: DatasetReaderInput, start_date: datetime,
- end_date: datetime) -> None:
+ end_date: datetime, verbose = False) -> None:
  """Initialize Dataset Reader."""
  super().__init__(
  dataset, attributes, location_input, start_date, end_date)
  self.errors = None
  self.warnings = None
  self.results = []
+ self.verbose = verbose
 
  @classmethod
  def init_provider(cls):
@@ -276,8 +277,11 @@ def read(self):
  self.end_date
  )
  else:
+ min_available_timelines = today - timedelta(days=6)
  self.read_forecast_data(
- self.start_date if self.start_date >= today else today,
+ self.start_date if
+ self.start_date >= min_available_timelines else
+ min_available_timelines,
  self.end_date
  )
 
@@ -288,6 +292,8 @@ def _log_errors(self):
 
  def _log_warnings(self):
  """Log any warnings from the API."""
+ if not self.verbose:
+ return
  logger.warning(f'Tomorrow.io API warnings: {len(self.warnings)}')
  logger.warning(json.dumps(self.warnings))
 

django_project/gap/tests/ingestor/test_arable.py

@@ -18,7 +18,7 @@
  Country, Station, Measurement,
  IngestorSession, IngestorSessionStatus, IngestorType
 )
-from gap.tests.mock_response import BaseTestWithPatchResponses, PatchReqeust
+from gap.tests.mock_response import BaseTestWithPatchResponses, PatchRequest
 
 
 class ArableIngestorTest(BaseTestWithPatchResponses, TestCase):
@@ -44,20 +44,20 @@ def mock_requests(self):
  arable_api = ArableAPI()
  return [
  # Devices API
- PatchReqeust(
+ PatchRequest(
  arable_api.DEVICES + '?page=1',
  file_response=os.path.join(
  self.responses_folder, 'devices.json'
  )
  ),
  # Devices API
- PatchReqeust(
+ PatchRequest(
  arable_api.DEVICES + '?page=2',
  response={},
  status_code=404
  ),
  # Data API
- PatchReqeust(
+ PatchRequest(
  (
  f'{arable_api.DATA}?device=A00&'
  'select=et,max_rh,maxt,mean_rh,meant,min_rh,mint,prate,'
@@ -69,7 +69,7 @@ def mock_requests(self):
  )
  ),
  # Data API
- PatchReqeust(
+ PatchRequest(
  (
  f'{arable_api.DATA}?device=A01&'
  'select=et,max_rh,maxt,mean_rh,meant,min_rh,mint,prate,'

django_project/gap/tests/ingestor/test_tahmo_api.py

@@ -23,7 +23,7 @@
  Country, Station, IngestorSession, IngestorSessionStatus, IngestorType
 )
 from gap.models.preferences import Preferences
-from gap.tests.mock_response import BaseTestWithPatchResponses, PatchReqeust
+from gap.tests.mock_response import BaseTestWithPatchResponses, PatchRequest
 
 
 class TahmoAPIIngestorTest(BaseTestWithPatchResponses, TestCase):
@@ -54,7 +54,7 @@ def mock_requests(self):
  continue
  for station_code in ['TD00001', 'TD00002']:
  patches.append(
- PatchReqeust(
+ PatchRequest(
  (
  f'{preferences.tahmo_api_url}'
  f'/services/measurements/v2/'
@@ -66,7 +66,7 @@ def mock_requests(self):
  )
  )
  patches.append(
- PatchReqeust(
+ PatchRequest(
  (
  f'{preferences.tahmo_api_url}'
  f'/services/measurements/v2/'
@@ -80,13 +80,13 @@ def mock_requests(self):
 
  return patches + [
  # Devices API
- PatchReqeust(
+ PatchRequest(
  f'{preferences.tahmo_api_url}/services/assets/v2/stations',
  file_response=os.path.join(
  self.responses_folder, 'devices.json'
  )
  ),
- PatchReqeust(
+ PatchRequest(
  (
  f'{preferences.tahmo_api_url}/services/measurements/v2/'
  f'stations/TD00001/measurements/raw?'
@@ -97,7 +97,7 @@ def mock_requests(self):
  self.responses_folder, 'TD00001.1.json'
  )
  ),
- PatchReqeust(
+ PatchRequest(
  (
  f'{preferences.tahmo_api_url}/services/measurements/v2/'
  f'stations/TD00002/measurements/raw?'
@@ -108,7 +108,7 @@ def mock_requests(self):
  self.responses_folder, 'TD00001.1.json'
  )
  ),
- PatchReqeust(
+ PatchRequest(
  (
  f'{preferences.tahmo_api_url}/services/measurements/v2/'
  f'stations/TD00001/measurements/raw?'
@@ -119,7 +119,7 @@ def mock_requests(self):
  self.responses_folder, 'TD00001.2.json'
  )
  ),
- PatchReqeust(
+ PatchRequest(
  (
  f'{preferences.tahmo_api_url}/services/measurements/v2/'
  f'stations/TD00002/measurements/raw?'
@@ -130,7 +130,7 @@ def mock_requests(self):
  self.responses_folder, 'TD00001.1.json'
  )
  ),
- PatchReqeust(
+ PatchRequest(
  (
  f'{preferences.tahmo_api_url}/services/measurements/v2/'
  f'stations/TD00002/measurements/raw?'

django_project/gap/tests/ingestor/test_tio_shortterm_collector.py

@@ -26,7 +26,7 @@
 )
 from gap.models.dataset import DataSourceFile
 from gap.models.ingestor import CollectorSession
-from gap.tests.mock_response import BaseTestWithPatchResponses, PatchReqeust
+from gap.tests.mock_response import BaseTestWithPatchResponses, PatchRequest
 
 
 class TioShortTermCollectorTest(BaseTestWithPatchResponses, TestCase):
@@ -71,7 +71,7 @@ def mock_requests(self):
  """Mock requests."""
  return [
  # Devices API
- PatchReqeust(
+ PatchRequest(
  f'https://api.tomorrow.io/v4/timelines?apikey={self.api_key}',
  file_response=os.path.join(
  self.responses_folder, 'test.json'
@@ -323,6 +323,19 @@ def test_collector_one_grid(self, mock_timezone):
  'humidity_minimum': 72,
  'wind_speed_avg': 4.74
  }
+ },
+ {
+ "datetime": "2024-10-15T06:00:00+00:00",
+ "values": {
+ 'total_rainfall': 0,
+ 'total_evapotranspiration_flux': None,
+ 'max_temperature': 24.9,
+ 'min_temperature': 24.12,
+ 'precipitation_probability': 5,
+ 'humidity_maximum': 77.83,
+ 'humidity_minimum': 72.77,
+ 'wind_speed_avg': 3.17
+ }
  }
  ]
  }

django_project/gap/tests/ingestor/test_wind_borne_sistems.py

@@ -20,7 +20,7 @@
 from gap.models import (
  Country, Station, IngestorSession, IngestorSessionStatus, IngestorType
 )
-from gap.tests.mock_response import BaseTestWithPatchResponses, PatchReqeust
+from gap.tests.mock_response import BaseTestWithPatchResponses, PatchRequest
 
 
 class WindBorneSystemsAPIIngestorTest(BaseTestWithPatchResponses, TestCase):
@@ -45,7 +45,7 @@ def mock_requests(self):
  """Mock requests."""
  base_url = WindBorneSystemsAPI.base_url
  return [
- PatchReqeust(
+ PatchRequest(
  (
  f'{base_url}/observations.json?'
  f'include_ids=True&include_mission_name=True'
@@ -54,7 +54,7 @@ def mock_requests(self):
  self.responses_folder, 'since_1.json'
  )
  ),
- PatchReqeust(
+ PatchRequest(
  (
  f'{base_url}/observations.json?'
  f'include_ids=True&include_mission_name=True&'
@@ -64,7 +64,7 @@ def mock_requests(self):
  self.responses_folder, 'since_2.json'
  )
  ),
- PatchReqeust(
+ PatchRequest(
  (
  f'{base_url}/observations.json?'
  f'include_ids=True&include_mission_name=True&'

django_project/gap/tests/mock_response.py

@@ -9,14 +9,14 @@
 import responses
 
 
-class PatchReqeust:
+class PatchRequest:
  """Request object for patch."""
 
  def __init__(
  self, url: str, response: dict = None, file_response: str = None,
  request_method='GET', status_code=200
  ):
- """Initialize the PatchReqeust.
+ """Initialize the PatchRequest.
 
  :param url: URL to mock
  :param response: Response to be used as responses.
@@ -37,7 +37,7 @@ class BaseTestWithPatchResponses:
 
  mock_requests = []
 
- def _mock_request(self, patch_request: PatchReqeust):
+ def _mock_request(self, patch_request: PatchRequest):
  """Mock response with file."""
  request_method = patch_request.request_method
  response = {}

django_project/gap/tests/providers/test_tio.py

@@ -281,7 +281,7 @@ def test_read_forecast_data_only(
  self.reader.end_date = dt_now
  self.reader.dataset.type = DatasetType(
  type=CastType.FORECAST, name='TestForecast')
- self.reader.start_date = dt_now - timedelta(days=1)
+ self.reader.start_date = dt_now - timedelta(days=7)
 
  # Call the read method
  with patch('gap.providers.tio.datetime') as mock_datetime:
@@ -290,7 +290,9 @@ def test_read_forecast_data_only(
 
  # Check that the correct method is called
  mock_read_forecast_data.assert_called_once()
- self.assertEqual(mock_read_forecast_data.call_args[0][0], dt_now)
+ self.assertEqual(
+ mock_read_forecast_data.call_args[0][0],
+ dt_now - timedelta(days=6))
  self.assertEqual(
  mock_read_forecast_data.call_args[0][1], self.reader.end_date)
  mock_read_historical_data.assert_not_called()