Update spw mode to also have new attributes (#161)

* Update spw mode to also have new attributes

* Fic conflict

* Update default fields

* Bump version and put the crop file to group folder

* Update generate_crop_plan task

django_project/_version.txt

@@ -1 +1 @@
-0.0.2
+0.0.3

django_project/core/admin.py

@@ -111,8 +111,10 @@ def cancel_background_task(modeladmin, request, queryset):
 class BackgroundTaskAdmin(admin.ModelAdmin):
  """Admin class for BackgroundTask model."""
 
- list_display = ('task_name', 'task_id', 'status', 'started_at',
- 'finished_at', 'last_update')
+ list_display = (
+ 'task_name', 'task_id', 'status', 'started_at',
+ 'finished_at', 'last_update', 'context_id'
+ )
  search_fields = ['task_name', 'status', 'task_id']
  actions = [cancel_background_task]
  list_filter = ["status", "task_name"]

django_project/core/settings/project.py

@@ -7,6 +7,7 @@
 import os # noqa
 
 from boto3.s3.transfer import TransferConfig
+
 from .contrib import * # noqa
 from .utils import absolute_path
 
@@ -69,3 +70,5 @@
 STORAGE_DIR_PREFIX = os.environ.get("MINIO_AWS_DIR_PREFIX", "media")
 if STORAGE_DIR_PREFIX and not STORAGE_DIR_PREFIX.endswith("/"):
  STORAGE_DIR_PREFIX = f"{STORAGE_DIR_PREFIX}/"
+
+DATA_UPLOAD_MAX_NUMBER_FIELDS = 1500

django_project/gap/migrations/0025_farmsuitableplantingwindowsignal_last_2_days_and_more.py

@@ -0,0 +1,38 @@
+# Generated by Django 4.2.7 on 2024-10-01 02:12
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('gap', '0024_alter_farm_category_alter_farm_rsvp_status'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='farmsuitableplantingwindowsignal',
+ name='last_2_days',
+ field=models.FloatField(blank=True, help_text='The rain accumulationSum for last 2 days.', null=True),
+ ),
+ migrations.AddField(
+ model_name='farmsuitableplantingwindowsignal',
+ name='last_4_days',
+ field=models.FloatField(blank=True, help_text='The rain accumulationSum for last 4 days.', null=True),
+ ),
+ migrations.AddField(
+ model_name='farmsuitableplantingwindowsignal',
+ name='today_tomorrow',
+ field=models.FloatField(blank=True, help_text='The rain accumulationSum for today and tomorrow.', null=True),
+ ),
+ migrations.AddField(
+ model_name='farmsuitableplantingwindowsignal',
+ name='too_wet_indicator',
+ field=models.CharField(blank=True, help_text='Too wet indicator.', max_length=512, null=True),
+ ),
+ migrations.AlterField(
+ model_name='farmsuitableplantingwindowsignal',
+ name='signal',
+ field=models.CharField(help_text='GoNoGo signals.', max_length=512),
+ ),
+ ]

django_project/gap/models/crop_insight.py

@@ -4,7 +4,7 @@
 
 .. note:: Models
 """
-
+import os.path
 import uuid
 from datetime import date, timedelta
 
@@ -27,6 +27,14 @@
 User = get_user_model()
 
 
+class FarmGroupIsNotSetException(Exception):
+ """Farm group is not set."""
+
+ def __init__(self): # noqa
+ self.message = 'Farm group is not set.'
+ super().__init__(self.message)
+
+
 def ingestor_file_path(instance, filename):
  """Return upload path for Ingestor files."""
  return f'{settings.STORAGE_DIR_PREFIX}crop-insight/{filename}'
@@ -171,7 +179,24 @@ class FarmSuitablePlantingWindowSignal(models.Model):
  )
  signal = models.CharField(
  max_length=512,
- help_text='Signal value of Suitable Planting Window l.'
+ help_text='GoNoGo signals.'
+ )
+ too_wet_indicator = models.CharField(
+ max_length=512,
+ help_text='Too wet indicator.',
+ null=True, blank=True
+ )
+ last_4_days = models.FloatField(
+ help_text='The rain accumulationSum for last 4 days.',
+ null=True, blank=True
+ )
+ last_2_days = models.FloatField(
+ help_text='The rain accumulationSum for last 2 days.',
+ null=True, blank=True
+ )
+ today_tomorrow = models.FloatField(
+ help_text='The rain accumulationSum for today and tomorrow.',
+ null=True, blank=True
  )
 
  def __str__(self):
@@ -288,6 +313,22 @@ def default_fields():
  'longitude',
  'SPWTopMessage',
  'SPWDescription',
+ 'TooWet',
+ 'last_4_days_mm',
+ 'last_2_days_mm',
+ 'today_tomorrow_mm'
+ ]
+
+ @staticmethod
+ def default_fields_used():
+ """Return list of default fields that being used by crop insight."""
+ return [
+ 'farmID',
+ 'phoneNumber',
+ 'latitude',
+ 'longitude',
+ 'SPWTopMessage',
+ 'SPWDescription'
  ]
 
  def __init__(
@@ -362,6 +403,10 @@ def data(self) -> dict:
  # Spw data
  spw_top_message = ''
  spw_description = ''
+ too_wet = ''
+ last_4_days = ''
+ last_2_days = ''
+ today_tomorrow = ''
  spw = FarmSuitablePlantingWindowSignal.objects.filter(
  farm=self.farm,
  generated_date=self.generated_date
@@ -374,15 +419,31 @@ def data(self) -> dict:
  except SPWOutput.DoesNotExist:
  spw_top_message = spw.signal
  spw_description = ''
- # ---------------------------------------
 
+ if spw.too_wet_indicator is not None:
+ too_wet = spw.too_wet_indicator
+ if spw.last_4_days is not None:
+ last_4_days = spw.last_4_days
+ if spw.last_2_days is not None:
+ last_2_days = spw.last_2_days
+ if spw.today_tomorrow is not None:
+ today_tomorrow = spw.today_tomorrow
+
+ # ---------------------------------------
+ # Check default_fields functions
+ default_fields = CropPlanData.default_fields()
  output = {
- 'farmID': self.farm.unique_id,
- 'phoneNumber': self.farm.phone_number,
- 'latitude': self.latitude,
- 'longitude': self.longitude,
- 'SPWTopMessage': spw_top_message,
- 'SPWDescription': spw_description,
+ default_fields[0]: self.farm.unique_id,
+ default_fields[1]: self.farm.phone_number,
+ default_fields[2]: self.latitude,
+ default_fields[3]: self.longitude,
+ default_fields[4]: spw_top_message,
+ default_fields[5]: spw_description,
+ default_fields[6]: too_wet,
+ default_fields[7]: last_4_days,
+ default_fields[8]: last_2_days,
+ default_fields[9]: today_tomorrow
+
  }
 
  # ----------------------------------------
@@ -542,9 +603,10 @@ def _generate_report(self):
  from spw.generator.crop_insight import CropInsightFarmGenerator
 
  # If farm is empty, put empty farm
- farms = []
  if self.farm_group:
  farms = self.farm_group.farms.all()
+ else:
+ raise FarmGroupIsNotSetException()
 
  output = [
  self.farm_group.headers
@@ -583,7 +645,10 @@ def _generate_report(self):
  for row in output:
  csv_content += ','.join(map(str, row)) + '\n'
  content_file = ContentFile(csv_content)
- self.file.save(f'{self.unique_id}.csv', content_file)
+ self.file.save(
+ os.path.join(f'{self.farm_group.id}', f'{self.unique_id}.csv'),
+ content_file
+ )
  self.save()
 
  # Send email

django_project/gap/models/farm_group.py

@@ -51,11 +51,13 @@ def prepare_fields(self):
  self.farmgroupcropinsightfield_set.all().delete()
  column_num = 1
  for default_field in CropPlanData.default_fields():
+ active = default_field in CropPlanData.default_fields_used()
  FarmGroupCropInsightField.objects.update_or_create(
  farm_group=self,
  field=default_field,
  defaults={
- 'column_number': column_num
+ 'column_number': column_num,
+ 'active': active
  }
  )
  column_num += 1

django_project/gap/tasks/crop_insight.py

@@ -42,8 +42,7 @@ def generate_crop_plan():
  requested_by=user,
  farm_group=group,
  )
- # generate report
- request.run()
+ generate_insight_report.delay(request.id)
 
 
 @app.task(name="retry_crop_plan_generators")

django_project/spw/generator/crop_insight.py

@@ -35,14 +35,29 @@ def __init__(self, farm: Farm):
  self.tomorrow = self.today + timedelta(days=1)
  self.attributes = calculate_from_point_attrs()
 
- def save_spw(self, signal, farm: Farm):
+ def return_float(self, value):
+ """Return float value."""
+ try:
+ return float(value)
+ except ValueError:
+ return None
+
+ def save_spw(
+ self, farm: Farm, signal, too_wet_indicator, last_4_days,
+ last_2_days, today_tomorrow
+ ):
  """Save spw data."""
  # Save SPW
  FarmSuitablePlantingWindowSignal.objects.update_or_create(
  farm=farm,
  generated_date=self.today,
  defaults={
- 'signal': signal
+ 'signal': signal,
+ 'too_wet_indicator': too_wet_indicator,
+ 'last_4_days': self.return_float(last_4_days),
+ 'last_2_days': self.return_float(last_2_days),
+ 'today_tomorrow': self.return_float(today_tomorrow)
+
  }
  )
 
@@ -124,5 +139,9 @@ def _generate_spw(self):
  farms = Farm.objects.filter(grid=self.farm.grid)
 
  for farm in farms:
- self.save_spw(output.data.goNoGo, farm)
+ self.save_spw(
+ farm,
+ output.data.goNoGo, output.data.tooWet, output.data.last4Days,
+ output.data.last2Days, output.data.todayTomorrow
+ )
  self.save_shortterm_forecast(historical_dict, farm)

django_project/spw/migrations/0003_alter_rmodeloutput_type.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.7 on 2024-09-27 06:22
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('spw', '0002_spwoutput'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='rmodeloutput',
+ name='type',
+ field=models.CharField(choices=[('goNoGo', 'goNoGo'), ('days_h2to_f2', 'days_h2to_f2'), ('days_f3to_f5', 'days_f3to_f5'), ('days_f6to_f13', 'days_f6to_f13'), ('nearDaysLTNPercent', 'nearDaysLTNPercent'), ('nearDaysCurPercent', 'nearDaysCurPercent'), ('tooWet', 'tooWet'), ('last4Days', 'last4Days'), ('last2Days', 'last2Days'), ('todayTomorrow', 'todayTomorrow')], max_length=100),
+ ),
+ ]

django_project/spw/models.py

@@ -45,6 +45,10 @@ class RModelOutputType:
  DAYS_f6TO_F13 = 'days_f6to_f13'
  NEAR_DAYS_LTN_PERCENT = 'nearDaysLTNPercent'
  NEAR_DAYS_CUR_PERCENT = 'nearDaysCurPercent'
+ TOO_WET_STATUS = 'tooWet'
+ LAST_4_DAYS = 'last4Days'
+ LAST_2_DAYS = 'last2Days'
+ TODAY_TOMORROW = 'todayTomorrow'
 
 
 class RModelOutput(models.Model):
@@ -66,6 +70,14 @@ class RModelOutput(models.Model):
  RModelOutputType.NEAR_DAYS_LTN_PERCENT),
  (RModelOutputType.NEAR_DAYS_CUR_PERCENT,
  RModelOutputType.NEAR_DAYS_CUR_PERCENT),
+ (RModelOutputType.TOO_WET_STATUS,
+ RModelOutputType.TOO_WET_STATUS),
+ (RModelOutputType.LAST_4_DAYS,
+ RModelOutputType.LAST_4_DAYS),
+ (RModelOutputType.LAST_2_DAYS,
+ RModelOutputType.LAST_2_DAYS),
+ (RModelOutputType.TODAY_TOMORROW,
+ RModelOutputType.TODAY_TOMORROW),
  )
  )
  variable_name = models.CharField(max_length=100)

django_project/spw/tests/test_crop_insight_generator.py

@@ -18,7 +18,8 @@
 from gap.factories.farm import FarmFactory, FarmGroupFactory
 from gap.factories.grid import GridFactory
 from gap.models.crop_insight import (
- FarmSuitablePlantingWindowSignal, CropInsightRequest
+ FarmSuitablePlantingWindowSignal, CropInsightRequest,
+ FarmGroupIsNotSetException
 )
 from gap.models.preferences import Preferences
 from gap.tasks.crop_insight import (
@@ -183,6 +184,10 @@ def create_timeline_data(
  'goNoGo': ['Plant NOW Tier 1b'],
  'nearDaysLTNPercent': [10.0],
  'nearDaysCurPercent': [60.0],
+ 'tooWet': ['Likely too wet to plant'],
+ 'last4Days': [80],
+ 'last2Days': [60],
+ 'todayTomorrow': [40],
  }
  )
  fetch_timelines_data_val = {}
@@ -233,6 +238,10 @@ def create_timeline_data(
  'goNoGo': ['Do NOT plant, DRY Tier 4b'],
  'nearDaysLTNPercent': [10.0],
  'nearDaysCurPercent': [60.0],
+ 'tooWet': ['Too wet to plant'],
+ 'last4Days': [100],
+ 'last2Days': [80],
+ 'todayTomorrow': [80],
  }
  )
  fetch_timelines_data_val = {}
@@ -264,16 +273,29 @@ def create_timeline_data(
  'goNoGo': '',
  'nearDaysLTNPercent': [10.0],
  'nearDaysCurPercent': [60.0],
+ 'tooWet': '',
+ 'last4Days': '',
+ 'last2Days': '',
+ 'todayTomorrow': '',
  }
  )
  mock_fetch_timelines_data.return_value = {}
 
+ # Farm group is required, raise error
+ with self.assertRaises(FarmGroupIsNotSetException):
+ request = CropInsightRequestFactory.create()
+ generate_insight_report(request.id)
+
  # Crop insight report
  self.request = CropInsightRequestFactory.create(
  farm_group=self.farm_group
  )
  generate_insight_report(self.request.id)
  self.request.refresh_from_db()
+
+ # Check the if of farm group in the path
+ self.assertTrue(f'{self.farm_group.id}/' in self.request.file.path)
+
  with self.request.file.open(mode='r') as csv_file:
  csv_reader = csv.reader(csv_file)
  row_num = 1