Get centroid of park from csv (#4140)

* Get centroid of park from csv

* Update uploader and tests

bims/migrations/0432_sitesetting_park_layer_csv.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.11 on 2024-08-09 15:01
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('bims', '0431_auto_20240807_1231'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='sitesetting',
+ name='park_layer_csv',
+ field=models.FileField(blank=True, null=True, upload_to='park_layer_csv/'),
+ ),
+ ]

bims/models/site_setting.py

@@ -331,6 +331,12 @@ class SiteSetting(Preferences):
  blank=True
  )
 
+ park_layer_csv = models.FileField(
+ null=True,
+ blank=True,
+ upload_to='park_layer_csv/'
+ )
+
  @property
  def project_name(self):
  if self.default_data_source:

bims/scripts/occurrences_upload.py

@@ -1,3 +1,4 @@
+import csv
 import uuid
 import json
 import logging
@@ -71,6 +72,7 @@ class OccurrenceProcessor(object):
  # collection data
  fetch_location_context = True
  park_centroid = {}
+ parks_data = {}
 
  def start_process(self):
  signals.post_save.disconnect(
@@ -283,48 +285,78 @@ def location_site(self, record):
  accuracy_of_coordinates = 100
 
  if not longitude and not latitude and park_name:
+ park_mpa_csv_file = preferences.SiteSetting.park_layer_csv
  wfs_url = preferences.SiteSetting.park_wfs_url
  layer_name = preferences.SiteSetting.park_wfs_layer_name
  attribute_key = preferences.SiteSetting.park_wfs_attribute_key
  attribute_value = park_name
 
- if park_name in self.park_centroid:
- latitude = self.park_centroid[park_name][0]
- longitude = self.park_centroid[park_name][1]
+ if park_mpa_csv_file:
+ if not self.parks_data:
+ with open(park_mpa_csv_file.path, mode='r') as file:
+ reader = csv.DictReader(file, delimiter=',')
+ for row in reader:
+ _park_name = (
+ row['Park_Name'].strip().lower()
+ )
+ lat = float(row['x'])
+ lon = float(row['y'])
+ self.parks_data[_park_name] = {
+ 'lat': lat,
+ 'lon': lon
+ }
+ park_name_low = park_name.strip().lower()
+ if park_name_low in self.parks_data:
+ latitude = self.parks_data[
+ park_name_low
+ ]['lat']
+ longitude = self.parks_data[
+ park_name_low
+ ]['lat']
+ else:
+ self.handle_error(
+ row=record,
+ message='Park or MPA name does not exist in the database'
+ )
+ return None
  else:
- # Check if there is already site with the same park name
- site = LocationSite.objects.filter(
- name=park_name
- ).first()
- if site:
- latitude = site.latitude
- longitude = site.longitude
- self.park_centroid[site.name] = [latitude, longitude]
- # Check if site with same park name and accuracy of coordinates exists
+ if park_name in self.park_centroid:
+ latitude = self.park_centroid[park_name][0]
+ longitude = self.park_centroid[park_name][1]
+ else:
+ # Check if there is already site with the same park name
  site = LocationSite.objects.filter(
- name=park_name,
- accuracy_of_coordinates=accuracy_of_coordinates
- ).exclude(site_code='').first()
+ name=park_name
+ ).first()
  if site:
- # Return existing site
- return site
- else:
- park_centroid = get_feature_centroid(
- wfs_url,
- layer_name,
- attribute_key=attribute_key,
- attribute_value=attribute_value
- )
- if park_centroid:
- latitude = park_centroid[0]
- longitude = park_centroid[1]
- self.park_centroid[park_name] = park_centroid
+ latitude = site.latitude
+ longitude = site.longitude
+ self.park_centroid[site.name] = [latitude, longitude]
+ # Check if site with same park name and accuracy of coordinates exists
+ site = LocationSite.objects.filter(
+ name=park_name,
+ accuracy_of_coordinates=accuracy_of_coordinates
+ ).exclude(site_code='').first()
+ if site:
+ # Return existing site
+ return site
  else:
- self.handle_error(
- row=record,
- message='Park or MPA name does not exist in the database'
+ park_centroid = get_feature_centroid(
+ wfs_url,
+ layer_name,
+ attribute_key=attribute_key,
+ attribute_value=attribute_value
  )
- return None
+ if park_centroid:
+ latitude = park_centroid[0]
+ longitude = park_centroid[1]
+ self.park_centroid[park_name] = park_centroid
+ else:
+ self.handle_error(
+ row=record,
+ message='Park or MPA name does not exist in the database'
+ )
+ return None
 
  if not longitude or not latitude:
  self.handle_error(

bims/tests/data/csv_upload_test.csv

@@ -1,4 +1,5 @@
 UUID,User River Name,River Name,User Wetland Name,Wetland name,User Site Code,Site Code,Ecosystem type,Site description,User geomorphological zone,Latitude,Longitude,Park or MPA name,Sampling date,Kingdom,Phylum,Class,Order,Family,Genus,Species,SubSpecies,Taxon,Taxon rank,Sampling method,Sampling effort measure,Sampling effort value,Abundance measure,Abundance value,Hydroperiod,Wetland indicator status,Broad biotope,Specific biotope,Substratum,Origin,Endemism,Conservation status global,Conservation status national,Collector/owner,Collector/owner institute,Analyst,Analyst institute,Authors,Year,Source,Reference category,Title,Doi/url,Notes,Upstream id,Taxon key,Species key,Basis of record,Institution code,Collection code,Catalog number,Identified by,Rights holder,Recorded by,Decision support tool,Record type,Curation process,Biomass indicator: chl a,Biomass indicator: afdm,Autotrophic index (ai),Water level,Water turbidity,Embeddedness,Temperature (degrees c),Conductivity (ms/m),Ph (ph unit),Oxygen: dissolved (mg/l),Oxygen: dissolved: percentage saturation (%),Turbidity (ntu scale) (ntu),Depth of stream or river (m),Near bed velocity (m/s),Phosphorus: soluble reactive (mg/l),Phosphorus: total (mg/l),Nitrogen: ammonia (mg/l),Nitrogen: ammonium (mg/l),Nitrogen: nitrate + nitrite (mg/l),Nitrogen: nitrite (mg/l),Nitrogen: nitrate (mg/l),Nitrogen: total inorganic (mg/l),Chlorophyll a: benthic (mg/m2),Ash free dry mass: benthic (mg/m2),Sa province and sadc boundaries,Sa ecoregion level 2,Sa ecoregion level 1,Strategic water source areas - surface,Strategic water source areas - ground,Freshwater ecoregion of the world,Nfepa wetlands vegetation,Tertiary catchment area,Primary catchment area,Quaternary catchment area,Secondary catchment area,Quinary catchment area,Sub water management areas,Nfepa fish sanctuaries all species,Nfepa wetlands 2011,Nfepa rivers 2011,Nfepa fish sanctuaries 2011,Nfepa river fepas 2011,Nfepa wetland clusters 2011,National critical biodiversity,National ecological support areas,Present ecological state 1999,River condition 1999,Present ecological state 2018,Ecosystem threat status,Ecosystem protection level,Thermal framework,Hydrological regions,System resilience,Model accuracy,Hydrological region flow type,Geomorphological zone,Monthly average temperature january,Monthly average temperature february,Monthly average temperature march,Monthly average temperature april,Monthly average temperature may,Monthly average temperature june,Monthly average temperature july,Monthly average temperature august,Monthly average temperature september,Monthly average temperature october,Monthly average temperature november,Monthly average temperature december,Algae pesticide risk,Fish pesticide risk,Invertebrate pesticide risk,River management unit,Water management area,Nfepa quinaries,Division,Taxonomic status
 5a08bfe1-0e9b-4e0e-bf30-5b50156d35a9,User River Name,Unknown,User Wetland Name,-,TEST-123,A1BRAK-00007,River,Soutpan,-,-24.746111,26.071944,,2023-10-23,Chromista,Ochrophyta,Bacillariophyceae,Naviculales,Achnanthaceae,Achnanthes,eutrophila,-,Achnanthes eutrophila,Species,In situ,min,12,Species valve/frustule count,3.0,-,-,Aquatic vegetation,Bedrock,Bedrock,Unknown,Unknown,Not evaluated,Not evaluated,Kartoza_Admin ,-,-,-,-,-,"FrogMap Virtual Museum, FitzPatrick Institute of African Ornithology, University of Cape Town",Database,"FrogMap Virtual Museum, FitzPatrick Institute of African Ornithology, University of Cape Town, 2022",,,,-,-,-,-,-,-,-,-,-,-,Photographic recor,Preserved lugols,Whole cobble,Whole cobble,13.0,,,,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,11.0,12.0,North West,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,Bacillariophyta,Accepted
 5a08bfe1-0e9b-4e0e-bf30-5b50156d35a2,User River Name,Unknown,User Wetland Name,-,TEST-124,A1BRAK-00008,River,Soutpan,-,"","",test,2023-10-23,Chromista,Ochrophyta,Bacillariophyceae,Naviculales,Achnanthaceae,Achnanthes,eutrophila,-,Achnanthes eutrophila,Species,In situ,min,12,Species valve/frustule count,3.0,-,-,Aquatic vegetation,Bedrock,Bedrock,Unknown,Unknown,Not evaluated,Not evaluated,Kartoza_Admin ,-,-,-,-,-,"FrogMap Virtual Museum, FitzPatrick Institute of African Ornithology, University of Cape Town",Database,"FrogMap Virtual Museum, FitzPatrick Institute of African Ornithology, University of Cape Town, 2022",,,,-,-,-,-,-,-,-,-,-,-,Photographic recor,Preserved lugols,Whole cobble,Whole cobble,13.0,,,,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,11.0,12.0,North West,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,Bacillariophyta,Accepted
-5a08bfe1-0e9b-4e0e-bf30-5b50156d35a1,User River Name,Unknown,User Wetland Name,-,TEST-125,A1BRAK-00009,River,Soutpan,-,"","",test,2023-10-23,Chromista,Ochrophyta,Bacillariophyceae,Naviculales,Achnanthaceae,Achnanthes,eutrophila,-,Achnanthes eutrophila,Species,In situ,min,12,Species valve/frustule count,3.0,-,-,Aquatic vegetation,Bedrock,Bedrock,Unknown,Unknown,Not evaluated,Not evaluated,Kartoza_Admin ,-,-,-,-,-,"FrogMap Virtual Museum, FitzPatrick Institute of African Ornithology, University of Cape Town",Database,"FrogMap Virtual Museum, FitzPatrick Institute of African Ornithology, University of Cape Town, 2022",,,,-,-,-,-,-,-,-,-,-,-,Photographic recor,Preserved lugols,Whole cobble,Whole cobble,13.0,,,,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,11.0,12.0,North West,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,Bacillariophyta,Accepted
+5a08bfe1-0e9b-4e0e-bf30-5b50156d35a1,User River Name,Unknown,User Wetland Name,-,TEST-125,A1BRAK-00009,River,Soutpan,-,"","",test,2023-10-23,Chromista,Ochrophyta,Bacillariophyceae,Naviculales,Achnanthaceae,Achnanthes,eutrophila,-,Achnanthes eutrophila,Species,In situ,min,12,Species valve/frustule count,3.0,-,-,Aquatic vegetation,Bedrock,Bedrock,Unknown,Unknown,Not evaluated,Not evaluated,Kartoza_Admin ,-,-,-,-,-,"FrogMap Virtual Museum, FitzPatrick Institute of African Ornithology, University of Cape Town",Database,"FrogMap Virtual Museum, FitzPatrick Institute of African Ornithology, University of Cape Town, 2022",,,,-,-,-,-,-,-,-,-,-,-,Photographic recor,Preserved lugols,Whole cobble,Whole cobble,13.0,,,,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,11.0,12.0,North West,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,Bacillariophyta,Accepted
+5a08bfe1-0e9b-4e0e-bf30-5b50156d35a3,User River Name,Unknown,User Wetland Name,-,TEST-125,A1BRAK-00009,River,Soutpan,-,"","",Park A ,2023-10-23,Chromista,Ochrophyta,Bacillariophyceae,Naviculales,Achnanthaceae,Achnanthes,eutrophila,-,Achnanthes eutrophila,Species,In situ,min,12,Species valve/frustule count,3.0,-,-,Aquatic vegetation,Bedrock,Bedrock,Unknown,Unknown,Not evaluated,Not evaluated,Kartoza_Admin ,-,-,-,-,-,"FrogMap Virtual Museum, FitzPatrick Institute of African Ornithology, University of Cape Town",Database,"FrogMap Virtual Museum, FitzPatrick Institute of African Ornithology, University of Cape Town, 2022",,,,-,-,-,-,-,-,-,-,-,-,Photographic recor,Preserved lugols,Whole cobble,Whole cobble,13.0,,,,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,,,North West,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,Bacillariophyta,Accepted

bims/tests/data/park_data.csv

@@ -0,0 +1,3 @@
+Park_Name,x,y
+Park A,34.0522,-118.2437
+Park B,36.1699,-115.1398

bims/tests/test_occurrences_upload.py

@@ -1,7 +1,11 @@
+import io
 import os
 import mock
 import json
 
+from django.core.files.uploadedfile import SimpleUploadedFile
+from django_tenants.test.cases import FastTenantTestCase
+from preferences import preferences
 from django.db.models import Q
 from django.test import TestCase, override_settings
 from django.core.files import File
@@ -60,7 +64,7 @@ def mocked_doi_loader(
  return ''
 
 
-class TestCollectionUpload(TestCase):
+class TestCollectionUpload(FastTenantTestCase):
  document_link = 'site/document/11'
  owner = None
  reference_title = 'title'
@@ -160,19 +164,32 @@ def test_reference_database_created(self):
  )
 
  @override_settings(GEOCONTEXT_URL="test.gecontext.com")
- @mock.patch('requests.get', mock.Mock(
- side_effect=mocked_location_context_data))
- @mock.patch.object(SiteSetting, 'default_data_source', new_callable=mock.PropertyMock)
+ @mock.patch('requests.get', mock.Mock(side_effect=mocked_location_context_data))
  @mock.patch('bims.scripts.data_upload.DataCSVUpload.finish')
  @mock.patch('bims.scripts.occurrences_upload.OccurrenceProcessor.update_location_site_context')
  @mock.patch('bims.scripts.occurrences_upload.get_feature_centroid')
  def test_csv_upload(self,
  mock_get_feature_centroid,
  mock_update_location_context,
- mock_finish,
- mock_default_data_source):
+ mock_finish):
+
+ site_setting = preferences.SiteSetting
+
+ if not site_setting:
+ site_setting = SiteSetting.objects.create()
+
+ if site_setting:
+ csv_content = b"Park_Name,x,y\nPark A,34.0522,-118.2437\nPark B,36.1699,-115.1398\n"
+ csv_file = SimpleUploadedFile(
+ "park_data.csv",
+ csv_content,
+ content_type="text/csv"
+ )
+ site_setting.default_data_source = 'fbis'
+ site_setting.park_layer_csv = csv_file
+ site_setting.save()
+
  mock_finish.return_value = None
- mock_default_data_source.return_value = "fbis"
 
  mock_get_feature_centroid.return_value = (1, 1)
 
@@ -247,5 +264,8 @@ def test_csv_upload(self,
  )
  self.assertEqual(bio.count(), 1)
  self.assertEqual(bio.first().site.legacy_river_name, 'User River Name 2')
- mock_get_feature_centroid.assert_called_once_with(
- 'https://maps.kartoza.com/geoserver/wfs', '', attribute_key='', attribute_value='test')
+ self.assertTrue(
+ BiologicalCollectionRecord.objects.filter(
+ site__name='Park A'
+ ).exists()
+ )

bims/utils/user.py

@@ -3,6 +3,9 @@
 from django.utils.text import slugify
 from django.db.models import Q
 
+from geonode.people.models import Profile
+from bims.models.profile import Profile as BimsProfile
+
 
 def get_user_from_name(first_name, last_name):
  """
@@ -16,7 +19,6 @@ def get_user_from_name(first_name, last_name):
  User = get_user_model()
  try:
  if last_name.strip():
- print(last_name)
  user = User.objects.get(
  Q(last_name__iexact=last_name),
  Q(first_name__iexact=first_name) |
@@ -26,7 +28,7 @@ def get_user_from_name(first_name, last_name):
  user = User.objects.get(
  Q(first_name__iexact=first_name)
  )
- except User.DoesNotExist:
+ except (User.DoesNotExist, Profile.DoesNotExist, BimsProfile.DoesNotExist):
  username = slugify('{first_name} {last_name}'.format(
  first_name=first_name,
  last_name=last_name