diff --git a/bims/migrations/0432_sitesetting_park_layer_csv.py b/bims/migrations/0432_sitesetting_park_layer_csv.py new file mode 100644 index 000000000..6783e777a --- /dev/null +++ b/bims/migrations/0432_sitesetting_park_layer_csv.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.11 on 2024-08-09 15:01 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('bims', '0431_auto_20240807_1231'), + ] + + operations = [ + migrations.AddField( + model_name='sitesetting', + name='park_layer_csv', + field=models.FileField(blank=True, null=True, upload_to='park_layer_csv/'), + ), + ] diff --git a/bims/models/site_setting.py b/bims/models/site_setting.py index 34400dc98..b4f8792f5 100644 --- a/bims/models/site_setting.py +++ b/bims/models/site_setting.py @@ -331,6 +331,12 @@ class SiteSetting(Preferences): blank=True ) + park_layer_csv = models.FileField( + null=True, + blank=True, + upload_to='park_layer_csv/' + ) + @property def project_name(self): if self.default_data_source: diff --git a/bims/scripts/occurrences_upload.py b/bims/scripts/occurrences_upload.py index 307d5a98e..6d54d59f3 100644 --- a/bims/scripts/occurrences_upload.py +++ b/bims/scripts/occurrences_upload.py @@ -1,3 +1,4 @@ +import csv import uuid import json import logging @@ -71,6 +72,7 @@ class OccurrenceProcessor(object): # collection data fetch_location_context = True park_centroid = {} + parks_data = {} def start_process(self): signals.post_save.disconnect( @@ -283,48 +285,78 @@ def location_site(self, record): accuracy_of_coordinates = 100 if not longitude and not latitude and park_name: + park_mpa_csv_file = preferences.SiteSetting.park_layer_csv wfs_url = preferences.SiteSetting.park_wfs_url layer_name = preferences.SiteSetting.park_wfs_layer_name attribute_key = preferences.SiteSetting.park_wfs_attribute_key attribute_value = park_name - if park_name in self.park_centroid: - latitude = self.park_centroid[park_name][0] - longitude = self.park_centroid[park_name][1] + if park_mpa_csv_file: + if not self.parks_data: + with open(park_mpa_csv_file.path, mode='r') as file: + reader = csv.DictReader(file, delimiter=',') + for row in reader: + _park_name = ( + row['Park_Name'].strip().lower() + ) + lat = float(row['x']) + lon = float(row['y']) + self.parks_data[_park_name] = { + 'lat': lat, + 'lon': lon + } + park_name_low = park_name.strip().lower() + if park_name_low in self.parks_data: + latitude = self.parks_data[ + park_name_low + ]['lat'] + longitude = self.parks_data[ + park_name_low + ]['lat'] + else: + self.handle_error( + row=record, + message='Park or MPA name does not exist in the database' + ) + return None else: - # Check if there is already site with the same park name - site = LocationSite.objects.filter( - name=park_name - ).first() - if site: - latitude = site.latitude - longitude = site.longitude - self.park_centroid[site.name] = [latitude, longitude] - # Check if site with same park name and accuracy of coordinates exists + if park_name in self.park_centroid: + latitude = self.park_centroid[park_name][0] + longitude = self.park_centroid[park_name][1] + else: + # Check if there is already site with the same park name site = LocationSite.objects.filter( - name=park_name, - accuracy_of_coordinates=accuracy_of_coordinates - ).exclude(site_code='').first() + name=park_name + ).first() if site: - # Return existing site - return site - else: - park_centroid = get_feature_centroid( - wfs_url, - layer_name, - attribute_key=attribute_key, - attribute_value=attribute_value - ) - if park_centroid: - latitude = park_centroid[0] - longitude = park_centroid[1] - self.park_centroid[park_name] = park_centroid + latitude = site.latitude + longitude = site.longitude + self.park_centroid[site.name] = [latitude, longitude] + # Check if site with same park name and accuracy of coordinates exists + site = LocationSite.objects.filter( + name=park_name, + accuracy_of_coordinates=accuracy_of_coordinates + ).exclude(site_code='').first() + if site: + # Return existing site + return site else: - self.handle_error( - row=record, - message='Park or MPA name does not exist in the database' + park_centroid = get_feature_centroid( + wfs_url, + layer_name, + attribute_key=attribute_key, + attribute_value=attribute_value ) - return None + if park_centroid: + latitude = park_centroid[0] + longitude = park_centroid[1] + self.park_centroid[park_name] = park_centroid + else: + self.handle_error( + row=record, + message='Park or MPA name does not exist in the database' + ) + return None if not longitude or not latitude: self.handle_error( diff --git a/bims/tests/data/csv_upload_test.csv b/bims/tests/data/csv_upload_test.csv index 88c9f4cd1..30f681151 100644 --- a/bims/tests/data/csv_upload_test.csv +++ b/bims/tests/data/csv_upload_test.csv @@ -1,4 +1,5 @@ UUID,User River Name,River Name,User Wetland Name,Wetland name,User Site Code,Site Code,Ecosystem type,Site description,User geomorphological zone,Latitude,Longitude,Park or MPA name,Sampling date,Kingdom,Phylum,Class,Order,Family,Genus,Species,SubSpecies,Taxon,Taxon rank,Sampling method,Sampling effort measure,Sampling effort value,Abundance measure,Abundance value,Hydroperiod,Wetland indicator status,Broad biotope,Specific biotope,Substratum,Origin,Endemism,Conservation status global,Conservation status national,Collector/owner,Collector/owner institute,Analyst,Analyst institute,Authors,Year,Source,Reference category,Title,Doi/url,Notes,Upstream id,Taxon key,Species key,Basis of record,Institution code,Collection code,Catalog number,Identified by,Rights holder,Recorded by,Decision support tool,Record type,Curation process,Biomass indicator: chl a,Biomass indicator: afdm,Autotrophic index (ai),Water level,Water turbidity,Embeddedness,Temperature (degrees c),Conductivity (ms/m),Ph (ph unit),Oxygen: dissolved (mg/l),Oxygen: dissolved: percentage saturation (%),Turbidity (ntu scale) (ntu),Depth of stream or river (m),Near bed velocity (m/s),Phosphorus: soluble reactive (mg/l),Phosphorus: total (mg/l),Nitrogen: ammonia (mg/l),Nitrogen: ammonium (mg/l),Nitrogen: nitrate + nitrite (mg/l),Nitrogen: nitrite (mg/l),Nitrogen: nitrate (mg/l),Nitrogen: total inorganic (mg/l),Chlorophyll a: benthic (mg/m2),Ash free dry mass: benthic (mg/m2),Sa province and sadc boundaries,Sa ecoregion level 2,Sa ecoregion level 1,Strategic water source areas - surface,Strategic water source areas - ground,Freshwater ecoregion of the world,Nfepa wetlands vegetation,Tertiary catchment area,Primary catchment area,Quaternary catchment area,Secondary catchment area,Quinary catchment area,Sub water management areas,Nfepa fish sanctuaries all species,Nfepa wetlands 2011,Nfepa rivers 2011,Nfepa fish sanctuaries 2011,Nfepa river fepas 2011,Nfepa wetland clusters 2011,National critical biodiversity,National ecological support areas,Present ecological state 1999,River condition 1999,Present ecological state 2018,Ecosystem threat status,Ecosystem protection level,Thermal framework,Hydrological regions,System resilience,Model accuracy,Hydrological region flow type,Geomorphological zone,Monthly average temperature january,Monthly average temperature february,Monthly average temperature march,Monthly average temperature april,Monthly average temperature may,Monthly average temperature june,Monthly average temperature july,Monthly average temperature august,Monthly average temperature september,Monthly average temperature october,Monthly average temperature november,Monthly average temperature december,Algae pesticide risk,Fish pesticide risk,Invertebrate pesticide risk,River management unit,Water management area,Nfepa quinaries,Division,Taxonomic status 5a08bfe1-0e9b-4e0e-bf30-5b50156d35a9,User River Name,Unknown,User Wetland Name,-,TEST-123,A1BRAK-00007,River,Soutpan,-,-24.746111,26.071944,,2023-10-23,Chromista,Ochrophyta,Bacillariophyceae,Naviculales,Achnanthaceae,Achnanthes,eutrophila,-,Achnanthes eutrophila,Species,In situ,min,12,Species valve/frustule count,3.0,-,-,Aquatic vegetation,Bedrock,Bedrock,Unknown,Unknown,Not evaluated,Not evaluated,Kartoza_Admin ,-,-,-,-,-,"FrogMap Virtual Museum, FitzPatrick Institute of African Ornithology, University of Cape Town",Database,"FrogMap Virtual Museum, FitzPatrick Institute of African Ornithology, University of Cape Town, 2022",,,,-,-,-,-,-,-,-,-,-,-,Photographic recor,Preserved lugols,Whole cobble,Whole cobble,13.0,,,,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,11.0,12.0,North West,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,Bacillariophyta,Accepted 5a08bfe1-0e9b-4e0e-bf30-5b50156d35a2,User River Name,Unknown,User Wetland Name,-,TEST-124,A1BRAK-00008,River,Soutpan,-,"","",test,2023-10-23,Chromista,Ochrophyta,Bacillariophyceae,Naviculales,Achnanthaceae,Achnanthes,eutrophila,-,Achnanthes eutrophila,Species,In situ,min,12,Species valve/frustule count,3.0,-,-,Aquatic vegetation,Bedrock,Bedrock,Unknown,Unknown,Not evaluated,Not evaluated,Kartoza_Admin ,-,-,-,-,-,"FrogMap Virtual Museum, FitzPatrick Institute of African Ornithology, University of Cape Town",Database,"FrogMap Virtual Museum, FitzPatrick Institute of African Ornithology, University of Cape Town, 2022",,,,-,-,-,-,-,-,-,-,-,-,Photographic recor,Preserved lugols,Whole cobble,Whole cobble,13.0,,,,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,11.0,12.0,North West,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,Bacillariophyta,Accepted -5a08bfe1-0e9b-4e0e-bf30-5b50156d35a1,User River Name,Unknown,User Wetland Name,-,TEST-125,A1BRAK-00009,River,Soutpan,-,"","",test,2023-10-23,Chromista,Ochrophyta,Bacillariophyceae,Naviculales,Achnanthaceae,Achnanthes,eutrophila,-,Achnanthes eutrophila,Species,In situ,min,12,Species valve/frustule count,3.0,-,-,Aquatic vegetation,Bedrock,Bedrock,Unknown,Unknown,Not evaluated,Not evaluated,Kartoza_Admin ,-,-,-,-,-,"FrogMap Virtual Museum, FitzPatrick Institute of African Ornithology, University of Cape Town",Database,"FrogMap Virtual Museum, FitzPatrick Institute of African Ornithology, University of Cape Town, 2022",,,,-,-,-,-,-,-,-,-,-,-,Photographic recor,Preserved lugols,Whole cobble,Whole cobble,13.0,,,,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,11.0,12.0,North West,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,Bacillariophyta,Accepted \ No newline at end of file +5a08bfe1-0e9b-4e0e-bf30-5b50156d35a1,User River Name,Unknown,User Wetland Name,-,TEST-125,A1BRAK-00009,River,Soutpan,-,"","",test,2023-10-23,Chromista,Ochrophyta,Bacillariophyceae,Naviculales,Achnanthaceae,Achnanthes,eutrophila,-,Achnanthes eutrophila,Species,In situ,min,12,Species valve/frustule count,3.0,-,-,Aquatic vegetation,Bedrock,Bedrock,Unknown,Unknown,Not evaluated,Not evaluated,Kartoza_Admin ,-,-,-,-,-,"FrogMap Virtual Museum, FitzPatrick Institute of African Ornithology, University of Cape Town",Database,"FrogMap Virtual Museum, FitzPatrick Institute of African Ornithology, University of Cape Town, 2022",,,,-,-,-,-,-,-,-,-,-,-,Photographic recor,Preserved lugols,Whole cobble,Whole cobble,13.0,,,,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,11.0,12.0,North West,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,Bacillariophyta,Accepted +5a08bfe1-0e9b-4e0e-bf30-5b50156d35a3,User River Name,Unknown,User Wetland Name,-,TEST-125,A1BRAK-00009,River,Soutpan,-,"","",Park A ,2023-10-23,Chromista,Ochrophyta,Bacillariophyceae,Naviculales,Achnanthaceae,Achnanthes,eutrophila,-,Achnanthes eutrophila,Species,In situ,min,12,Species valve/frustule count,3.0,-,-,Aquatic vegetation,Bedrock,Bedrock,Unknown,Unknown,Not evaluated,Not evaluated,Kartoza_Admin ,-,-,-,-,-,"FrogMap Virtual Museum, FitzPatrick Institute of African Ornithology, University of Cape Town",Database,"FrogMap Virtual Museum, FitzPatrick Institute of African Ornithology, University of Cape Town, 2022",,,,-,-,-,-,-,-,-,-,-,-,Photographic recor,Preserved lugols,Whole cobble,Whole cobble,13.0,,,,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,,,North West,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,-,Bacillariophyta,Accepted \ No newline at end of file diff --git a/bims/tests/data/park_data.csv b/bims/tests/data/park_data.csv new file mode 100644 index 000000000..3f521125e --- /dev/null +++ b/bims/tests/data/park_data.csv @@ -0,0 +1,3 @@ +Park_Name,x,y +Park A,34.0522,-118.2437 +Park B,36.1699,-115.1398 \ No newline at end of file diff --git a/bims/tests/test_occurrences_upload.py b/bims/tests/test_occurrences_upload.py index 512734f0e..f814fcbbb 100644 --- a/bims/tests/test_occurrences_upload.py +++ b/bims/tests/test_occurrences_upload.py @@ -1,7 +1,11 @@ +import io import os import mock import json +from django.core.files.uploadedfile import SimpleUploadedFile +from django_tenants.test.cases import FastTenantTestCase +from preferences import preferences from django.db.models import Q from django.test import TestCase, override_settings from django.core.files import File @@ -60,7 +64,7 @@ def mocked_doi_loader( return '' -class TestCollectionUpload(TestCase): +class TestCollectionUpload(FastTenantTestCase): document_link = 'site/document/11' owner = None reference_title = 'title' @@ -160,19 +164,32 @@ def test_reference_database_created(self): ) @override_settings(GEOCONTEXT_URL="test.gecontext.com") - @mock.patch('requests.get', mock.Mock( - side_effect=mocked_location_context_data)) - @mock.patch.object(SiteSetting, 'default_data_source', new_callable=mock.PropertyMock) + @mock.patch('requests.get', mock.Mock(side_effect=mocked_location_context_data)) @mock.patch('bims.scripts.data_upload.DataCSVUpload.finish') @mock.patch('bims.scripts.occurrences_upload.OccurrenceProcessor.update_location_site_context') @mock.patch('bims.scripts.occurrences_upload.get_feature_centroid') def test_csv_upload(self, mock_get_feature_centroid, mock_update_location_context, - mock_finish, - mock_default_data_source): + mock_finish): + + site_setting = preferences.SiteSetting + + if not site_setting: + site_setting = SiteSetting.objects.create() + + if site_setting: + csv_content = b"Park_Name,x,y\nPark A,34.0522,-118.2437\nPark B,36.1699,-115.1398\n" + csv_file = SimpleUploadedFile( + "park_data.csv", + csv_content, + content_type="text/csv" + ) + site_setting.default_data_source = 'fbis' + site_setting.park_layer_csv = csv_file + site_setting.save() + mock_finish.return_value = None - mock_default_data_source.return_value = "fbis" mock_get_feature_centroid.return_value = (1, 1) @@ -247,5 +264,8 @@ def test_csv_upload(self, ) self.assertEqual(bio.count(), 1) self.assertEqual(bio.first().site.legacy_river_name, 'User River Name 2') - mock_get_feature_centroid.assert_called_once_with( - 'https://maps.kartoza.com/geoserver/wfs', '', attribute_key='', attribute_value='test') + self.assertTrue( + BiologicalCollectionRecord.objects.filter( + site__name='Park A' + ).exists() + ) diff --git a/bims/utils/user.py b/bims/utils/user.py index 7019a1504..8be926f34 100644 --- a/bims/utils/user.py +++ b/bims/utils/user.py @@ -3,6 +3,9 @@ from django.utils.text import slugify from django.db.models import Q +from geonode.people.models import Profile +from bims.models.profile import Profile as BimsProfile + def get_user_from_name(first_name, last_name): """ @@ -16,7 +19,6 @@ def get_user_from_name(first_name, last_name): User = get_user_model() try: if last_name.strip(): - print(last_name) user = User.objects.get( Q(last_name__iexact=last_name), Q(first_name__iexact=first_name) | @@ -26,7 +28,7 @@ def get_user_from_name(first_name, last_name): user = User.objects.get( Q(first_name__iexact=first_name) ) - except User.DoesNotExist: + except (User.DoesNotExist, Profile.DoesNotExist, BimsProfile.DoesNotExist): username = slugify('{first_name} {last_name}'.format( first_name=first_name, last_name=last_name