Improve query (#4289)

* Improve query

* Fix map dashboard

bims/api_views/location_site_overview.py

@@ -3,12 +3,14 @@
 import time
 from collections import OrderedDict
 
+from braces.views import SuperuserRequiredMixin
 from django.contrib.sites.models import Site
+from django.db import connection
 
 from bims.models.search_process import SITES_SUMMARY, SEARCH_PROCESSING
 
 from bims.models.water_temperature import WaterTemperature
-from django.db.models import F, Value, Case, When, Count, Q
+from django.db.models import F, Value, Case, When, Count, Q, CharField
 from rest_framework.views import APIView
 from rest_framework.response import Response
 from sorl.thumbnail import get_thumbnail
@@ -64,12 +66,8 @@ def biodiversity_data(self):
  category=TaxonomicGroupCategory.SPECIES_MODULE.name
  ).order_by('display_order')
 
- collection_results = collection_results.select_related(
- 'taxonomy', 'taxonomy__endemism', 'taxonomy__iucn_status'
- )
  for group in groups:
- location_site_ids = set()
- taxonomy_ids = set()
+ time.sleep(0.5)
  group_data = {}
  try:
  group_data[self.GROUP_ICON] = get_thumbnail(
@@ -82,91 +80,98 @@ def biodiversity_data(self):
  biodiversity_data[group.name] = group_data
 
  group_records = collection_results.filter(module_group=group)
+ group_records = group_records.annotate(
+ endemism_name=Case(
+ When(taxonomy__endemism__isnull=False, then=F('taxonomy__endemism__name')),
+ default=Value('Unknown'),
+ output_field=CharField()
+ ),
+ origin_name=Case(
+ When(taxonomy__origin='', then=Value('Unknown')),
+ default=F('taxonomy__origin'),
+ output_field=CharField()
+ ),
+ iucn_category=Case(
+ When(
+ taxonomy__iucn_status__isnull=False,
+ then=F('taxonomy__iucn_status__category')
+ ),
+ default=Value('Not evaluated'),
+ output_field=CharField()
+ )
+ )
  group_records_count = group_records.count()
 
- if group_records.count() > 0 and not self.is_sass_exist:
+ if group_records_count > 0 and not self.is_sass_exist:
  try:
  if isinstance(
  collection_results.first(),
  SiteVisitTaxon):
  self.is_sass_exist = group_records.filter(
  site_visit__isnull=False
- ).count() > 0
+ ).exists()
  else:
  self.is_sass_exist = group_records.filter(
  sitevisittaxon__isnull=False
- ).count() > 0
+ ).exists()
  except: # noqa
  self.is_sass_exist = False
 
  group_data[self.GROUP_OCCURRENCES] = group_records_count
- location_site_ids.update(group_records.values_list('site', flat=True))
- taxonomy_ids.update(group_records.values_list('taxonomy', flat=True))
-
- group_data[self.GROUP_SITES] = LocationSite.objects.filter(
- id__in=location_site_ids
- ).count()
 
- group_data[self.GROUP_NUM_OF_TAXA] = Taxonomy.objects.filter(
- id__in=taxonomy_ids
- ).count()
+ aggregate_result = group_records.aggregate(
+ unique_site_count=Count('site_id', distinct=True),
+ unique_taxonomy_count=Count('taxonomy_id', distinct=True)
+ )
 
- endemism_counts = group_records.annotate(
- name=Case(
- When(
- taxonomy__endemism__isnull=False,
- then=F('taxonomy__endemism__name')),
- default=Value('Unknown')
- )
- ).values('name').annotate(count=Count('name')).order_by('name')
+ group_data[self.GROUP_SITES] = aggregate_result['unique_site_count']
+ group_data[self.GROUP_NUM_OF_TAXA] = aggregate_result['unique_taxonomy_count']
 
+ # Endemism counts
+ endemism_counts = group_records.values('endemism_name').annotate(
+ count=Count('endemism_name')
+ ).order_by('endemism_name')
  group_data[self.GROUP_ENDEMISM] = list(endemism_counts)
 
- group_origins = group_records.annotate(
- name=Case(
- When(taxonomy__origin='', then=Value('Unknown')),
- default=F('taxonomy__origin')
- )
- ).values('name').annotate(count=Count('name')).order_by('name')
-
+ # Origin counts
+ group_origins = group_records.values('origin_name').annotate(
+ count=Count('origin_name')
+ ).order_by('origin_name')
  if group_origins:
  category = dict(Taxonomy.CATEGORY_CHOICES)
  for group_origin in group_origins:
- if group_origin['name'] in category:
- group_origin['name'] = category[group_origin['name']]
+ if group_origin['origin_name'] in category:
+ group_origin['name'] = category[group_origin['origin_name']]
  group_data[self.GROUP_ORIGIN] = list(group_origins)
 
+ # Conservation status counts
  all_cons_status = group_records.filter(
  taxonomy__iucn_status__national=False
- ).annotate(
- name=Case(
- When(
- taxonomy__iucn_status__isnull=False,
- then=F('taxonomy__iucn_status__category')),
- default=Value('Not evaluated')
- )
- ).values('name').annotate(count=Count('name')).order_by('name')
-
+ ).values('iucn_category').annotate(
+ count=Count('iucn_category')
+ ).order_by('iucn_category')
  if all_cons_status:
  category = dict(IUCNStatus.CATEGORY_CHOICES)
  for cons_status in all_cons_status:
- if cons_status['name'] in category:
- cons_status['name'] = category[cons_status['name']]
+ if cons_status['iucn_category'] in category:
+ cons_status['name'] = category[cons_status['iucn_category']]
  group_data[self.GROUP_CONS_STATUS] = list(all_cons_status)
 
  return biodiversity_data
 
-class MultiLocationSitesOverview(APIView, LocationSiteOverviewData):
+class MultiLocationSitesOverview(SuperuserRequiredMixin, APIView, LocationSiteOverviewData):
 
  def get(self, request):
  """
  Get overview data for multiple sites
  """
+ start_time = time.time()
  self.search_filters = request.GET
 
  response_data = dict()
  response_data[self.BIODIVERSITY_DATA] = self.biodiversity_data()
  response_data[self.SASS_EXIST] = self.is_sass_exist
+ response_data['duration'] = time.time() - start_time
  return Response(response_data)
 
 

bims/api_views/search.py

@@ -421,23 +421,17 @@ def thermal_module(self):
  else:
  return False
 
- def filter_taxa_records(self, query_dict, select_related=None):
+ def filter_taxa_records(self, query_dict):
  """
  Filter taxa records
  :param query_dict: dict of query
  """
  if self.filtered_taxa_records is None:
  taxa = Taxonomy.objects
- if select_related:
- taxa = taxa.select_related(select_related)
  self.filtered_taxa_records = taxa.filter(
  **query_dict
  )
  else:
- if select_related:
- self.filtered_taxa_records = self.filtered_taxa_records.select_related(
- select_related
- )
  self.filtered_taxa_records = self.filtered_taxa_records.filter(
  **query_dict
  )
@@ -478,7 +472,7 @@ def process_search(self):
  'biologicalcollectionrecord__isnull': False
  })
  elif self.search_query:
- bio = collection_records_by_site.select_related('taxonomy').filter(
+ bio = collection_records_by_site.filter(
  Q(taxonomy__canonical_name__icontains=self.search_query) |
  Q(taxonomy__accepted_taxonomy__canonical_name__icontains=
  self.search_query) |
@@ -507,7 +501,7 @@ def process_search(self):
  )
  if not bio.exists():
  # Search by vernacular names
- bio = collection_records_by_site.select_related('taxonomy').filter(
+ bio = collection_records_by_site.filter(
  taxonomy__vernacular_names__name__icontains=
  self.search_query
  )
@@ -530,11 +524,12 @@ def process_search(self):
 
  if self.site_ids:
  filters['site__in'] = self.site_ids
+
  invasions = self.invasions()
  if invasions:
  self.filter_taxa_records({
  'invasion__id__in': invasions
- }, 'invasion')
+ })
  if self.categories:
  self.filter_taxa_records(
  {
@@ -596,8 +591,7 @@ def process_search(self):
  self.filter_taxa_records(
  {
  'endemism__in': endemism_list
- },
- 'endemism'
+ }
  )
  if self.taxon_id:
  self.filter_taxa_records({
@@ -633,7 +627,7 @@ def process_search(self):
  filters['taxonomy__isnull'] = False
  bio_filtered = True
 
- bio = bio.select_related('taxonomy').filter(**filters)
+ bio = bio.filter(**filters)
 
  requester_id = self.parameters.get('requester', None)
 
@@ -649,8 +643,6 @@ def process_search(self):
  except get_user_model().DoesNotExist:
  pass
 
- bio = bio.select_related('owner')
-
  bio = bio.filter(
  Q(owner_id=requester_id) |
  Q(
@@ -749,15 +741,11 @@ def process_search(self):
  spatial_filters = self.spatial_filter
  if spatial_filters:
  if not isinstance(filtered_location_sites, QuerySet):
- filtered_location_sites = LocationSite.objects.select_related(
- 'locationcontextgroup'
- ).filter(
+ filtered_location_sites = LocationSite.objects.filter(
  spatial_filters
  )
  else:
- filtered_location_sites = filtered_location_sites.select_related(
- 'locationcontextgroup'
- ).filter(
+ filtered_location_sites = filtered_location_sites.filter(
  spatial_filters
  )
 
@@ -772,22 +760,9 @@ def process_search(self):
  bio_filtered = True
 
  if self.modules:
- # For Intersection methods :
- if len(self.modules) > 1:
- module_filter = []
- for taxon_group in self.modules:
- module_filter.append(
- bio.filter(
- module_group__id=taxon_group
- ).values_list('site_id'))
- intersections = module_filter[0].intersection(
- *module_filter[1:]
- )
- bio = bio.filter(site_id__in=intersections)
- else:
- bio = bio.filter(
- module_group__id__in=self.modules
- )
+ bio = bio.filter(
+ module_group__id__in=self.modules
+ )
  bio_filtered = True
 
  if self.get_request_data('polygon'):
@@ -820,7 +795,7 @@ def process_search(self):
  ):
  bio = bio.filter(
  site__in=filtered_location_sites
- ).select_related()
+ )
  bio_filtered = True
 
  water_temperature = []
@@ -843,7 +818,7 @@ def process_search(self):
  if bio_filtered:
  bio = bio.filter(
  site__in=filtered_location_sites
- ).select_related()
+ )
 
  if bio.exists() or water_temperature:
  location_sites_filter = LocationSite.objects.filter(

bims/static/js/map-dashboard.js

@@ -19,7 +19,7 @@ function createDashboardMap(map, coordinates) {
  )
  }
  map = new ol.Map({
- controls: ol.control.defaults.defaults().extend([
+ controls: ol.control.defaults().extend([
  scaleLineControl
  ]),
  layers: baseLayer,