Skip to content

Commit

Permalink
Merge pull request #397 from geoadmin/develop
Browse files Browse the repository at this point in the history
New Release v1.20.0 - #minor
  • Loading branch information
ltshb authored Jun 26, 2023
2 parents 4bbcd92 + ef8e4ab commit 0fe5783
Show file tree
Hide file tree
Showing 14 changed files with 5,168 additions and 4,207 deletions.
12 changes: 12 additions & 0 deletions app/stac_api/s3_multipart_upload.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@

from rest_framework import serializers

from stac_api.serializers import UploadNotInProgressError
from stac_api.utils import get_s3_cache_control_value
from stac_api.utils import get_s3_client
from stac_api.utils import isoformat
Expand Down Expand Up @@ -191,6 +192,17 @@ def complete_multipart_upload(self, key, asset, parts, upload_id):
)
except ClientError as error:
raise serializers.ValidationError(str(error)) from None
except KeyError as error:
# If we try to complete an upload that has already be completed, then a KeyError is
# generated. Although this should never happens because we check the state of the upload
# via the DB before sending the complete command to S3, it could happend if the previous
# complete was abruptly aborted (e.g. server crash), leaving the S3 completed without
# updating the DB.
logger.error(
"Failed to complete upload, probably because the upload was not in progress: %s",
error
)
raise UploadNotInProgressError() from None

if 'Location' not in response:
logger.error(
Expand Down
7 changes: 7 additions & 0 deletions app/stac_api/serializers.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
from django.utils.translation import gettext_lazy as _

from rest_framework import serializers
from rest_framework.exceptions import APIException
from rest_framework.utils.serializer_helpers import ReturnDict
from rest_framework.validators import UniqueValidator
from rest_framework_gis import serializers as gis_serializers
Expand Down Expand Up @@ -827,3 +828,9 @@ class Meta:
parts = serializers.ListField(
source='Parts', child=UploadPartSerializer(), default=list, read_only=True
)


class UploadNotInProgressError(APIException):
status_code = 409
default_detail = 'Upload not in progress'
default_code = 'conflict'
18 changes: 12 additions & 6 deletions app/stac_api/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
from datetime import datetime
from datetime import timezone
from decimal import Decimal
from decimal import InvalidOperation
from urllib import parse

import boto3
Expand Down Expand Up @@ -329,18 +330,23 @@ def geometry_from_bbox(bbox):
list_bbox_values = bbox.split(',')
if len(list_bbox_values) != 4:
raise ValueError('A bbox is based of four values')
if (
Decimal(list_bbox_values[0]) == Decimal(list_bbox_values[2]) and
Decimal(list_bbox_values[1]) == Decimal(list_bbox_values[3])
):
bbox_geometry = Point(float(list_bbox_values[0]), float(list_bbox_values[1]))
try:
list_bbox_values = list(map(Decimal, list_bbox_values))
except InvalidOperation as exc:
raise ValueError(f'Cannot convert list {list_bbox_values} to bbox') from exc

if (list_bbox_values[0] == list_bbox_values[2] and list_bbox_values[1] == list_bbox_values[3]):
bbox_geometry = Point(list_bbox_values[:2])
else:
bbox_geometry = Polygon.from_bbox(list_bbox_values)

# if large values, SRID is LV95. The default SRID is 4326
if Decimal(list_bbox_values[0]) > 360:
if list_bbox_values[0] > 360:
bbox_geometry.srid = 2056

if not bbox_geometry.valid:
raise ValueError(f'{bbox_geometry.valid_reason} for bbox with {bbox_geometry.wkt}')

return bbox_geometry


Expand Down
3 changes: 3 additions & 0 deletions app/stac_api/views.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
from stac_api.serializers import ConformancePageSerializer
from stac_api.serializers import ItemSerializer
from stac_api.serializers import LandingPageSerializer
from stac_api.serializers import UploadNotInProgressError
from stac_api.serializers_utils import get_relation_links
from stac_api.utils import get_asset_path
from stac_api.utils import harmonize_post_get_for_search
Expand Down Expand Up @@ -661,6 +662,8 @@ def complete_multipart_upload(self, executor, validated_data, asset_upload, asse
raise serializers.ValidationError({'parts': [_("Too many parts")]}, code='invalid')
if len(parts) < asset_upload.number_parts:
raise serializers.ValidationError({'parts': [_("Too few parts")]}, code='invalid')
if asset_upload.status != AssetUpload.Status.IN_PROGRESS:
raise UploadNotInProgressError()
executor.complete_multipart_upload(key, asset, parts, asset_upload.upload_id)
asset_upload.update_asset_from_upload()
asset_upload.status = AssetUpload.Status.COMPLETED
Expand Down
40 changes: 40 additions & 0 deletions app/tests/test_asset_upload_endpoint.py
Original file line number Diff line number Diff line change
Expand Up @@ -948,6 +948,46 @@ def test_asset_upload_1_parts_invalid_complete(self):
)
self.assertS3ObjectNotExists(key)

def test_asset_upload_1_parts_duplicate_complete(self):
key = get_asset_path(self.item, self.asset.name)
self.assertS3ObjectNotExists(key)
number_parts = 1
size = 1 * KB
file_like, checksum_multihash = get_file_like_object(size)
offset = size // number_parts
md5_parts = create_md5_parts(number_parts, offset, file_like)

response = self.client.post(
self.get_create_multipart_upload_path(),
data={
'number_parts': number_parts,
'checksum:multihash': checksum_multihash,
'md5_parts': md5_parts
},
content_type="application/json"
)
self.assertStatusCode(201, response)
json_data = response.json()
self.check_urls_response(json_data['urls'], number_parts)

parts = self.s3_upload_parts(json_data['upload_id'], file_like, size, number_parts)

response = self.client.post(
self.get_complete_multipart_upload_path(json_data['upload_id']),
data={'parts': parts},
content_type="application/json"
)
self.assertStatusCode(200, response)

response = self.client.post(
self.get_complete_multipart_upload_path(json_data['upload_id']),
data={'parts': parts},
content_type="application/json"
)
self.assertStatusCode(409, response)
self.assertEqual(response.json()['code'], 409)
self.assertEqual(response.json()['description'], {'detail': 'Upload not in progress'})


class AssetUploadDeleteInProgressEndpointTestCase(AssetUploadBaseTest):

Expand Down
71 changes: 0 additions & 71 deletions app/tests/test_items_endpoint.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,9 @@

from django.conf import settings
from django.contrib.auth import get_user_model
from django.contrib.gis.geos.geometry import GEOSGeometry
from django.test import Client
from django.urls import reverse

from stac_api.models import BBOX_CH
from stac_api.models import Item
from stac_api.utils import fromisoformat
from stac_api.utils import get_link
Expand Down Expand Up @@ -417,75 +415,6 @@ def test_items_endpoint_datetime_open_start_range_query(self):
self._navigate_to_previous_items(['item-yesterday-1', 'item-2', 'item-1'], json_response)


class ItemsBboxQueryEndpointTestCase(StacBaseTestCase):

@classmethod
def setUpTestData(cls):
cls.factory = Factory()
cls.collection = cls.factory.create_collection_sample().model

cls.items = cls.factory.create_item_samples(
[
'item-switzerland',
'item-switzerland-west',
'item-switzerland-east',
'item-switzerland-north',
'item-switzerland-south',
'item-paris',
],
cls.collection,
db_create=True,
)

def setUp(self):
self.client = Client()

def test_items_endpoint_bbox_valid_query(self):
# test bbox
ch_bbox = ','.join(map(str, GEOSGeometry(BBOX_CH).extent))
response = self.client.get(
f"/{STAC_BASE_V}/collections/{self.collection.name}/items"
f"?bbox={ch_bbox}&limit=100"
)
json_data = response.json()
self.assertStatusCode(200, response)
self.assertEqual(5, len(json_data['features']), msg="More than one item found")

def test_items_endpoint_bbox_invalid_query(self):
# test invalid bbox
response = self.client.get(
f"/{STAC_BASE_V}/collections/{self.collection.name}/items"
f"?bbox=5.96,45.82,10.49,47.81,screw;&limit=100"
)
self.assertStatusCode(400, response)

response = self.client.get(
f"/{STAC_BASE_V}/collections/{self.collection.name}/items"
f"?bbox=5.96,45.82,10.49,47.81,42,42&limit=100"
)
self.assertStatusCode(400, response)

def test_items_endpoint_bbox_from_pseudo_point(self):
response = self.client.get(
f"/{STAC_BASE_V}/collections/{self.collection.name}/items"
f"?bbox=5.96,45.82,5.97,45.83&limit=100"
)
json_data = response.json()
self.assertStatusCode(200, response)
nb_features_polygon = len(json_data['features'])

response = self.client.get(
f"/{STAC_BASE_V}/collections/{self.collection.name}/items"
f"?bbox=5.96,45.82,5.96,45.82&limit=100"
)
json_data = response.json()
self.assertStatusCode(200, response)
nb_features_point = len(json_data['features'])
self.assertEqual(3, nb_features_point, msg="More than one item found")
# do both queries return the same amount of items:
self.assertEqual(nb_features_polygon, nb_features_point)


class ItemsUnImplementedEndpointTestCase(StacBaseTestCase):

@classmethod
Expand Down
105 changes: 105 additions & 0 deletions app/tests/test_items_endpoint_bbox.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
import logging

from django.conf import settings
from django.contrib.gis.geos.geometry import GEOSGeometry
from django.test import Client

from stac_api.models import BBOX_CH

from tests.base_test import StacBaseTestCase
from tests.data_factory import Factory

logger = logging.getLogger(__name__)

STAC_BASE_V = settings.STAC_BASE_V


class ItemsBboxQueryEndpointTestCase(StacBaseTestCase):

@classmethod
def setUpTestData(cls):
cls.factory = Factory()
cls.collection = cls.factory.create_collection_sample().model

cls.items = cls.factory.create_item_samples(
[
'item-switzerland',
'item-switzerland-west',
'item-switzerland-east',
'item-switzerland-north',
'item-switzerland-south',
'item-paris',
],
cls.collection,
db_create=True,
)

def setUp(self):
self.client = Client()

def test_items_endpoint_bbox_valid_query(self):
# test bbox
ch_bbox = ','.join(map(str, GEOSGeometry(BBOX_CH).extent))
response = self.client.get(
f"/{STAC_BASE_V}/collections/{self.collection.name}/items"
f"?bbox={ch_bbox}&limit=100"
)
json_data = response.json()
self.assertStatusCode(200, response)
self.assertEqual(5, len(json_data['features']), msg="More than one item found")

def test_items_endpoint_bbox_invalid_query(self):
# cannot converted to bbox
response = self.client.get(
f"/{STAC_BASE_V}/collections/{self.collection.name}/items"
f"?bbox=a,b,c,d&limit=100"
)
self.assertStatusCode(400, response)

# wrong number of argument for bbox
response = self.client.get(
f"/{STAC_BASE_V}/collections/{self.collection.name}/items"
f"?bbox=0,0,2&limit=100"
)
self.assertStatusCode(400, response)

# the geometry is not in the correct order (should be minx, miny,
# maxx, maxy) but is still valid
response = self.client.get(
f"/{STAC_BASE_V}/collections/{self.collection.name}/items"
f"?bbox=1,1,0,0&limit=100"
)
self.assertStatusCode(200, response)

# test invalid bbox
response = self.client.get(
f"/{STAC_BASE_V}/collections/{self.collection.name}/items"
f"?bbox=5.96,45.82,10.49,47.81,screw;&limit=100"
)
self.assertStatusCode(400, response)

response = self.client.get(
f"/{STAC_BASE_V}/collections/{self.collection.name}/items"
f"?bbox=5.96,45.82,10.49,47.81,42,42&limit=100"
)
self.assertStatusCode(400, response)

def test_items_endpoint_bbox_from_pseudo_point(self):
response = self.client.get(
f"/{STAC_BASE_V}/collections/{self.collection.name}/items"
f"?bbox=5.96,45.82,5.97,45.83&limit=100"
)
json_data = response.json()
self.assertStatusCode(200, response)
nb_features_polygon = len(json_data['features'])

response = self.client.get(
f"/{STAC_BASE_V}/collections/{self.collection.name}/items"
f"?bbox=5.96,45.82,5.96,45.82&limit=100"
)
json_data = response.json()
self.assertStatusCode(200, response)
nb_features_point = len(json_data['features'])
self.assertEqual(3, nb_features_point, msg="More than one item found")
# do both queries return the same amount of items:
self.assertEqual(nb_features_polygon, nb_features_point)
Binary file added doc/assets/service-stac-upload-process.dia
Binary file not shown.
Loading

0 comments on commit 0fe5783

Please sign in to comment.