Merge pull request #86 from RockefellerArchiveCenter/issue-82

Validates accession and bag data against local schema

bagdiscovery/library.py

@@ -1,12 +1,10 @@
 import json
+import jsonschema
 import os
 import requests
 import shutil
 import tarfile
 
-from bravado_core.spec import Spec
-from bravado_core.validate import validate_object
-
 from .models import Bag
 from ursa_major import settings
 
@@ -15,6 +13,12 @@ class BagDiscoveryException(Exception): pass
 class CleanupException(Exception): pass
 
 
+def validate_data(data):
+ with open(os.path.join(settings.BASE_DIR, settings.SCHEMA_PATH), "r") as jf:
+ schema = json.load(jf)
+ jsonschema.validate(instance=data, schema=schema)
+
+
 class BagDiscovery:
  """Discovers and stores bags, and delivers data to another service."""
  def __init__(self, url, dirs=None):
@@ -59,8 +63,11 @@ def save_bag_data(self, bag):
  try:
  with open(os.path.join(self.tmp_dir, bag.bag_identifier, "{}.json".format(bag.bag_identifier))) as json_file:
  bag_data = json.load(json_file)
+ validate_data(bag_data)
  bag.data = bag_data
  bag.save()
+ except jsonschema.exceptions.ValidationError as e:
+ raise BagDiscoveryException("Invalid bag data: {}: {}".format(list(e.path), e.message))
  except Exception as e:
  raise BagDiscoveryException("Error saving bag data: {}".format(e), bag.bag_identifier)
 
@@ -104,19 +111,3 @@ def run(self):
  return ("Transfer was not found.", self.identifier)
  except Exception as e:
  raise CleanupException(e, self.identifier)
-
-
-class DataValidator:
- def __init__(self, schema_url):
- bravado_config = {
- 'validate_swagger_spec': False,
- 'validate_requests': False,
- 'validate_responses': False,
- 'use_models': True,
- }
- spec_dict = requests.get(schema_url).json()
- self.spec = Spec.from_dict(spec_dict, config=bravado_config)
- self.Accession = spec_dict['definitions']['Accession']
-
- def validate(self, data):
- validate_object(self.spec, self.Accession, data)

bagdiscovery/tests.py

@@ -41,20 +41,19 @@ def setUp(self):
  makedirs(dir)
 
  def createobjects(self):
- with process_vcr.use_cassette('store_bags.json'):
- print('*** Creating objects ***')
- accession_count = 0
- transfer_count = 0
- for f in listdir(data_fixture_dir):
- with open(join(data_fixture_dir, f), 'r') as json_file:
- accession_data = json.load(json_file)
- request = self.factory.post(reverse('accession-list'), accession_data, format='json')
- response = AccessionViewSet.as_view(actions={"post": "create"})(request)
- self.assertEqual(response.status_code, 201, "Wrong HTTP code")
- accession_count += 1
- transfer_count += len(accession_data['transfers'])
- self.assertEqual(len(Accession.objects.all()), accession_count, "Wrong number of accessions created")
- self.assertEqual(len(Bag.objects.all()), transfer_count, "Wrong number of transfers created")
+ print('*** Creating objects ***')
+ accession_count = 0
+ transfer_count = 0
+ for f in listdir(data_fixture_dir):
+ with open(join(data_fixture_dir, f), 'r') as json_file:
+ accession_data = json.load(json_file)
+ request = self.factory.post(reverse('accession-list'), accession_data, format='json')
+ response = AccessionViewSet.as_view(actions={"post": "create"})(request)
+ self.assertEqual(response.status_code, 201, "Response error: {}".format(response.data))
+ accession_count += 1
+ transfer_count += len(accession_data['transfers'])
+ self.assertEqual(len(Accession.objects.all()), accession_count, "Wrong number of accessions created")
+ self.assertEqual(len(Bag.objects.all()), transfer_count, "Wrong number of transfers created")
 
  def process_bags(self):
  with process_vcr.use_cassette('process_bags.json'):
@@ -73,24 +72,24 @@ def run_view(self):
  print('*** Test run view ***')
  request = self.factory.post(reverse('bagdiscovery'), {"test": True})
  response = BagDiscoveryView.as_view()(request)
- self.assertEqual(response.status_code, 200, "Wrong HTTP code")
+ self.assertEqual(response.status_code, 200, "Response error: {}".format(response.data))
 
  def cleanup_view(self):
  print('*** Test cleanup view ***')
  for bag in Bag.objects.all():
  request = self.factory.post(reverse('cleanup'), {"test": True, "identifier": bag.bag_identifier})
  response = CleanupRoutineView.as_view()(request)
- self.assertEqual(response.status_code, 200, "Wrong HTTP code")
+ self.assertEqual(response.status_code, 200, "Response error: {}".format(response.data))
 
  def schema(self):
  print('*** Getting schema view ***')
  schema = self.client.get(reverse('schema'))
- self.assertEqual(schema.status_code, 200, "Wrong HTTP code")
+ self.assertEqual(schema.status_code, 200, "Response error: {}".format(schema))
 
  def health_check(self):
  print('*** Getting status view ***')
  status = self.client.get(reverse('api_health_ping'))
- self.assertEqual(status.status_code, 200, "Wrong HTTP code")
+ self.assertEqual(status.status_code, 200, "Response error: {}".format(status))
 
  def tearDown(self):
  for d in [self.src_dir, self.dest_dir]:

bagdiscovery/views.py

@@ -7,7 +7,7 @@
 from rest_framework.viewsets import ModelViewSet
 from rest_framework.response import Response
 
-from .library import BagDiscovery, CleanupRoutine, DataValidator
+from .library import BagDiscovery, CleanupRoutine, validate_data
 from .models import Accession, Bag
 from .serializers import AccessionSerializer, AccessionListSerializer, BagSerializer, BagListSerializer
 from ursa_major import settings
@@ -35,18 +35,9 @@ def get_serializer_class(self):
  return AccessionListSerializer
  return AccessionSerializer
 
- def format_field_path(self, absolute_path):
- path = []
- for part in absolute_path:
- if type(part) is str:
- path.append(part)
- elif type(part) is int:
- path[-1] = path[-1] + "[{}]".format(part)
- return '.'.join(path)
-
  def create(self, request):
  try:
- DataValidator(settings.SCHEMA_URL).validate(request.data)
+ validate_data(request.data)
  accession = Accession.objects.create(
  data=request.data
  )
@@ -59,7 +50,7 @@ def create(self, request):
  transfer_ids.append(t['identifier'])
  return Response(prepare_response(("Accession stored and transfer objects created", transfer_ids)), status=201)
  except ValidationError as e:
- return Response(prepare_response("Invalid accession data: {}: {}".format(self.format_field_path(e.absolute_path), e.message)), status=400)
+ return Response(prepare_response("Invalid accession data: {}: {}".format(list(e.path), e.message)), status=400)
  except IntegrityError as e:
  return Response(prepare_response(e), status=409)
  except Exception as e: