Skip to content

Commit

Permalink
transform: Compile Releases crashes if date missing
Browse files Browse the repository at this point in the history 
#279
  • Loading branch information
@jarofgreen
jarofgreen committed Apr 21, 2020
1 parent f5a94e5 commit bcaec51
Show file tree
Hide file tree
Showing 5 changed files with 1,425 additions and 7 deletions.
27 changes: 20 additions & 7 deletions ocdskingfisherprocess/transform/compile_releases.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,9 +117,16 @@ def _process_record(self, ocid, record, warnings=None):
if releases and not releases_linked:
# We have releases and none are linked (have URL's).
# We can compile them ourselves.
merger = ocdsmerge.Merger()
out = merger.create_compiled_release(releases)
self._store_result(ocid, out, warnings=warnings)
try:
merger = ocdsmerge.Merger()
out = merger.create_compiled_release(releases)
self._store_result(ocid, out, warnings=warnings)
except ocdsmerge.exceptions.OCDSMergeError as error:
self.database.add_collection_note(
self.destination_collection.database_id,
'OCID ' + ocid + ' could not be compiled because merge library threw an error: '
+ error.__class__.__name__ + ' ' + str(error)
)
return

compiled_release = record.get('compiledRelease')
Expand Down Expand Up @@ -182,10 +189,16 @@ def _process_releases(self, ocid):
self._store_result(ocid, releases_compiled[0], warnings=[warning])
else:
# There is no compiled release - we will do it ourselves.
merger = ocdsmerge.Merger()
out = merger.create_compiled_release(releases)

self._store_result(ocid, out)
try:
merger = ocdsmerge.Merger()
out = merger.create_compiled_release(releases)
self._store_result(ocid, out)
except ocdsmerge.exceptions.OCDSMergeError as error:
self.database.add_collection_note(
self.destination_collection.database_id,
'OCID ' + ocid + ' could not be compiled because merge library threw an error: '
+ error.__class__.__name__ + ' ' + str(error)
)

def _store_result(self, ocid, data, warnings=None):

Expand Down
320 changes: 320 additions & 0 deletions tests/fixtures/sample_1_0_releases_no_dates.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,320 @@
{
"uri": "http://standard.open-contracting.org/examples/1.0/releases/ocds-213czf-000-00001-01-planning.json",
"publishedDate": "2009-03-15T14:45:00Z",
"publisher": {
"scheme": "GB-COH",
"uid": "09506232",
"name": "Open Data Services Co-operative Limited",
"uri": "http://standard.open-contracting.org/examples/"
},
"license": "http://opendatacommons.org/licenses/pddl/1.0/",
"publicationPolicy": "https://github.com/open-contracting/sample-data/",
"releases": [
{
"language": "en",
"ocid": "ocds-213czf-000-00001",
"id": "ocds-213czf-000-00001-01-planning",
"tag": [
"planning"
],
"initiationType": "tender",
"buyer": {
"identifier": {
"scheme": "GB-LAC",
"id": "E09000003",
"legalName": "London Borough of Barnet",
"uri": "http://www.barnet.gov.uk/"
},
"name": "London Borough of Barnet",
"address": {
"streetAddress": "4, North London Business Park, Oakleigh Rd S",
"locality": "London",
"region": "London",
"postalCode": "N11 1NP",
"countryName": "United Kingdom"
},
"contactPoint": {
"name": "Procurement Team",
"email": "procurement-team@example.com",
"telephone": "01234 345 346",
"faxNumber": "01234 345 345",
"url": "http://example.com/contact/"
}
},
"planning": {
"budget": {
"source": "https://openspending.org/uk-barnet-budget/entries/6801ad388f3a38b7740dde20108c58b35984ee91",
"id": "6801ad388f3a38b7740dde20108c58b35984ee91",
"description": "Budget allocation for highway maintenance, aligned with 2015 strategic plan. ",
"amount": {
"amount": 6700000.0,
"currency": "GBP"
},
"project": "Central Junction Cycle Scheme",
"projectID": "SP001",
"uri": "https://openspending.org/uk-barnet-budget/entries/6801ad388f3a38b7740dde20108c58b35984ee91"
},
"rationale": "The 2009 Strategic Plan identifies a need for an improved cycle route in the centre of town.",
"documents": [
{
"id": "0001",
"documentType": "procurementPlan",
"title": "Area Wide Cycle Improvements - Procurement Plan",
"description": "The overall strategic framework for procurement to enhance cycle provision.",
"url": "http://example.com/opencontracting/documents/planning/highways/procurementPlan.pdf",
"datePublished": "2009-01-05T00:00:00Z",
"format": "application/pdf",
"language": "en"
},
{
"id": "0002",
"documentType": "needsAssessment",
"title": "Cycle provision - Needs Assessment",
"description": "Needs assessment for provision for cyclists in the centre of town.",
"url": "http://example.com/opencontracting/documents/ocds-213czf-000-00001/needsAssessment.pdf",
"datePublished": "2009-01-15T00:00:00Z",
"format": "application/pdf",
"language": "en"
}
]
},
"tender": {
"id": "ocds-213czf-000-00001-01-planning",
"title": "Planned cycle lane improvements",
"description": "The authority plans to tender for improvements to the cycle lane in early 2010. This notice provides advanced notice of the intention to tender, and details to upcoming consultation events.",
"status": "planned",
"items": [
{
"id": "0001",
"description": "string",
"classification": {
"scheme": "CPV",
"id": "45233130",
"description": "Construction work for highways",
"uri": "http://cpv.data.ac.uk/code-45233130"
},
"additionalClassifications": [
{
"scheme": "CPV",
"id": "45233162-2",
"description": "Cycle path construction work",
"uri": "http://cpv.data.ac.uk/code-45233162.html"
}
],
"quantity": 10,
"unit": {
"name": "Miles",
"value": {
"amount": 100000,
"currency": "GBP"
}
}
}
],
"minValue": {
"amount": 500000,
"currency": "GBP"
},
"value": {
"amount": 1000000,
"currency": "GBP"
},
"procurementMethod": "open",
"procurementMethodRationale": "An open competitive tender is required by EU Rules",
"awardCriteria": "bestProposal",
"awardCriteriaDetails": "The best proposal, subject to value for money requirements, will be accepted.",
"tenderPeriod": {
"startDate": "2010-02-01T00:00:00Z"
},
"awardPeriod": {
"startDate": "2010-06-01T00:00:00Z",
"endDate": "2011-06-01T23:59:59Z"
},
"procuringEntity": {
"identifier": {
"scheme": "GB-LAC",
"id": "E09000003",
"legalName": "London Borough of Barnet",
"uri": "http://www.barnet.gov.uk/"
},
"name": "London Borough of Barnet",
"address": {
"streetAddress": "4, North London Business Park, Oakleigh Rd S",
"locality": "London",
"region": "London",
"postalCode": "N11 1NP",
"countryName": "United Kingdom"
},
"contactPoint": {
"name": "Procurement Team",
"email": "procurement-team@example.com",
"telephone": "01234 345 346",
"faxNumber": "01234 345 345",
"url": "http://example.com/contact/"
}
},
"documents": [
{
"id": "0003",
"documentType": "x_consultationDocument",
"title": "Consultation on cycle provision",
"description": "A consultation document inviting citizen input into cycle provision.",
"url": "http://example.com/consultations/cycle-provision/",
"datePublished": "2015-02-15T00:00:00Z",
"dateModified": "2015-02-15T00:00:00Z",
"format": "text/html",
"language": "en"
},
{
"id": "0004",
"documentType": "x_map",
"title": "Map of affected areas",
"description": "A map showing areas affected by the planned highway updates. Available from local libraries.",
"datePublished": "2015-02-15T00:00:00Z",
"format": "offline/print",
"language": "en"
}
],
"milestones": [
{
"id": "0001",
"title": "Consultation Period",
"description": "A consultation period is open for citizen input to shape the final plans.",
"dueDate": "2015-04-15T17:00:00Z"
}
]
}
},
{
"ocid": "ocds-213czf-000-00001",
"id": "ocds-213czf-000-00001-02-tender",
"language": "en",
"initiationType": "tender",
"tag": [
"tender"
],
"parties": [
{
"id": "GB-LAC-E09000003",
"name": "London Borough of Barnet",
"roles": [
"buyer"
],
"identifier": {
"scheme": "GB-LAC",
"id": "E09000003",
"legalName": "London Borough of Barnet"
},
"address": {
"streetAddress": "4, North London Business Park, Oakleigh Rd S",
"locality": "London",
"region": "London",
"postalCode": "N11 1NP",
"countryName": "United Kingdom"
},
"contactPoint": {
"name": "Procurement Team",
"email": "procurement-team@example.com",
"telephone": "01234 345 346",
"faxNumber": "01234 345 345",
"url": "http://example.com/contact/"
}
}
],
"buyer": {
"id": "GB-LAC-E09000003",
"name": "London Borough of Barnet"
},
"tender": {
"id": "ocds-213czf-000-00001-01-tender",
"title": "Planned cycle lane improvements",
"description": "Tenders solicited for work to build new cycle lanes in the centre of town.",
"mainProcurementCategory": "works",
"status": "active",
"minValue": {
"amount": 600000,
"currency": "GBP"
},
"value": {
"amount": 1100000,
"currency": "GBP"
},
"procurementMethod": "open",
"procurementMethodDetails": "In open procedures, any interested economic operator may submit a tender in response to a contract notice. ",
"procurementMethodRationale": "An open competitive tender is required by EU Rules",
"awardCriteria": "bestProposal",
"awardCriteriaDetails": "The best proposal, subject to value for money requirements, will be accepted.",
"submissionMethod": [
"electronicSubmission"
],
"submissionMethodDetails": "Submit through the online portal at http://example.com/submissions/ocds-213czf-000-00001-01/",
"tenderPeriod": {
"startDate": "2010-03-01T09:00:00Z",
"endDate": "2010-04-01T18:00:00Z",
"durationInDays": 31
},
"enquiryPeriod": {
"startDate": "2010-03-01T09:00:00Z",
"endDate": "2010-03-14T17:30:00Z",
"durationInDays": 14
},
"awardPeriod": {
"startDate": "2010-04-02T09:00:00Z",
"endDate": "2010-05-01T18:00:00Z"
},
"contractPeriod": {
"startDate": "2010-07-01T00:00:00Z",
"endDate": "2011-06-30T23:59:00Z",
"durationInDays": 365
},
"procuringEntity": {
"id": "GB-LAC-E09000003",
"name": "London Borough of Barnet"
},
"hasEnquiries": false,
"documents": [
{
"id": "5.0",
"documentType": "tenderNotice",
"title": "Tender Notice",
"description": "Official tender notice.",
"url": "http://example.com/tender-notices/ocds-213czf-000-00001-01.html",
"datePublished": "2010-03-01T09:00:00Z",
"format": "text/html",
"language": "en"
}
],
"items": [
{
"id": "1.0",
"description": "Cycle lane improvements",
"classification": {
"scheme": "CPV",
"id": "45233130.0",
"description": "Construction work for highways",
"uri": "http://cpv.data.ac.uk/code-45233130"
},
"quantity": 8,
"unit": {
"name": "Miles",
"id": "SMI",
"scheme": "UNCEFACT",
"value": {
"amount": 120000,
"currency": "GBP"
}
},
"additionalClassifications": [
{
"scheme": "CPV",
"id": "45233162-2",
"description": "Cycle path construction work",
"uri": "http://cpv.data.ac.uk/code-45233162.html"
}
]
}
]
}
}
]
}
Loading

0 comments on commit bcaec51

Please sign in to comment.