From 8ae8eff2f108ec036a20aefab28b2e0ef2069b41 Mon Sep 17 00:00:00 2001
From: James Baster <james.baster@opendataservices.coop>
Date: Wed, 17 Apr 2019 12:12:46 +0100
Subject: [PATCH 1/3] Add update_requirements.sh from CoVE

---
 update_requirements.sh | 36 ++++++++++++++++++++++++++++++++++++
 1 file changed, 36 insertions(+)
 create mode 100755 update_requirements.sh

diff --git a/update_requirements.sh b/update_requirements.sh
new file mode 100755
index 00000000..2ee99493
--- /dev/null
+++ b/update_requirements.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+# This follows broadly the approach from
+# http://www.kennethreitz.org/essays/a-better-pip-workflow but with the
+# addition of requirements_dev
+
+# Delete and recreate a virtualenv to ensure that we don't have any extra
+# packages installed in it
+rm -rf .ve
+virtualenv --python=python3 .ve
+source .ve/bin/activate
+
+if [[ "$1" == "--new-only" ]]; then
+    # If --new-only is supplied then we install the current versions of
+    # packages into the virtualenv, so that the only change will be any new
+    # packages and their dependencies.
+    pip install -r requirements.txt
+    dashupgrade=""
+else
+    dashupgrade="--upgrade"
+fi
+pip install $dashupgrade -r requirements.in
+pip freeze -r requirements.in > requirements.txt
+
+# Same again for requirements_dev
+if [[ "$1" == "--new-only" ]]; then
+    pip install -r requirements_dev.txt
+fi
+pip install $dashupgrade -r requirements_dev.in
+cat requirements.in requirements_dev.in > requirements_combined_tmp.in
+pip freeze -r requirements_combined_tmp.in > requirements_dev.txt
+rm requirements_combined_tmp.in
+
+# Put comments back on the same line (mostly for requires.io's benefit)
+sed -i '$!N;s/\n#\^\^/ #/;P;D' requirements.txt requirements_dev.txt
+sed -i 's/^-r.*//' requirements.txt requirements_dev.txt
+sed -i 's/pkg-resources==0.0.0//' requirements.txt requirements_dev.txt

From 6787bc366dc8a4b2ac1f651dcba5520d5c8f48a9 Mon Sep 17 00:00:00 2001
From: James Baster <james.baster@opendataservices.coop>
Date: Wed, 17 Apr 2019 12:13:00 +0100
Subject: [PATCH 2/3] Update Lib-Cove and Lib-Cove-OCDS to the latest versions

Also Lock SQLAlchemy to 1.2 branch (1.3 branch has issues with an identifier being too long)
---
 requirements.in      |  7 ++---
 requirements.txt     | 35 ++++++++++++------------
 requirements_dev.txt | 64 +++++++++++++++++++++++++-------------------
 3 files changed, 58 insertions(+), 48 deletions(-)
 mode change 100644 => 100755 requirements.txt
 mode change 100644 => 100755 requirements_dev.txt

diff --git a/requirements.in b/requirements.in
index 81d30ad0..7205082c 100644
--- a/requirements.in
+++ b/requirements.in
@@ -1,7 +1,6 @@
 -e git+https://github.com/OpenDataServices/flatten-tool.git@v0.5.0#egg=flattentool
-# Normally lib-cove-ocds is v0.1.0 but we are hacking it to be a specially speeded up version.
-# See https://github.com/open-contracting/lib-cove-ocds/issues/9
--e git+https://github.com/open-contracting/lib-cove-ocds.git@9cf68780f9e30921e432b12e3e346a057e2bbba5#egg=libcoveocds
+-e git+https://github.com/OpenDataServices/lib-cove.git@v0.6.0#egg=libcove
+-e git+https://github.com/open-contracting/lib-cove-ocds.git@v0.4.0#egg=libcoveocds
 ocdskit
 Flask
 alembic
@@ -11,3 +10,5 @@ ocdsmerge
 blinker
 redis
 sentry-sdk
+# Lock to 1.2 branch (1.3 branch has issues with an identifier being too long)
+SQLAlchemy==1.2.19
diff --git a/requirements.txt b/requirements.txt
old mode 100644
new mode 100755
index 563a9198..6596576e
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,48 +1,49 @@
 -e git+https://github.com/OpenDataServices/flatten-tool.git@4c13ef0b32a59e810919a3de09bc8f64ce8f9392#egg=flattentool
--e git+https://github.com/open-contracting/lib-cove-ocds.git@9cf68780f9e30921e432b12e3e346a057e2bbba5#egg=libcoveocds
+-e git+https://github.com/OpenDataServices/lib-cove.git@52acb2f0dcd296b9d952ff2adbae61dc4f8dd296#egg=libcove
+-e git+https://github.com/open-contracting/lib-cove-ocds.git@8a3d696eb614063bdec1790c0e3fda68d0445040#egg=libcoveocds
 ocdskit==0.0.5
 Flask==1.0.2
-alembic==1.0.7
+alembic==1.0.9
 pgpasslib==1.1.0
-psycopg2==2.7.7
+psycopg2==2.8.2
 ocdsmerge==0.5.1
 blinker==1.4
-redis==3.2.0
-sentry-sdk==0.7.4
+redis==3.2.1
+sentry-sdk==0.7.10
+# Lock to 1.2 branch (1.3 branch has issues with an identifier being too long)
+SQLAlchemy==1.2.19
 ## The following requirements were added by pip freeze:
-attrs==18.2.0
 bleach==3.1.0
 cached-property==1.5.1
-certifi==2018.11.29
+certifi==2019.3.9
 chardet==3.0.4
 Click==7.0
 commonmark==0.8.1
 contextlib2==0.5.5
-Django==2.1.7
+Django==2.2
 et-xmlfile==1.0.1
 future==0.17.1
 idna==2.8
 itsdangerous==1.1.0
 jdcal==1.4
-Jinja2==2.10
+Jinja2==2.10.1
 json-merge-patch==0.2
 jsonref==0.2
-jsonschema==3.0.1
-lxml==4.3.2
-Mako==1.0.7
+jsonschema==2.6.0
+lxml==4.3.3
+Mako==1.0.9
 MarkupSafe==1.1.1
-openpyxl==2.6.0
-pyrsistent==0.14.11
+openpyxl==2.6.2
 python-dateutil==2.8.0
 python-editor==1.0.4
-pytz==2018.9
+pytz==2019.1
 requests==2.21.0
 rfc3987==1.3.8
 schema==0.7.0
 six==1.12.0
-SQLAlchemy==1.2.18
+sqlparse==0.3.0
 strict-rfc3339==0.7
 urllib3==1.24.1
 webencodings==0.5.1
-Werkzeug==0.14.1
+Werkzeug==0.15.2
 xmltodict==0.12.0
diff --git a/requirements_dev.txt b/requirements_dev.txt
old mode 100644
new mode 100755
index 69007b63..aff5a47f
--- a/requirements_dev.txt
+++ b/requirements_dev.txt
@@ -1,68 +1,76 @@
+-e git+https://github.com/OpenDataServices/flatten-tool.git@4c13ef0b32a59e810919a3de09bc8f64ce8f9392#egg=flattentool
+-e git+https://github.com/OpenDataServices/lib-cove.git@52acb2f0dcd296b9d952ff2adbae61dc4f8dd296#egg=libcove
+-e git+https://github.com/open-contracting/lib-cove-ocds.git@8a3d696eb614063bdec1790c0e3fda68d0445040#egg=libcoveocds
+ocdskit==0.0.5
+Flask==1.0.2
+alembic==1.0.9
+pgpasslib==1.1.0
+psycopg2==2.8.2
+ocdsmerge==0.5.1
+blinker==1.4
+redis==3.2.1
+sentry-sdk==0.7.10
+# Lock to 1.2 branch (1.3 branch has issues with an identifier being too long)
+SQLAlchemy==1.2.19
+
 flake8==3.7.7
-pytest==4.3.0
-Sphinx==1.8.4
+pytest==4.4.1
+Sphinx==2.0.1
 ## The following requirements were added by pip freeze:
 alabaster==0.7.12
-alembic==1.0.7
 atomicwrites==1.3.0
-attrs==18.2.0
+attrs==19.1.0
 Babel==2.6.0
 bleach==3.1.0
-blinker==1.4
 cached-property==1.5.1
-certifi==2018.11.29
+certifi==2019.3.9
 chardet==3.0.4
 Click==7.0
 commonmark==0.8.1
 contextlib2==0.5.5
-Django==2.1.7
+Django==2.2
 docutils==0.14
 entrypoints==0.3
 et-xmlfile==1.0.1
-Flask==1.0.2
--e git+https://github.com/OpenDataServices/flatten-tool.git@4c13ef0b32a59e810919a3de09bc8f64ce8f9392#egg=flattentool
 future==0.17.1
 idna==2.8
 imagesize==1.1.0
 itsdangerous==1.1.0
 jdcal==1.4
-Jinja2==2.10
+Jinja2==2.10.1
 json-merge-patch==0.2
 jsonref==0.2
-jsonschema==3.0.1
--e git+https://github.com/open-contracting/lib-cove-ocds.git@9cf68780f9e30921e432b12e3e346a057e2bbba5#egg=libcoveocds
-lxml==4.3.2
-Mako==1.0.7
+jsonschema==2.6.0
+lxml==4.3.3
+Mako==1.0.9
 MarkupSafe==1.1.1
 mccabe==0.6.1
-more-itertools==6.0.0
-ocdskit==0.0.5
-ocdsmerge==0.5.1
-openpyxl==2.6.0
+more-itertools==7.0.0
+openpyxl==2.6.2
 packaging==19.0
-pgpasslib==1.1.0
 pluggy==0.9.0
-psycopg2==2.7.7
 py==1.8.0
 pycodestyle==2.5.0
 pyflakes==2.1.1
 Pygments==2.3.1
-pyparsing==2.3.1
-pyrsistent==0.14.11
+pyparsing==2.4.0
 python-dateutil==2.8.0
 python-editor==1.0.4
-pytz==2018.9
-redis==3.2.0
+pytz==2019.1
 requests==2.21.0
 rfc3987==1.3.8
 schema==0.7.0
-sentry-sdk==0.7.4
 six==1.12.0
 snowballstemmer==1.2.1
-sphinxcontrib-websupport==1.1.0
-SQLAlchemy==1.2.18
+sphinxcontrib-applehelp==1.0.1
+sphinxcontrib-devhelp==1.0.1
+sphinxcontrib-htmlhelp==1.0.2
+sphinxcontrib-jsmath==1.0.1
+sphinxcontrib-qthelp==1.0.2
+sphinxcontrib-serializinghtml==1.1.3
+sqlparse==0.3.0
 strict-rfc3339==0.7
 urllib3==1.24.1
 webencodings==0.5.1
-Werkzeug==0.14.1
+Werkzeug==0.15.2
 xmltodict==0.12.0

From 9715cd560bc01eea6e323bb71ac1e7482de01a0b Mon Sep 17 00:00:00 2001
From: James Baster <james.baster@opendataservices.coop>
Date: Wed, 17 Apr 2019 12:13:52 +0100
Subject: [PATCH 3/3] ocds_json_output from libcoveocds - cache_schema is
 deprecated, use config instead

---
 ocdskingfisherprocess/checks.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/ocdskingfisherprocess/checks.py b/ocdskingfisherprocess/checks.py
index feb895d7..cadfa68f 100644
--- a/ocdskingfisherprocess/checks.py
+++ b/ocdskingfisherprocess/checks.py
@@ -3,6 +3,7 @@
 import datetime
 import logging
 
+from libcoveocds.config import LibCoveOCDSConfig
 from libcoveocds.api import ocds_json_output, APIException
 
 
@@ -13,6 +14,8 @@ def __init__(self, database, collection, run_until_timestamp=None):
         self.collection = collection
         self.run_until_timestamp = run_until_timestamp
         self.logger = logging.getLogger('ocdskingfisher.checks')
+        self.libcoveocds_config = LibCoveOCDSConfig()
+        self.libcoveocds_config.config['cache_all_requests'] = True
 
     def process_all_files(self):
 
@@ -93,7 +96,11 @@ def process_all_files_records_with_override_schema_version_1_1(self):
     def handle_package(self, package):
         cove_temp_folder = tempfile.mkdtemp(prefix='ocdskingfisher-cove-', dir=tempfile.gettempdir())
         try:
-            return ocds_json_output(cove_temp_folder, None, None, convert=False, cache_schema=True, file_type='json', json_data=package)
+            return ocds_json_output(cove_temp_folder, None, None,
+                                    convert=False,
+                                    lib_cove_ocds_config=self.libcoveocds_config,
+                                    file_type='json',
+                                    json_data=package)
         finally:
             shutil.rmtree(cove_temp_folder)