Fix tech debt

ocdskingfisherprocess/checks.py

@@ -3,6 +3,7 @@
 import datetime
 import logging
 
+from libcoveocds.config import LibCoveOCDSConfig
 from libcoveocds.api import ocds_json_output, APIException
 
 
@@ -13,6 +14,8 @@ def __init__(self, database, collection, run_until_timestamp=None):
         self.collection = collection
         self.run_until_timestamp = run_until_timestamp
         self.logger = logging.getLogger('ocdskingfisher.checks')
+        self.libcoveocds_config = LibCoveOCDSConfig()
+        self.libcoveocds_config.config['cache_all_requests'] = True
 
     def process_all_files(self):
 
@@ -93,7 +96,11 @@ def process_all_files_records_with_override_schema_version_1_1(self):
     def handle_package(self, package):
         cove_temp_folder = tempfile.mkdtemp(prefix='ocdskingfisher-cove-', dir=tempfile.gettempdir())
         try:
-            return ocds_json_output(cove_temp_folder, None, None, convert=False, cache_schema=True, file_type='json', json_data=package)
+            return ocds_json_output(cove_temp_folder, None, None,
+                                    convert=False,
+                                    lib_cove_ocds_config=self.libcoveocds_config,
+                                    file_type='json',
+                                    json_data=package)
         finally:
             shutil.rmtree(cove_temp_folder)
 

requirements.in

@@ -1,7 +1,6 @@
 -e git+https://github.com/OpenDataServices/flatten-tool.git@v0.5.0#egg=flattentool
-# Normally lib-cove-ocds is v0.1.0 but we are hacking it to be a specially speeded up version.
-# See https://github.com/open-contracting/lib-cove-ocds/issues/9
--e git+https://github.com/open-contracting/lib-cove-ocds.git@9cf68780f9e30921e432b12e3e346a057e2bbba5#egg=libcoveocds
+-e git+https://github.com/OpenDataServices/lib-cove.git@v0.6.0#egg=libcove
+-e git+https://github.com/open-contracting/lib-cove-ocds.git@v0.4.0#egg=libcoveocds
 ocdskit
 Flask
 alembic
@@ -11,3 +10,5 @@ ocdsmerge
 blinker
 redis
 sentry-sdk
+# Lock to 1.2 branch (1.3 branch has issues with an identifier being too long)
+SQLAlchemy==1.2.19

requirements.txt

@@ -1,48 +1,49 @@
 -e git+https://github.com/OpenDataServices/flatten-tool.git@4c13ef0b32a59e810919a3de09bc8f64ce8f9392#egg=flattentool
--e git+https://github.com/open-contracting/lib-cove-ocds.git@9cf68780f9e30921e432b12e3e346a057e2bbba5#egg=libcoveocds
+-e git+https://github.com/OpenDataServices/lib-cove.git@52acb2f0dcd296b9d952ff2adbae61dc4f8dd296#egg=libcove
+-e git+https://github.com/open-contracting/lib-cove-ocds.git@8a3d696eb614063bdec1790c0e3fda68d0445040#egg=libcoveocds
 ocdskit==0.0.5
 Flask==1.0.2
-alembic==1.0.7
+alembic==1.0.9
 pgpasslib==1.1.0
-psycopg2==2.7.7
+psycopg2==2.8.2
 ocdsmerge==0.5.1
 blinker==1.4
-redis==3.2.0
-sentry-sdk==0.7.4
+redis==3.2.1
+sentry-sdk==0.7.10
+# Lock to 1.2 branch (1.3 branch has issues with an identifier being too long)
+SQLAlchemy==1.2.19
 ## The following requirements were added by pip freeze:
-attrs==18.2.0
 bleach==3.1.0
 cached-property==1.5.1
-certifi==2018.11.29
+certifi==2019.3.9
 chardet==3.0.4
 Click==7.0
 commonmark==0.8.1
 contextlib2==0.5.5
-Django==2.1.7
+Django==2.2
 et-xmlfile==1.0.1
 future==0.17.1
 idna==2.8
 itsdangerous==1.1.0
 jdcal==1.4
-Jinja2==2.10
+Jinja2==2.10.1
 json-merge-patch==0.2
 jsonref==0.2
-jsonschema==3.0.1
-lxml==4.3.2
-Mako==1.0.7
+jsonschema==2.6.0
+lxml==4.3.3
+Mako==1.0.9
 MarkupSafe==1.1.1
-openpyxl==2.6.0
-pyrsistent==0.14.11
+openpyxl==2.6.2
 python-dateutil==2.8.0
 python-editor==1.0.4
-pytz==2018.9
+pytz==2019.1
 requests==2.21.0
 rfc3987==1.3.8
 schema==0.7.0
 six==1.12.0
-SQLAlchemy==1.2.18
+sqlparse==0.3.0
 strict-rfc3339==0.7
 urllib3==1.24.1
 webencodings==0.5.1
-Werkzeug==0.14.1
+Werkzeug==0.15.2
 xmltodict==0.12.0

requirements_dev.txt

@@ -1,68 +1,76 @@
+-e git+https://github.com/OpenDataServices/flatten-tool.git@4c13ef0b32a59e810919a3de09bc8f64ce8f9392#egg=flattentool
+-e git+https://github.com/OpenDataServices/lib-cove.git@52acb2f0dcd296b9d952ff2adbae61dc4f8dd296#egg=libcove
+-e git+https://github.com/open-contracting/lib-cove-ocds.git@8a3d696eb614063bdec1790c0e3fda68d0445040#egg=libcoveocds
+ocdskit==0.0.5
+Flask==1.0.2
+alembic==1.0.9
+pgpasslib==1.1.0
+psycopg2==2.8.2
+ocdsmerge==0.5.1
+blinker==1.4
+redis==3.2.1
+sentry-sdk==0.7.10
+# Lock to 1.2 branch (1.3 branch has issues with an identifier being too long)
+SQLAlchemy==1.2.19
+
 flake8==3.7.7
-pytest==4.3.0
-Sphinx==1.8.4
+pytest==4.4.1
+Sphinx==2.0.1
 ## The following requirements were added by pip freeze:
 alabaster==0.7.12
-alembic==1.0.7
 atomicwrites==1.3.0
-attrs==18.2.0
+attrs==19.1.0
 Babel==2.6.0
 bleach==3.1.0
-blinker==1.4
 cached-property==1.5.1
-certifi==2018.11.29
+certifi==2019.3.9
 chardet==3.0.4
 Click==7.0
 commonmark==0.8.1
 contextlib2==0.5.5
-Django==2.1.7
+Django==2.2
 docutils==0.14
 entrypoints==0.3
 et-xmlfile==1.0.1
-Flask==1.0.2
--e git+https://github.com/OpenDataServices/flatten-tool.git@4c13ef0b32a59e810919a3de09bc8f64ce8f9392#egg=flattentool
 future==0.17.1
 idna==2.8
 imagesize==1.1.0
 itsdangerous==1.1.0
 jdcal==1.4
-Jinja2==2.10
+Jinja2==2.10.1
 json-merge-patch==0.2
 jsonref==0.2
-jsonschema==3.0.1
--e git+https://github.com/open-contracting/lib-cove-ocds.git@9cf68780f9e30921e432b12e3e346a057e2bbba5#egg=libcoveocds
-lxml==4.3.2
-Mako==1.0.7
+jsonschema==2.6.0
+lxml==4.3.3
+Mako==1.0.9
 MarkupSafe==1.1.1
 mccabe==0.6.1
-more-itertools==6.0.0
-ocdskit==0.0.5
-ocdsmerge==0.5.1
-openpyxl==2.6.0
+more-itertools==7.0.0
+openpyxl==2.6.2
 packaging==19.0
-pgpasslib==1.1.0
 pluggy==0.9.0
-psycopg2==2.7.7
 py==1.8.0
 pycodestyle==2.5.0
 pyflakes==2.1.1
 Pygments==2.3.1
-pyparsing==2.3.1
-pyrsistent==0.14.11
+pyparsing==2.4.0
 python-dateutil==2.8.0
 python-editor==1.0.4
-pytz==2018.9
-redis==3.2.0
+pytz==2019.1
 requests==2.21.0
 rfc3987==1.3.8
 schema==0.7.0
-sentry-sdk==0.7.4
 six==1.12.0
 snowballstemmer==1.2.1
-sphinxcontrib-websupport==1.1.0
-SQLAlchemy==1.2.18
+sphinxcontrib-applehelp==1.0.1
+sphinxcontrib-devhelp==1.0.1
+sphinxcontrib-htmlhelp==1.0.2
+sphinxcontrib-jsmath==1.0.1
+sphinxcontrib-qthelp==1.0.2
+sphinxcontrib-serializinghtml==1.1.3
+sqlparse==0.3.0
 strict-rfc3339==0.7
 urllib3==1.24.1
 webencodings==0.5.1
-Werkzeug==0.14.1
+Werkzeug==0.15.2
 xmltodict==0.12.0

update_requirements.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+# This follows broadly the approach from
+# http://www.kennethreitz.org/essays/a-better-pip-workflow but with the
+# addition of requirements_dev
+
+# Delete and recreate a virtualenv to ensure that we don't have any extra
+# packages installed in it
+rm -rf .ve
+virtualenv --python=python3 .ve
+source .ve/bin/activate
+
+if [[ "$1" == "--new-only" ]]; then
+    # If --new-only is supplied then we install the current versions of
+    # packages into the virtualenv, so that the only change will be any new
+    # packages and their dependencies.
+    pip install -r requirements.txt
+    dashupgrade=""
+else
+    dashupgrade="--upgrade"
+fi
+pip install $dashupgrade -r requirements.in
+pip freeze -r requirements.in > requirements.txt
+
+# Same again for requirements_dev
+if [[ "$1" == "--new-only" ]]; then
+    pip install -r requirements_dev.txt
+fi
+pip install $dashupgrade -r requirements_dev.in
+cat requirements.in requirements_dev.in > requirements_combined_tmp.in
+pip freeze -r requirements_combined_tmp.in > requirements_dev.txt
+rm requirements_combined_tmp.in
+
+# Put comments back on the same line (mostly for requires.io's benefit)
+sed -i '$!N;s/\n#\^\^/ #/;P;D' requirements.txt requirements_dev.txt
+sed -i 's/^-r.*//' requirements.txt requirements_dev.txt
+sed -i 's/pkg-resources==0.0.0//' requirements.txt requirements_dev.txt