diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..649e0d0 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,27 @@ +name: Tests +on: [push, pull_request] +jobs: + test: + strategy: + matrix: + python-version: [2.7, 3.6, 3.7, 3.8, 3.9] + fail-fast: false + name: Python ${{ matrix.python-version }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install requirements (Python 2) + if: ${{ matrix.python-version == '2.7' }} + run: pip install -r requirements-dev-py2.txt && pip install . + - name: Install requirements (Python 3) + if: ${{ matrix.python-version != '2.7' }} + run: pip install -r requirements-dev.txt && pip install . + - name: Run tests + run: pytest --cov=datapusher --cov-append --cov-report=xml --disable-warnings tests + - name: Upload coverage report to codecov + uses: codecov/codecov-action@v1 + with: + file: ./coverage.xml diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..e22d34f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,46 @@ +FROM debian:buster + +# Install required system packages +RUN apt-get -q -y update \ + && DEBIAN_FRONTEND=noninteractive apt-get -q -y upgrade \ + && apt-get -q -y install \ + python3-dev \ + python3-pip \ + python3-virtualenv \ + zlib1g-dev \ + libxml2-dev \ + libxslt1-dev \ + libffi-dev \ + # else error https://stackoverflow.com/questions/14547631/python-locale-error-unsupported-locale-setting + locales \ + postgresql-client \ + build-essential \ + git \ + vim \ + wget \ + && apt-get -q clean \ + && rm -rf /var/lib/apt/lists/* + + +RUN python3 -m virtualenv --python=python3 /venv +ENV PATH="/venv/bin:$PATH" + +# else error https://stackoverflow.com/questions/59633558/python-based-dockerfile-throws-locale-error-unsupported-locale-setting +ENV LC_ALL=C + +# NO else https://github.com/ckan/datapusher/issues/132 +#datapusher | File "/venv/src/datapusher/jobs.py", line 158, in check_response +#datapusher | request_url=request_url, response=response.text) +#datapusher | datapusher.jobs.HTTPError: +#ENV DATAPUSHER_SSL_VERIFY=true + +# Setup Datapusher +ADD . /venv/src/ +RUN pip install -U pip && \ + cd /venv/src/ && \ + pip install --upgrade --no-cache-dir -r requirements.txt && \ + pip install --upgrade --no-cache-dir -r requirements-dev.txt && \ + #pip install -e . + python setup.py develop + +CMD [ "python", "/venv/src/datapusher/main.py", "/venv/src/deployment/datapusher_settings.py"] diff --git a/README.md b/README.md index 4fbea96..c55cdc6 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,4 @@ -[![Build Status](https://travis-ci.org/ckan/datapusher.png?branch=master)](https://travis-ci.org/ckan/datapusher) -[![Coverage Status](https://coveralls.io/repos/ckan/datapusher/badge.png?branch=master)](https://coveralls.io/r/ckan/datapusher?branch=master) +[![Tests](https://github.com/ckan/datapusher/actions/workflows/test.yml/badge.svg)](https://github.com/ckan/datapusher/actions/workflows/test.yml) [![Latest Version](https://img.shields.io/pypi/v/datapusher.svg)](https://pypi.python.org/pypi/datapusher/) [![Downloads](https://img.shields.io/pypi/dm/datapusher.svg)](https://pypi.python.org/pypi/datapusher/) [![Supported Python versions](https://img.shields.io/pypi/pyversions/datapusher.svg)](https://pypi.python.org/pypi/datapusher/) @@ -67,7 +66,7 @@ If you need to change the host or port, copy `deployment/datapusher_settings.py` To run the tests: - nosetests + pytest ## Production deployment @@ -85,24 +84,24 @@ probably need to set up Nginx as a reverse proxy in front of it and something li Supervisor to keep the process up. - # Install requirements for the DataPusher - sudo apt install python3-venv python3-dev build-essential - sudo apt-get install python-dev python-virtualenv build-essential libxslt1-dev libxml2-dev git libffi-dev + # Install requirements for the DataPusher + sudo apt install python3-venv python3-dev build-essential + sudo apt-get install python-dev python-virtualenv build-essential libxslt1-dev libxml2-dev git libffi-dev - # Create a virtualenv for datapusher + # Create a virtualenv for datapusher sudo python3 -m venv /usr/lib/ckan/datapusher - # Create a source directory and switch to it - sudo mkdir /usr/lib/ckan/datapusher/src - cd /usr/lib/ckan/datapusher/src + # Create a source directory and switch to it + sudo mkdir /usr/lib/ckan/datapusher/src + cd /usr/lib/ckan/datapusher/src - # Clone the source (you should target the latest tagged version) - sudo git clone -b 0.0.17 https://github.com/ckan/datapusher.git + # Clone the source (you should target the latest tagged version) + sudo git clone -b 0.0.17 https://github.com/ckan/datapusher.git - # Install the DataPusher and its requirements - cd datapusher - sudo /usr/lib/ckan/datapusher/bin/pip install -r requirements.txt - sudo /usr/lib/ckan/datapusher/bin/python setup.py develop + # Install the DataPusher and its requirements + cd datapusher + sudo /usr/lib/ckan/datapusher/bin/pip install -r requirements.txt + sudo /usr/lib/ckan/datapusher/bin/python setup.py develop # Create a user to run the web service (if necessary) sudo addgroup www-data @@ -132,8 +131,8 @@ The default DataPusher configuration uses SQLite as the backend for the jobs dat sudo -u postgres createuser -S -D -R -P datapusher_jobs sudo -u postgres createdb -O datapusher_jobs datapusher_jobs -E utf-8 - # Run this in the virtualenv where DataPusher is installed - pip install psycopg2 + # Run this in the virtualenv where DataPusher is installed + pip install psycopg2 # Edit SQLALCHEMY_DATABASE_URI in datapusher_settings.py accordingly # eg SQLALCHEMY_DATABASE_URI=postgresql://datapusher_jobs:YOURPASSWORD@localhost/datapusher_jobs @@ -143,9 +142,9 @@ The default DataPusher configuration uses SQLite as the backend for the jobs dat ``` # ... rest of datapusher-uwsgi.ini - workers = 3 - threads = 3 - lazy-apps = true + workers = 3 + threads = 3 + lazy-apps = true ``` ## Configuring @@ -184,12 +183,14 @@ Here's a summary of the options available. | SSL_VERIFY | False | Do not validate SSL certificates when requesting the data file (*Warning*: Do not use this setting in production) | | TYPES | [messytables.StringType, messytables.DecimalType, messytables.IntegerType, messytables.DateUtilType] | [Messytables][] types used internally, can be modified to customize the type guessing | | TYPE_MAPPING | {'String': 'text', 'Integer': 'numeric', 'Decimal': 'numeric', 'DateUtil': 'timestamp'} | Internal Messytables type mapping | +| LOG_FILE | `/tmp/ckan_service.log` | Where to write the logs. Use an empty string to disable | +| STDERR | `True` | Log to stderr? | -Most of the configuration options above can be also provided as environment variables prepending the name with `DATAPUSHER_`, eg `DATAPUSHER_SQLALCHEMY_DATABASE_URI`, `DATAPUSHER_PORT`, etc. +Most of the configuration options above can be also provided as environment variables prepending the name with `DATAPUSHER_`, eg `DATAPUSHER_SQLALCHEMY_DATABASE_URI`, `DATAPUSHER_PORT`, etc. In the specific case of `DATAPUSHER_STDERR` the possible values are `1` and `0`. -By default DataPusher uses SQLite as the database backend for the jobs information. This is fine for local development and sites with low activity, but for sites that need more performance should use Postgres as the backend for the jobs database (eg `SQLALCHEMY_DATABASE_URI=postgresql://datapusher_jobs:YOURPASSWORD@localhost/datapusher_jobs`. See also [High Availability Setup](#high-availability-setup). If SQLite is used, is probably a good idea to store the database in a location other than `/tmp`. This will prevent the database being dropped, causing out of sync errors in the CKAN side. A good place to store it is the CKAN storage folder (if DataPusher is installed in the same server), generally in `/var/lib/ckan/`. +By default, DataPusher uses SQLite as the database backend for jobs information. This is fine for local development and sites with low activity, but for sites that need more performance, Postgres should be used as the backend for the jobs database (eg `SQLALCHEMY_DATABASE_URI=postgresql://datapusher_jobs:YOURPASSWORD@localhost/datapusher_jobs`. See also [High Availability Setup](#high-availability-setup). If SQLite is used, its probably a good idea to store the database in a location other than `/tmp`. This will prevent the database being dropped, causing out of sync errors in the CKAN side. A good place to store it is the CKAN storage folder (if DataPusher is installed in the same server), generally in `/var/lib/ckan/`. ## Usage diff --git a/datapusher/__init__.py b/datapusher/__init__.py index b47451b..d9fc5d6 100644 --- a/datapusher/__init__.py +++ b/datapusher/__init__.py @@ -1 +1 @@ -__version__ = '0.0.17' +__version__ = '0.0.18' diff --git a/deployment/datapusher-uwsgi.ini b/deployment/datapusher-uwsgi.ini index 44c6b7f..11b5754 100644 --- a/deployment/datapusher-uwsgi.ini +++ b/deployment/datapusher-uwsgi.ini @@ -12,3 +12,8 @@ max-requests = 5000 vacuum = true callable = application buffer-size = 32768 + +## see High Availability Setup +#workers = 3 +#threads = 3 +#lazy-apps = true diff --git a/deployment/datapusher_settings.py b/deployment/datapusher_settings.py index 0134a24..d4b198b 100644 --- a/deployment/datapusher_settings.py +++ b/deployment/datapusher_settings.py @@ -29,4 +29,5 @@ SSL_VERIFY = os.environ.get('DATAPUSHER_SSL_VERIFY', True) # logging -#LOG_FILE = '/tmp/ckan_service.log' +LOG_FILE = os.environ.get('DATAPUSHER_LOG_FILE', '/tmp/ckan_service.log') +STDERR = bool(int(os.environ.get('DATAPUSHER_STDERR', '1'))) diff --git a/requirements-dev-py2.txt b/requirements-dev-py2.txt new file mode 100644 index 0000000..4cc72ac --- /dev/null +++ b/requirements-dev-py2.txt @@ -0,0 +1,4 @@ +-r requirements.txt +httpretty==0.9.4 +pytest +pytest-cov diff --git a/requirements-dev.txt b/requirements-dev.txt index 512bb78..8197ee5 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,4 @@ -r requirements.txt -httpretty==0.9.4 -nose +httpretty==1.1.4 +pytest +pytest-cov diff --git a/requirements.txt b/requirements.txt index 483fa21..63a4518 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ argparse -ckanserviceprovider==0.0.10 +ckanserviceprovider==1.0.0 html5lib==1.0.1 messytables==0.15.2 certifi -requests[security]==2.24.0 +requests[security]==2.27.1 diff --git a/setup.py b/setup.py index df3eaca..cb5fbdf 100644 --- a/setup.py +++ b/setup.py @@ -46,7 +46,7 @@ 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', - + 'Programming Language :: Python :: 3.9', ], # What does your project relate to? diff --git a/tests/test_acceptance.py b/tests/test_acceptance.py index dc191de..117d0ee 100644 --- a/tests/test_acceptance.py +++ b/tests/test_acceptance.py @@ -12,12 +12,10 @@ """ import os import json -import unittest import datetime -from nose.tools import assert_equal, raises import httpretty -import requests +import pytest import datapusher.main as main import datapusher.jobs as jobs @@ -37,7 +35,7 @@ def get_static_file(filename): return open(join_static_path(filename), 'rb').read() -class TestImport(unittest.TestCase): +class TestImport(): @classmethod def setup_class(cls): cls.host = 'www.ckan.org' @@ -93,18 +91,15 @@ def register_urls(self, filename='simple.csv', format='CSV', body=json.dumps({'success': True}), content_type='application/json') - # A URL that mocks checking if a datastore table exists datastore_check_url = 'http://www.ckan.org/api/3/action/datastore_search' httpretty.register_uri(httpretty.POST, datastore_check_url, body=json.dumps({'success': True}), content_type='application/json') - return source_url, res_url @httpretty.activate - @raises(util.JobError) def test_too_large_content_length(self): """It should raise JobError if the returned Content-Length header is too large. @@ -136,10 +131,10 @@ def test_too_large_content_length(self): content_length=size, content_type='application/json') - jobs.push_to_datastore('fake_id', data, True) + with pytest.raises(util.JobError): + jobs.push_to_datastore('fake_id', data, True) @httpretty.activate - @raises(util.JobError) def test_too_large_file(self): """It should raise JobError if the data file is too large. @@ -172,7 +167,8 @@ def test_too_large_file(self): 'content-length': None }) - jobs.push_to_datastore('fake_id', data, True) + with pytest.raises(util.JobError): + jobs.push_to_datastore('fake_id', data, True) @httpretty.activate def test_content_length_string(self): @@ -246,12 +242,12 @@ def test_simple_csv(self): headers, results = jobs.push_to_datastore('fake_id', data, True) results = list(results) - assert_equal(headers, [{'type': 'timestamp', 'id': 'date'}, + assert (headers == [{'type': 'timestamp', 'id': 'date'}, {'type': 'numeric', 'id': 'temperature'}, {'type': 'text', 'id': 'place'}]) - assert_equal(len(results), 6) - assert_equal( - results[0], + assert len(results) == 6 + assert ( + results[0] == {'date': datetime.datetime(2011, 1, 1, 0, 0), 'place': 'Galway', 'temperature': 1}) @@ -277,11 +273,11 @@ def test_simple_tsv(self): headers, results = jobs.push_to_datastore('fake_id', data, True) results = list(results) - assert_equal(headers, [{'type': 'timestamp', 'id': 'date'}, + assert (headers == [{'type': 'timestamp', 'id': 'date'}, {'type': 'numeric', 'id': 'temperature'}, {'type': 'text', 'id': 'place'}]) - assert_equal(len(results), 6) - assert_equal(results[0], + assert len(results) == 6 + assert (results[0] == {'date': datetime.datetime(2011, 1, 1, 0, 0), 'place': 'Galway', 'temperature': 1}) @@ -307,11 +303,11 @@ def test_simple_ssv(self): headers, results = jobs.push_to_datastore('fake_id', data, True) results = list(results) - assert_equal(headers, [{'type': 'timestamp', 'id': 'date'}, - {'type': 'numeric', 'id': 'temperature'}, + assert (headers == [{'type': 'timestamp', 'id': 'date'}, + {'type': 'numeric', 'id': 'temperature'}, {'type': 'text', 'id': 'place'}]) - assert_equal(len(results), 6) - assert_equal(results[0], + assert len(results) == 6 + assert (results[0] == {'date': datetime.datetime(2011, 1, 1, 0, 0), 'place': 'Galway', 'temperature': 1}) @@ -336,11 +332,11 @@ def test_simple_xls(self): headers, results = jobs.push_to_datastore('fake_id', data, True) results = list(results) - assert_equal(headers, [{'type': 'timestamp', 'id': 'date'}, + assert (headers == [{'type': 'timestamp', 'id': 'date'}, {'type': 'numeric', 'id': 'temperature'}, {'type': 'text', 'id': 'place'}]) - assert_equal(len(results), 6) - assert_equal(results[0], + assert len(results) == 6 + assert (results[0] == {'date': datetime.datetime(2011, 1, 1, 0, 0), 'place': 'Galway', 'temperature': 1}) @@ -365,7 +361,7 @@ def test_real_csv(self): headers, results = jobs.push_to_datastore('fake_id', data, True) results = list(results) - assert_equal(headers, [{'type': 'text', 'id': 'Directorate'}, + assert (headers == [{'type': 'text', 'id': 'Directorate'}, {'type': 'text', 'id': 'Service Area'}, {'type': 'text', 'id': 'Expenditure Category'}, {'type': 'timestamp', 'id': 'Payment Date'}, @@ -376,8 +372,8 @@ def test_real_csv(self): {'type': 'text', 'id': 'Cost Centre Description'}, {'type': 'numeric', 'id': 'Grand Total'}]) - assert_equal(len(results), 230) - assert_equal(results[0], + assert len(results) == 230 + assert (results[0] == {'Directorate': 'Adult and Culture', 'Service Area': 'Ad Serv-Welfare Rights- ', 'Expenditure Category': 'Supplies & Services', @@ -411,12 +407,11 @@ def test_weird_header(self): headers, results = jobs.push_to_datastore('fake_id', data, True) results = list(results) - assert_equal(len(headers), 9) - assert_equal(len(results), 82) - assert_equal(headers[0]['id'].strip(), '1985') - assert_equal(results[1]['1993'].strip(), '379') + assert len(headers) == 9 + assert len(results) == 82 + assert headers[0]['id'].strip() == '1985' + assert results[1]['1993'].strip() == '379' - @raises(util.JobError) @httpretty.activate def test_bad_url(self): """It should raise HTTPError(JobError) if the resource.url is badly @@ -436,9 +431,9 @@ def test_bad_url(self): } } - jobs.push_to_datastore('fake_id', data, True) + with pytest.raises(util.JobError): + jobs.push_to_datastore('fake_id', data, True) - @raises(util.JobError) @httpretty.activate def test_bad_scheme(self): """It should raise HTTPError(JobError) if the resource.url is an @@ -458,7 +453,8 @@ def test_bad_scheme(self): } } - jobs.push_to_datastore('fake_id', data, True) + with pytest.raises(util.JobError): + jobs.push_to_datastore('fake_id', data, True) @httpretty.activate def test_mostly_numbers(self): @@ -481,8 +477,8 @@ def test_mostly_numbers(self): headers, results = jobs.push_to_datastore('fake_id', data, True) results = list(results) - assert_equal(len(headers), 19) - assert_equal(len(results), 133) + assert len(headers) == 19 + assert len(results) == 133 @httpretty.activate def test_long_file(self): @@ -505,8 +501,8 @@ def test_long_file(self): headers, results = jobs.push_to_datastore('fake_id', data, True) results = list(results) - assert_equal(len(headers), 1) - assert_equal(len(results), 4000) + assert len(headers) == 1 + assert len(results) == 4000 @httpretty.activate def test_do_not_push_when_same_hash(self): diff --git a/tests/test_mocked.py b/tests/test_mocked.py index 3bf5a8e..2f6c300 100644 --- a/tests/test_mocked.py +++ b/tests/test_mocked.py @@ -5,8 +5,8 @@ import os import json -import unittest +import pytest import httpretty import datapusher.main as main @@ -27,7 +27,7 @@ def get_static_file(filename): return open(join_static_path(filename)).read() -class TestImport(unittest.TestCase): +class TestImport(): @classmethod def setup_class(cls): cls.host = 'www.ckan.org' @@ -110,4 +110,5 @@ def test_wrong_api_key(self): } } - self.assertRaises(util.JobError, jobs.push_to_datastore, 'fake_id', data) + with pytest.raises(util.JobError): + jobs.push_to_datastore('fake_id', data) diff --git a/tests/test_unit.py b/tests/test_unit.py index 9c1105d..334565c 100644 --- a/tests/test_unit.py +++ b/tests/test_unit.py @@ -4,11 +4,8 @@ ''' import json -import unittest import requests - -from nose.tools import assert_equal, raises - +import pytest import httpretty import datapusher.jobs as jobs @@ -18,8 +15,8 @@ class TestChunky(): def test_simple(self): chunks = jobs.chunky('abcdefg', 3) - assert_equal( - list(chunks), + assert ( + list(chunks) == [ (['a', 'b', 'c'], False), (['d', 'e', 'f'], False), @@ -28,8 +25,8 @@ def test_simple(self): def test_length_is_the_exact_multiple(self): chunks = jobs.chunky('abcdef', 3) - assert_equal( - list(chunks), + assert ( + list(chunks) == [ (['a', 'b', 'c'], False), (['d', 'e', 'f'], True), @@ -37,33 +34,33 @@ def test_length_is_the_exact_multiple(self): def test_empty(self): chunks = jobs.chunky('', 3) - assert_equal( - list(chunks), []) + assert ( + list(chunks) == []) class TestGetUrl(): def test_get_action_url(self): - assert_equal( - jobs.get_url('datastore_create', 'http://www.ckan.org'), + assert ( + jobs.get_url('datastore_create', 'http://www.ckan.org') == 'http://www.ckan.org/api/3/action/datastore_create') def test_get_action_url_with_stuff(self): - assert_equal( - jobs.get_url('datastore_create', 'http://www.ckan.org/'), + assert ( + jobs.get_url('datastore_create', 'http://www.ckan.org/') == 'http://www.ckan.org/api/3/action/datastore_create') def test_get_action_url_with_https(self): - assert_equal( - jobs.get_url('datastore_create', 'https://www.ckan.org/'), + assert ( + jobs.get_url('datastore_create', 'https://www.ckan.org/') == 'https://www.ckan.org/api/3/action/datastore_create') def test_get_action_url_missing_http(self): - assert_equal( - jobs.get_url('datastore_create', 'www.ckan.org/'), + assert ( + jobs.get_url('datastore_create', 'www.ckan.org/') == 'http://www.ckan.org/api/3/action/datastore_create') -class TestValidation(unittest.TestCase): +class TestValidation(): def test_validate_input(self): jobs.validate_input({ 'metadata': { @@ -73,42 +70,46 @@ def test_validate_input(self): 'api_key': 'köi' }) - @raises(util.JobError) def test_validate_input_raises_if_metadata_missing(self): - jobs.validate_input({ - 'foo': {}, - 'api_key': 'my-key' - }) - @raises(util.JobError) + with pytest.raises(util.JobError): + jobs.validate_input({ + 'foo': {}, + 'api_key': 'my-key' + }) + def test_validate_input_raises_if_res_id_missing(self): - jobs.validate_input({ - 'metadata': { - 'ckan_url': 'http://www.ckan.org' - }, - 'api_key': 'my-key' - }) - @raises(util.JobError) + with pytest.raises(util.JobError): + jobs.validate_input({ + 'metadata': { + 'ckan_url': 'http://www.ckan.org' + }, + 'api_key': 'my-key' + }) + def test_validate_input_raises_if_ckan_url_missing(self): - jobs.validate_input({ - 'metadata': { - 'resource_id': 'h32jk4h34k5' - }, - 'api_key': 'my-key' - }) - @raises(util.JobError) + with pytest.raises(util.JobError): + jobs.validate_input({ + 'metadata': { + 'resource_id': 'h32jk4h34k5' + }, + 'api_key': 'my-key' + }) + def test_validate_api_key(self): - jobs.validate_input({ - 'metadata': { - 'resource_id': 'h32jk4h34k5', - 'ckan_url': 'http://www.ckan.org' - } - }) + with pytest.raises(util.JobError): + jobs.validate_input({ + 'metadata': { + 'resource_id': 'h32jk4h34k5', + 'ckan_url': 'http://www.ckan.org' + } + }) -class TestCkanActionCalls(unittest.TestCase): + +class TestCkanActionCalls(): @httpretty.activate def test_get_resource(self): url = 'http://www.ckan.org/api/3/action/resource_show' @@ -120,7 +121,7 @@ def test_get_resource(self): }}), content_type="application/json") resource = jobs.get_resource('an_id', 'http://www.ckan.org/', None) - assert_equal(resource, {'foo': 42}) + assert resource == {'foo': 42} assert json.loads(httpretty.last_request().body)['id'] == 'an_id' @httpretty.activate @@ -164,7 +165,7 @@ def test_send_resource_to_datastore(self): jobs.send_resource_to_datastore({'id': 'an_id'}, [], [], False, 'my_key', 'http://www.ckan.org/') -class TestCheckResponse(unittest.TestCase): +class TestCheckResponse(): """Unit tests for the check_response() function.""" @httpretty.activate @@ -210,14 +211,15 @@ def test_text_500_with_false_success(self): assert err.request_url == url @httpretty.activate - @raises(util.JobError) def test_text_404(self): - httpretty.register_uri(httpretty.GET, 'http://www.ckan.org/', - body='{"success": true}', - content_type='html/text', - status=404) - r = requests.get('http://www.ckan.org/') - jobs.check_response(r, 'http://www.ckan.org/', 'Me') + + with pytest.raises(util.JobError): + httpretty.register_uri(httpretty.GET, 'http://www.ckan.org/', + body='{"success": true}', + content_type='html/text', + status=404) + r = requests.get('http://www.ckan.org/') + jobs.check_response(r, 'http://www.ckan.org/', 'Me') @httpretty.activate def test_text_404_ignore(self): diff --git a/tests/test_web.py b/tests/test_web.py index 3eada0a..1940d5a 100644 --- a/tests/test_web.py +++ b/tests/test_web.py @@ -6,7 +6,6 @@ import os import json -from nose.tools import assert_equal import datapusher.main as main @@ -21,5 +20,5 @@ class TestWeb(): def test_status(self): rv = app.get('/status') result_dict = json.loads(rv.data) - assert_equal(result_dict['job_types'], ['push_to_datastore']) - assert_equal(result_dict['name'], 'datapusher') + assert result_dict['job_types'] == ['push_to_datastore'] + assert result_dict['name'] == 'datapusher'