PY3 extension upgrade

.gitignore

@@ -53,4 +53,10 @@ e2e/cypress/videos/*
 e2e/cypress/screenshots/*
 e2e/cypress/results/output.xml
 # Commit testing resource(s) csv
-!e2e/cypress/fixtures/*.csv
+!e2e/cypress/fixtures/*.csv
+
+# Future
+*.bak
+
+# Vim
+*.swp

Dockerfile

@@ -5,14 +5,18 @@ FROM openknowledge/ckan-dev:2.9
 ENV GIT_BRANCH=2.9
 ENV CKAN_HOME /srv/app
 ENV CKAN_CONFIG /app/config
+ENV APP_DIR /app
 # ENV CKAN_ENV docker
 
 # add dependencies for cryptography and vim
 # RUN apk add libressl-dev musl-dev libffi-dev xmlsec vim xmlsec-dev
 
-COPY requirements.txt ${APP_DIR}
+COPY requirements.txt requirements-dev.txt ${APP_DIR}/
+ADD setup.py README.md ${APP_DIR}/
+ADD ckanext ${APP_DIR}/ckanext/
 
-RUN pip3 install --ignore-installed -r requirements.txt
+RUN pip3 install --ignore-installed -r ${APP_DIR}/requirements.txt
+RUN pip3 install --ignore-installed -r ${APP_DIR}/requirements-dev.txt
 # COPY docker-entrypoint.d/* /docker-entrypoint.d/
 
 # What saml2 info do we need?

Makefile

@@ -10,7 +10,8 @@ clean:
 
 cypress:
 	# Turn on local system, and open cypress in interactive mode
-	docker-compose up -d && cd e2e && CYPRESS_BASE_URL=http://localhost:5000 npx cypress open
+	docker-compose up -d && cd e2e && CYPRESS_USER_PASSWORD=cypress-user-password \
+	CYPRESS_USER=cypress-user CYPRESS_BASE_URL=http://localhost:5000 npx cypress open
 
 debug:
 	# Stop the canonical app container to avoid a port collision. Use `run`
@@ -33,7 +34,7 @@ test:
 	docker-compose -f docker-compose.yml -f docker-compose.test.yml up --abort-on-container-exit test
 
 test_extension:
-	docker-compose run --rm app nosetests --ckan --with-pylons=docker_test.ini ckanext/datagov_inventory/tests/*
+	docker-compose run --rm app pytest --ckan-ini=/app/config/development.ini --cov=ckanext.datagov_inventory --disable-warnings /app/ckanext/datagov_inventory/tests/
 
 up:
 	docker-compose up

cfsetup.sh

@@ -4,7 +4,7 @@ set -o errexit
 set -o pipefail
 
 # Add the current directory to our virtualenv
-# pip3 install .
+pip3 install -e .
 
 function vcap_get_service () {
   local path name
@@ -79,4 +79,4 @@ DATASTORE_URL=$CKAN_DATASTORE_WRITE_URL DS_RO_USER=$DS_RO_USER DS_RO_PASSWORD=$D
 # paster --plugin=ckan db upgrade -c config/production.ini
 ckan -c $CKAN_INI db upgrade
 
-exec $@
+exec $@

ckanext/datagov_inventory/plugin.py

@@ -26,6 +26,11 @@ def _wrapper(next_auth, context, dict_data=None):
     return _wrapper
 
 
+@toolkit.auth_allow_anonymous_access
+def datagov_allow_anonymous_access(context, data_dict):
+    return {'success': True}
+
+
 class Datagov_IauthfunctionsPlugin(plugins.SingletonPlugin):
     plugins.implements(plugins.IAuthFunctions)
     plugins.implements(plugins.IConfigurer)
@@ -39,8 +44,8 @@ def get_auth_functions(self):
                 'organization_list': datagov_disallow_anonymous_access(),
                 'package_list': datagov_disallow_anonymous_access(),
                 'package_search': datagov_disallow_anonymous_access(),
-                'revision_list': datagov_disallow_anonymous_access(),
-                'revision_show': datagov_disallow_anonymous_access(),
+                'package_show': datagov_disallow_anonymous_access(),
+                'resource_show': datagov_allow_anonymous_access,
                 'site_read': datagov_disallow_anonymous_access(),
                 'tag_list': datagov_disallow_anonymous_access(),
                 'tag_show': datagov_disallow_anonymous_access(),

ckanext/datagov_inventory/tests/logic/auth/test_auth.py

@@ -1,8 +1,9 @@
 """Tests for datagov_inventory plugin.py."""
 
-from nose.tools import assert_raises
+from builtins import object
+from pytest import raises as assert_raises
+import pytest
 
-from ckan.lib import search
 import ckan.logic as logic
 import ckan.model as model
 
@@ -19,22 +20,13 @@
 is_denied = False
 
 
+@pytest.mark.usefixtures(u"clean_index")
+@pytest.mark.usefixtures(u"clean_db")
 class TestDatagovInventoryAuth(object):
 
-    @classmethod
-    def setup_class(self):
-        '''Nose runs this method once to setup our test class.'''
-
     def setup(self):
-        '''Nose runs this method before each test method in our test class.'''
-
         # Start with a clean database and index for each test
-        search.clear_all()
         self.clean_datastore()
-        model.repo.rebuild_db()
-
-    def teardown(self):
-        '''Nose runs this method after each test method in our test class.'''
 
     def setup_test_orgs_users(self):
 
@@ -97,8 +89,8 @@ def factory_dataset(self, **kwargs):
         # Return id string for the package and resoruce just created
         return({'package_id': dataset['id'],
                 'tag_id': dataset_params['tag_string'],
-                'resource_id': dataset['resources'][0]['id'],
-                'revision_id': dataset['revision_id']})
+                'resource_id': dataset['resources'][0]['id']})
+        # 'revision_id': dataset['revision_id']})
 
     def assert_user_authorization(self,
                                   auth_function,
@@ -127,11 +119,10 @@ def assert_user_authorization(self,
                 assert actual_authorization == expected_user_access_dict[user]
             else:
                 # We expect users to be denied
-                assert_raises(logic.NotAuthorized,
-                              helpers.call_auth,
-                              auth_function,
-                              context=context,
-                              id=object_id)
+                with assert_raises(logic.NotAuthorized):
+                    helpers.call_auth(auth_function,
+                                      context=context,
+                                      id=object_id)
 
     def test_auth_format_autocomplete(self):
         # Create test users and test group
@@ -285,33 +276,6 @@ def test_auth_resource_show_for_public_gsa_dataset(self):
             'anonymous': is_allowed
         }, object_id=dataset['resource_id'])
 
-    def test_auth_revision_list(self):
-        # Create test users and test data
-        self.setup_test_orgs_users()
-
-        self.assert_user_authorization('revision_list', {
-            'gsa_admin': is_allowed,
-            'gsa_editor': is_allowed,
-            'gsa_member': is_allowed,
-            'doi_admin': is_allowed,
-            'doi_member': is_allowed,
-            'anonymous': is_denied
-        })
-
-    def test_auth_revision_show(self):
-        # Create test users and test data
-        self.setup_test_orgs_users()
-        dataset = self.factory_dataset(owner_org='gsa', private=True)
-
-        self.assert_user_authorization('revision_show', {
-            'gsa_admin': is_allowed,
-            'gsa_editor': is_allowed,
-            'gsa_member': is_allowed,
-            'doi_admin': is_allowed,
-            'doi_member': is_allowed,
-            'anonymous': is_denied
-        }, dataset['revision_id'])
-
     def test_auth_site_read(self):
         # Create test users and test data
         self.setup_test_orgs_users()

config/development.ini

@@ -46,6 +46,11 @@ who.log_file = %(cache_dir)s/who_log.ini
 # Inactive by default, so the session doesn't expire.
 # who.timeout = 86400
 
+# If ckan redirects to https for whatever reason, the following two config params
+# allow ckan to run on https on-demand
+# ckan.devserver.ssl_cert = adhoc
+# ckan.devserver.ssl_key = adhoc
+
 ## Database Settings
 sqlalchemy.url = postgresql://ckan:pass@db/ckan
 
@@ -130,8 +135,8 @@ solr_url = http://solr:8983/solr/inventory
 # Note: Add ``datastore`` to enable the CKAN DataStore
 #               Add ``resource_proxy`` to enable resource proxying and get around the
 #               same origin policy
-# TODO: re-integrate datagov_inventory, dcat_usmetadata, usmetadata, datajson, saml2auth
-ckan.plugins = datastore xloader stats text_view recline_view googleanalyticsbasic s3filestore envvars
+# TODO: re-integrate dcat_usmetadata, usmetadata, datajson, saml2auth
+ckan.plugins = datagov_inventory datastore xloader stats text_view recline_view googleanalyticsbasic s3filestore envvars
 
 # Define which views should be created by default
 # (plugins must be loaded in ckan.plugins)

config/production.ini

@@ -129,8 +129,8 @@ ckan.auth.public_user_details = false
 #		Add ``resource_proxy`` to enable resorce proxying and get around the
 #		same origin policy
 
-# TODO: re-integrate datagov_inventory, dcat_usmetadata, usmetadata, datajson, saml2auth
-ckan.plugins = datastore xloader stats text_view recline_view googleanalyticsbasic s3filestore envvars
+# TODO: re-integrate dcat_usmetadata, usmetadata, datajson, saml2auth
+ckan.plugins = datagov_inventory datastore xloader stats text_view recline_view googleanalyticsbasic s3filestore envvars
 
 ckan.views.default_views = recline_view text_view image_view webpage_view recline_grid_view
 

docker-compose.test.yml

@@ -9,7 +9,7 @@ services:
   test:
     image: cypress/included:7.3.0
     entrypoint: []
-    command: /bin/bash -c "npx wait-on http://app:5000 && cypress run"
+    command: /bin/bash -c "npx wait-on http://app:5000/api/action/status_show && cypress run"
     # To run specific test, use the following syntax:
     # command: /bin/bash -c "npx wait-on http://app:5000 && cypress run --spec cypress/integration/organizations.spec.js"
     # To run against a dev or production image, use the following syntax:

docker-compose.yml

@@ -1,6 +1,7 @@
 version: '3'
 services:
   app:
+    image: datagov/inventory-app:2.9
     build: .
     command: /app/start.sh
     depends_on:
@@ -39,6 +40,8 @@ services:
 
   solr:
     image: datagov/catalog-solr:inventory2_8
+    ports:
+      - "8983:8983"
 
   redis:
     image: redis:alpine

e2e/cypress/integration/ckan_extensions.spec.js

@@ -19,12 +19,12 @@ describe('CKAN Extensions', () => {
             expect(installed_extensions).to.include('s3filestore');
             expect(installed_extensions).to.include('envvars');
             expect(installed_extensions).to.include('datastore');
+            expect(installed_extensions).to.include('datagov_inventory');
             // TODO: The following extensions should be verified to be installed:
-            // expect(installed_extensions).to.include('datagov_inventory');
             // expect(installed_extensions).to.include('dcat_usmetadata');
             // expect(installed_extensions).to.include('usmetadata');
             // expect(installed_extensions).to.include('datajson');
             // expect(installed_extensions).to.include('saml2auth');
         });
     })
-})
+})

e2e/cypress/integration/dataset.spec.js

@@ -1,14 +1,15 @@
 describe('Dataset', () => {
 
     before(() => {
-        cy.login()
-        cy.delete_organization('test-organization')
-        cy.create_organization('test-organization', 'Test organization')
-    })
+        cy.logout();
+        cy.login();
+        cy.delete_organization('test-organization');
+        cy.create_organization('test-organization', 'Test organization');
+    });
 
     beforeEach(() => {
-        Cypress.Cookies.preserveOnce('auth_tkt', 'ckan')
-    })
+        Cypress.Cookies.preserveOnce('auth_tkt', 'ckan');
+    });
 
     after(() => {
         cy.delete_dataset('test-dataset-1')
@@ -18,7 +19,7 @@ describe('Dataset', () => {
     it('Creates dataset via API', () => {
         cy.fixture('ckan_dataset.json').then((ckan_dataset) => {
             cy.create_dataset(ckan_dataset).should((response) => {
-                expect(response.body).to.have.property('success', true)
+                expect(response.body).to.have.property('success', true);
             });
         });
     });
@@ -28,7 +29,7 @@ describe('Dataset', () => {
         cy.contains('Test Dataset 1');
         // TODO: re-add the following check to validate usmetadata template is working
         // cy.contains('Common Core Metadata');
-    })
+    });
 
     it('Add resource to private dataset via API', () => {
         cy.fixture('ckan_resource.csv', 'binary').then((ckan_resource) => {
@@ -40,11 +41,12 @@ describe('Dataset', () => {
             formData.set('name', "test-resource-1");
             formData.set('resource_type', "CSV");
             formData.set('format', "CSV");
-            cy.form_request('POST', 'http://app:5000/api/action/resource_create', formData, function (response) {
+            cy.form_request('POST', '/api/action/resource_create', formData, function (response) {
                 expect(response.status).to.eq(200);
             });
         });
-    })
+        cy.visit('/dataset');
+    });
 
     it('Download resource file', () => {
         cy.visit('/dataset/test-dataset-1')

e2e/cypress/integration/login.spec.js

@@ -1,15 +1,11 @@
 describe('Login', () => {
 
     it('Invalid user login attempt', () => {
-        cy.visit('/dataset')
-        cy.get('a[href="/user/login"]').click()
         cy.login('not-user', 'not-password', true)
-        cy.get('.flash-messages .alert').should('contain', 'Login failed. Bad username or password.')
+        cy.contains('Not authorized to see this page')
     });
 
     it('Valid login attempt', () => {
-        cy.visit('/dataset')
-        cy.get('a[href="/user/login"]').click()
         cy.login()
         cy.get('.nav-tabs>li>a').should('contain', 'My Organizations')
     })

e2e/cypress/integration/main_page.spec.js

@@ -1,14 +1,19 @@
 describe('Main Page', () => {
 
+    beforeEach(() => {
+        cy.logout();
+        cy.login();
+    });
+
     it('Load main page with configuration', () => {
-        cy.visit('/dataset')
-        cy.contains('Inventory')
+        cy.visit('/dataset');
+        cy.contains('Inventory');
     });
 
     it('google tracker injected', () => {
         cy.request('/dataset').then((response) => {
-            expect(response.body).to.have.string('google-analytics-fake-key-testing-87654321')
-        })
+            expect(response.body).to.have.string('google-analytics-fake-key-testing-87654321');
+        });
         // cy.contains()
     });
-})
+});