[QOLDEV-985] convert scenario test to create dataset using standard f…

…unction

.docker/Dockerfile-template.ckan

@@ -12,9 +12,6 @@ ENV DOCKERIZE_VERSION v0.6.1
 RUN wget -O - https://github.com/jwilder/dockerize/releases/download/${DOCKERIZE_VERSION}/dockerize-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \
         | tar -C /usr/local/bin -xzvf -
 
-# Update urllib3 to fix urlopen bug
-RUN pip install -U urllib3
-
 COPY .docker/test.ini $CKAN_INI
 
 COPY . ${APP_DIR}/

.docker/test.ini

@@ -166,7 +166,7 @@ ckan.hide_activity_from_users = %(ckan.site_id)s
 smtp.test_server = localhost:8025
 smtp.mail_from = info@test.ckan.net
 
-# Logging configuration
+## Logging configuration
 [loggers]
 keys = root, ckan, ckanext
 

.flake8

@@ -15,4 +15,5 @@ max-line-length = 127
 
 # List ignore rules one per line.
 ignore =
+    E501
     W503

bin/init.sh

@@ -7,7 +7,6 @@ set -e
 . ${APP_DIR}/bin/activate
 CLICK_ARGS="--yes" ckan_cli db clean
 ckan_cli db init
-ckan_cli db upgrade
 
 # Create some base test data
 . $APP_DIR/bin/create-test-data.sh

bin/serve.sh

@@ -3,7 +3,7 @@ set -e
 
 . ${APP_DIR}/bin/activate
 if (which ckan > /dev/null); then
-    ckan -c ${CKAN_INI} run -r
+    ckan -c ${CKAN_INI} run --disable-reloader --threaded
 else
     paster serve ${CKAN_INI}
 fi

test/features/resource_type_validation.feature

@@ -3,14 +3,9 @@ Feature: Resource type validation
     Scenario: As an evil user, when I try to upload a resource with a MIME type not matching its extension, I should get an error
         Given "TestOrgEditor" as the persona
         When I log in
-        And I go to dataset "warandpeace"
-        And I press "Manage"
-        And I press "Resources"
-        And I press "Add new resource"
-        And I execute the script "button = document.getElementById('resource-upload-button'); if (button) button.click();"
-        And I attach the file "eicar.com.pdf" to "upload"
-        And I fill in "name" with "Testing EICAR PDF"
-        And I fill in "description" with "Testing EICAR sample virus file with PDF extension"
-        And I execute the script "document.getElementById('field-format').value='PDF'"
+        And I create a dataset with key-value parameters "notes=Testing mismatched MIME type"
+        And I open the new resource form for dataset "$last_generated_name"
+        And I fill in default resource fields
+        And I upload "upload=eicar.com.pdf" of type "PDF" to resource
         And I press the element with xpath "//form[contains(@class, 'resource-form')]//button[contains(@class, 'btn-primary')]"
         Then I should see "Mismatched file type"

test/features/steps/steps.py

@@ -5,6 +5,7 @@
 import email
 import quopri
 import six
+import uuid
 
 
 # Monkey-patch Selenium 3 to handle Python 3.9
@@ -85,6 +86,54 @@ def attempt_login(context, password):
     """.format(password))
 
 
+@when(u'I fill in "{name}" with "{value}" if present')
+def fill_in_field_if_present(context, name, value):
+    context.execute_steps(u"""
+        When I execute the script "field = $('#field-{0}'); if (!field.length) field = $('#{0}'); if (!field.length) field = $('[name={0}]'); field.val('{1}'); field.keyup();"
+    """.format(name, value))
+
+
+@when(u'I clear the URL field')
+def clear_url(context):
+    context.execute_steps(u"""
+        When I execute the script "$('a.btn-remove-url:contains(Clear)').click();"
+    """)
+
+
+@when(u'I open the new resource form for dataset "{name}"')
+def go_to_new_resource_form(context, name):
+    context.execute_steps(u"""
+        When I edit the "{0}" dataset
+    """.format(name))
+    if context.browser.is_element_present_by_xpath("//*[contains(@class, 'btn-primary') and contains(string(), 'Next:')]"):
+        # Draft dataset, proceed directly to resource form
+        context.execute_steps(u"""
+            When I press "Next:"
+        """)
+    else:
+        # Existing dataset, browse to the resource form
+        if context.browser.is_element_present_by_xpath(
+                "//a[contains(string(), 'Resources') and contains(@href, '/dataset/resources/')]"):
+            context.execute_steps(u"""
+                When I press "Resources"
+            """)
+        context.execute_steps(u"""
+            When I press "Add new resource"
+            And I take a debugging screenshot
+        """)
+
+
+@when(u'I fill in title with random text')
+def title_random_text(context):
+    assert context.persona
+    context.execute_steps(u"""
+        When I fill in "title" with "Test Title {0}"
+        And I fill in "name" with "test-title-{0}" if present
+        And I set "last_generated_title" to "Test Title {0}"
+        And I set "last_generated_name" to "test-title-{0}"
+    """.format(uuid.uuid4()))
+
+
 @when(u'I go to dataset page')
 def go_to_dataset_page(context):
     context.execute_steps(u"""
@@ -100,6 +149,172 @@ def go_to_dataset(context, name):
     """.format(name))
 
 
+@when(u'I edit the "{name}" dataset')
+def edit_dataset(context, name):
+    context.execute_steps(u"""
+        When I visit "/dataset/edit/{0}"
+    """.format(name))
+
+
+@when(u'I select the "{licence_id}" licence')
+def select_licence(context, licence_id):
+    # Licence requires special interaction due to fancy JavaScript
+    context.execute_steps(u"""
+        When I execute the script "$('#field-license_id').val('{0}').trigger('change')"
+    """.format(licence_id))
+
+
+@when(u'I enter the resource URL "{url}"')
+def enter_resource_url(context, url):
+    if url != "default":
+        context.execute_steps(u"""
+            When I clear the URL field
+            When I execute the script "$('#resource-edit [name=url]').val('{0}')"
+        """.format(url))
+
+
+@when(u'I fill in default dataset fields')
+def fill_in_default_dataset_fields(context):
+    context.execute_steps(u"""
+        When I fill in title with random text
+        And I fill in "notes" with "Description"
+        And I fill in "version" with "1.0"
+        And I fill in "author_email" with "test@me.com"
+        And I select the "other-open" licence
+        And I fill in "de_identified_data" with "NO" if present
+    """)
+
+
+@when(u'I fill in default resource fields')
+def fill_in_default_resource_fields(context):
+    context.execute_steps(u"""
+        When I fill in "name" with "Test Resource"
+        And I fill in "description" with "Test Resource Description"
+        And I fill in "size" with "1024" if present
+    """)
+
+
+@when(u'I fill in link resource fields')
+def fill_in_default_link_resource_fields(context):
+    context.execute_steps(u"""
+        When I enter the resource URL "https://example.com"
+        And I execute the script "document.getElementById('field-format').value='HTML'"
+        And I fill in "size" with "1024" if present
+    """)
+
+
+@when(u'I upload "{file_name}" of type "{file_format}" to resource')
+def upload_file_to_resource(context, file_name, file_format):
+    context.execute_steps(u"""
+        When I execute the script "$('#resource-upload-button').trigger('click');"
+        And I attach the file "{file_name}" to "upload"
+        # Don't quote the injected string since it can have trailing spaces
+        And I execute the script "document.getElementById('field-format').value='{file_format}'"
+        And I fill in "size" with "1024" if present
+    """.format(file_name=file_name, file_format=file_format))
+
+
+# Parse a "key=value::key2=value2" parameter string and return an iterator of (key, value) pairs.
+def _parse_params(param_string):
+    params = {}
+    for param in param_string.split("::"):
+        entry = param.split("=", 1)
+        params[entry[0]] = entry[1] if len(entry) > 1 else ""
+    return six.iteritems(params)
+
+
+def _create_dataset_from_params(context, params):
+    context.execute_steps(u"""
+        When I visit "/dataset/new"
+        And I fill in default dataset fields
+    """)
+    if 'private' not in params:
+        params = params + "::private=False"
+    for key, value in _parse_params(params):
+        if key == "name":
+            # 'name' doesn't need special input, but we want to remember it
+            context.execute_steps(u"""
+                When I set "last_generated_name" to "{0}"
+            """.format(value))
+
+        # Don't use elif here, we still want to type 'name' as usual
+        if key == "owner_org":
+            # Owner org uses UUIDs as its values, so we need to rely on displayed text
+            context.execute_steps(u"""
+                When I select by text "{1}" from "{0}"
+            """.format(key, value))
+        elif key in ["update_frequency", "request_privacy_assessment", "private"]:
+            context.execute_steps(u"""
+                When I select "{1}" from "{0}"
+            """.format(key, value))
+        elif key == "license_id":
+            context.execute_steps(u"""
+                When I select the "{0}" licence
+            """.format(value))
+        else:
+            context.execute_steps(u"""
+                When I fill in "{0}" with "{1}" if present
+            """.format(key, value))
+    context.execute_steps(u"""
+        When I take a debugging screenshot
+        And I press "Add Data"
+        Then I should see "Add New Resource"
+    """)
+
+
+@when(u'I create a dataset with key-value parameters "{params}"')
+def create_dataset_from_params(context, params):
+    _create_dataset_from_params(context, params)
+    context.execute_steps(u"""
+        When I go to dataset "$last_generated_name"
+    """)
+
+
+@when(u'I create a dataset and resource with key-value parameters "{params}" and "{resource_params}"')
+def create_dataset_and_resource_from_params(context, params, resource_params):
+    _create_dataset_from_params(context, params)
+    context.execute_steps(u"""
+        When I create a resource with key-value parameters "{0}"
+        Then I should see "Data and Resources"
+    """.format(resource_params))
+
+
+# Creates a resource using default values apart from the ones specified.
+# The browser should already be on the create/edit resource page.
+@when(u'I create a resource with key-value parameters "{resource_params}"')
+def create_resource_from_params(context, resource_params):
+    context.execute_steps(u"""
+        When I fill in default resource fields
+        And I fill in link resource fields
+    """)
+    for key, value in _parse_params(resource_params):
+        if key == "url":
+            context.execute_steps(u"""
+                When I enter the resource URL "{0}"
+            """.format(value))
+        elif key == "upload":
+            if value == "default":
+                value = "test_game_data.csv"
+            context.execute_steps(u"""
+                When I clear the URL field
+                And I execute the script "$('#resource-upload-button').click();"
+                And I attach the file "{0}" to "upload"
+            """.format(value))
+        elif key == "format":
+            context.execute_steps(u"""
+                When I execute the script "document.getElementById('field-format').value='{0}'"
+            """.format(value))
+        else:
+            context.execute_steps(u"""
+                When I fill in "{0}" with "{1}" if present
+            """.format(key, value))
+    context.execute_steps(u"""
+        When I take a debugging screenshot
+        And I press the element with xpath "//form[contains(@data-module, 'resource-form')]//button[contains(@class, 'btn-primary')]"
+        And I take a debugging screenshot
+    """)
+
+
 @then(u'I should receive a base64 email at "{address}" containing "{text}"')
 def should_receive_base64_email_containing_text(context, address, text):
     should_receive_base64_email_containing_texts(context, address, text, None)
@@ -121,7 +336,7 @@ def filter_contents(mail):
         else:
             import base64
             decoded_payload = six.ensure_text(base64.b64decode(six.ensure_binary(payload_bytes)))
-        print('decoded_payload: ', decoded_payload)
+        print('Searching for', text, ' and ', text2, ' in decoded_payload: ', decoded_payload)
         return text in decoded_payload and (not text2 or text2 in decoded_payload)
 
     assert context.mail.user_messages(address, filter_contents)