diff --git a/cms/djangoapps/contentstore/features/advanced-settings.feature b/cms/djangoapps/contentstore/features/advanced-settings.feature index ca5b62e59..6f6cc5070 100644 --- a/cms/djangoapps/contentstore/features/advanced-settings.feature +++ b/cms/djangoapps/contentstore/features/advanced-settings.feature @@ -11,7 +11,8 @@ Feature: Advanced (manual) course policy Given I am on the Advanced Course Settings page in Studio Then the settings are alphabetized - @skip-phantom + # Skipped because Ubuntu ChromeDriver cannot click notification "Cancel" + @skip Scenario: Test cancel editing key value Given I am on the Advanced Course Settings page in Studio When I edit the value of a policy key @@ -20,7 +21,8 @@ Feature: Advanced (manual) course policy And I reload the page Then the policy key value is unchanged - @skip-phantom + # Skipped because Ubuntu ChromeDriver cannot click notification "Save" + @skip Scenario: Test editing key value Given I am on the Advanced Course Settings page in Studio When I edit the value of a policy key and save @@ -28,7 +30,8 @@ Feature: Advanced (manual) course policy And I reload the page Then the policy key value is changed - @skip-phantom + # Skipped because Ubuntu ChromeDriver cannot edit CodeMirror input + @skip Scenario: Test how multi-line input appears Given I am on the Advanced Course Settings page in Studio When I create a JSON object as a value @@ -36,7 +39,8 @@ Feature: Advanced (manual) course policy And I reload the page Then it is displayed as formatted - @skip-phantom + # Skipped because Ubuntu ChromeDriver cannot edit CodeMirror input + @skip Scenario: Test automatic quoting of non-JSON values Given I am on the Advanced Course Settings page in Studio When I create a non-JSON value not in quotes diff --git a/cms/djangoapps/contentstore/features/checklists.feature b/cms/djangoapps/contentstore/features/checklists.feature index ddf1adf26..3767144c9 100644 --- a/cms/djangoapps/contentstore/features/checklists.feature +++ b/cms/djangoapps/contentstore/features/checklists.feature @@ -10,8 +10,6 @@ Feature: Course checklists Then I can check and uncheck tasks in a checklist And They are correctly selected after I reload the page - @skip-phantom - @skip-firefox Scenario: A task can link to a location within Studio Given I have opened Checklists When I select a link to the course outline @@ -19,8 +17,6 @@ Feature: Course checklists And I press the browser back button Then I am brought back to the course outline in the correct state - @skip-phantom - @skip-firefox Scenario: A task can link to a location outside Studio Given I have opened Checklists When I select a link to help page diff --git a/cms/djangoapps/contentstore/features/course-settings.feature b/cms/djangoapps/contentstore/features/course-settings.feature index fc9641cb4..e869bfe47 100644 --- a/cms/djangoapps/contentstore/features/course-settings.feature +++ b/cms/djangoapps/contentstore/features/course-settings.feature @@ -1,20 +1,17 @@ Feature: Course Settings As a course author, I want to be able to configure my course settings. - @skip-phantom Scenario: User can set course dates Given I have opened a new course in Studio When I select Schedule and Details And I set course dates Then I see the set dates on refresh - @skip-phantom Scenario: User can clear previously set course dates (except start date) Given I have set course dates And I clear all the dates except start Then I see cleared dates on refresh - @skip-phantom Scenario: User cannot clear the course start date Given I have set course dates And I clear the course start date diff --git a/cms/djangoapps/contentstore/features/section.feature b/cms/djangoapps/contentstore/features/section.feature index 24cbeb3db..236cf501f 100644 --- a/cms/djangoapps/contentstore/features/section.feature +++ b/cms/djangoapps/contentstore/features/section.feature @@ -3,7 +3,6 @@ Feature: Create Section As a course author I want to create and edit sections - @skip-phantom Scenario: Add a new section to a course Given I have opened a new course in Studio When I click the New Section link @@ -27,7 +26,8 @@ Feature: Create Section And I save a new section release date Then the section release date is updated - @skip-phantom + # Skipped because Ubuntu ChromeDriver hangs on alert + @skip Scenario: Delete section Given I have opened a new course in Studio And I have added a new section diff --git a/cms/djangoapps/contentstore/features/signup.py b/cms/djangoapps/contentstore/features/signup.py index 6ca358183..398f8d074 100644 --- a/cms/djangoapps/contentstore/features/signup.py +++ b/cms/djangoapps/contentstore/features/signup.py @@ -18,10 +18,7 @@ def i_fill_in_the_registration_form(step): @step('I press the Create My Account button on the registration form$') def i_press_the_button_on_the_registration_form(step): submit_css = 'form#register_form button#submit' - # Workaround for click not working on ubuntu - # for some unknown reason. - e = world.css_find(submit_css) - e.type(' ') + world.css_click(submit_css) @step('I should see be on the studio home page$') diff --git a/cms/djangoapps/contentstore/features/studio-overview-togglesection.feature b/cms/djangoapps/contentstore/features/studio-overview-togglesection.feature index a0e0a48f9..c9f5b43df 100644 --- a/cms/djangoapps/contentstore/features/studio-overview-togglesection.feature +++ b/cms/djangoapps/contentstore/features/studio-overview-togglesection.feature @@ -14,7 +14,6 @@ Feature: Overview Toggle Section When I navigate to the course overview page Then I do not see the "Collapse All Sections" link - @skip-phantom Scenario: Collapse link appears after creating first section of a course Given I have a course with no sections When I navigate to the course overview page @@ -22,7 +21,8 @@ Feature: Overview Toggle Section Then I see the "Collapse All Sections" link And all sections are expanded - @skip-phantom + # Skipped because Ubuntu ChromeDriver hangs on alert + @skip Scenario: Collapse link is not removed after last section of a course is deleted Given I have a course with 1 section And I navigate to the course overview page diff --git a/cms/djangoapps/contentstore/features/subsection.feature b/cms/djangoapps/contentstore/features/subsection.feature index 28285bf8a..8bb12467f 100644 --- a/cms/djangoapps/contentstore/features/subsection.feature +++ b/cms/djangoapps/contentstore/features/subsection.feature @@ -3,14 +3,12 @@ Feature: Create Subsection As a course author I want to create and edit subsections - @skip-phantom Scenario: Add a new subsection to a section Given I have opened a new course section in Studio When I click the New Subsection link And I enter the subsection name and click save Then I see my subsection on the Courseware page - @skip-phantom Scenario: Add a new subsection (with a name containing a quote) to a section (bug #216) Given I have opened a new course section in Studio When I click the New Subsection link @@ -27,7 +25,6 @@ Feature: Create Subsection And I reload the page Then I see it marked as Homework - @skip-phantom Scenario: Set a due date in a different year (bug #256) Given I have opened a new subsection in Studio And I have set a release date and due date in different years @@ -35,7 +32,8 @@ Feature: Create Subsection And I reload the page Then I see the correct dates - @skip-phantom + # Skipped because Ubuntu ChromeDriver hangs on alert + @skip Scenario: Delete a subsection Given I have opened a new course section in Studio And I have added a new subsection diff --git a/cms/djangoapps/contentstore/features/subsection.py b/cms/djangoapps/contentstore/features/subsection.py index f9e5b52bb..edc8b1716 100644 --- a/cms/djangoapps/contentstore/features/subsection.py +++ b/cms/djangoapps/contentstore/features/subsection.py @@ -63,14 +63,6 @@ def test_have_set_dates_in_different_years(step): set_date_and_time('input#due_date', '01/02/2012', 'input#due_time', '04:00') -@step('I see the correct dates$') -def i_see_the_correct_dates(step): - assert_equal('12/25/2011', world.css_find('input#start_date').first.value) - assert_equal('03:00', world.css_find('input#start_time').first.value) - assert_equal('01/02/2012', world.css_find('input#due_date').first.value) - assert_equal('04:00', world.css_find('input#due_time').first.value) - - @step('I mark it as Homework$') def i_mark_it_as_homework(step): world.css_click('a.menu-toggle') @@ -101,8 +93,20 @@ def the_subsection_does_not_exist(step): assert world.browser.is_element_not_present_by_css(css) +@step('I see the correct dates$') +def i_see_the_correct_dates(step): + assert_equal('12/25/2011', get_date('input#start_date')) + assert_equal('03:00', get_date('input#start_time')) + assert_equal('01/02/2012', get_date('input#due_date')) + assert_equal('04:00', get_date('input#due_time')) + + ############ HELPER METHODS ################### +def get_date(css): + return world.css_find(css).first.value.strip() + + def save_subsection_name(name): name_css = 'input.new-subsection-name-input' save_css = 'input.new-subsection-name-save' diff --git a/cms/envs/acceptance.py b/cms/envs/acceptance.py index 1e7a32dc6..f4b867d3c 100644 --- a/cms/envs/acceptance.py +++ b/cms/envs/acceptance.py @@ -8,27 +8,41 @@ # otherwise the browser will not render the pages correctly DEBUG = True -# Show the courses that are in the data directory -COURSES_ROOT = ENV_ROOT / "data" -DATA_DIR = COURSES_ROOT -# MODULESTORE = { -# 'default': { -# 'ENGINE': 'xmodule.modulestore.xml.XMLModuleStore', -# 'OPTIONS': { -# 'data_dir': DATA_DIR, -# 'default_class': 'xmodule.hidden_module.HiddenDescriptor', -# } -# } -# } +# Disable warnings for acceptance tests, to make the logs readable +import logging +logging.disable(logging.ERROR) +MODULESTORE_OPTIONS = { + 'default_class': 'xmodule.raw_module.RawDescriptor', + 'host': 'localhost', + 'db': 'test_xmodule', + 'collection': 'acceptance_modulestore', + 'fs_root': TEST_ROOT / "data", + 'render_template': 'mitxmako.shortcuts.render_to_string', +} + +MODULESTORE = { + 'default': { + 'ENGINE': 'xmodule.modulestore.mongo.MongoModuleStore', + 'OPTIONS': MODULESTORE_OPTIONS + }, + 'direct': { + 'ENGINE': 'xmodule.modulestore.mongo.MongoModuleStore', + 'OPTIONS': MODULESTORE_OPTIONS + }, + 'draft': { + 'ENGINE': 'xmodule.modulestore.mongo.DraftMongoModuleStore', + 'OPTIONS': MODULESTORE_OPTIONS + } +} # Set this up so that rake lms[acceptance] and running the # harvest command both use the same (test) database # which they can flush without messing up your dev db DATABASES = { 'default': { 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': ENV_ROOT / "db" / "test_mitx.db", - 'TEST_NAME': ENV_ROOT / "db" / "test_mitx.db", + 'NAME': TEST_ROOT / "db" / "test_mitx.db", + 'TEST_NAME': TEST_ROOT / "db" / "test_mitx.db", } } diff --git a/cms/envs/test.py b/cms/envs/test.py index 63b5efc64..06ea5309e 100644 --- a/cms/envs/test.py +++ b/cms/envs/test.py @@ -41,8 +41,8 @@ 'default_class': 'xmodule.raw_module.RawDescriptor', 'host': 'localhost', 'db': 'test_xmodule', - 'collection': 'modulestore', - 'fs_root': GITHUB_REPO_ROOT, + 'collection': 'test_modulestore', + 'fs_root': TEST_ROOT / "data", 'render_template': 'mitxmako.shortcuts.render_to_string', } diff --git a/common/djangoapps/terrain/browser.py b/common/djangoapps/terrain/browser.py index 1d371a324..b0b6db3ad 100644 --- a/common/djangoapps/terrain/browser.py +++ b/common/djangoapps/terrain/browser.py @@ -1,50 +1,102 @@ +""" +Browser set up for acceptance tests. +""" + +#pylint: disable=E1101 +#pylint: disable=W0613 +#pylint: disable=W0611 + from lettuce import before, after, world from splinter.browser import Browser from logging import getLogger from django.core.management import call_command from django.conf import settings +from selenium.common.exceptions import WebDriverException # Let the LMS and CMS do their one-time setup # For example, setting up mongo caches from lms import one_time_startup from cms import one_time_startup -logger = getLogger(__name__) -logger.info("Loading the lettuce acceptance testing terrain file...") +# There is an import issue when using django-staticfiles with lettuce +# Lettuce assumes that we are using django.contrib.staticfiles, +# but the rest of the app assumes we are using django-staticfiles +# (in particular, django-pipeline and our mako implementation) +# To resolve this, we check whether staticfiles is installed, +# then redirect imports for django.contrib.staticfiles +# to use staticfiles. +try: + import staticfiles +except ImportError: + pass +else: + import sys + sys.modules['django.contrib.staticfiles'] = staticfiles + +LOGGER = getLogger(__name__) +LOGGER.info("Loading the lettuce acceptance testing terrain file...") + +MAX_VALID_BROWSER_ATTEMPTS = 20 @before.harvest def initial_setup(server): - ''' - Launch the browser once before executing the tests - ''' + """ + Launch the browser once before executing the tests. + """ browser_driver = getattr(settings, 'LETTUCE_BROWSER', 'chrome') - world.browser = Browser(browser_driver) + + # There is an issue with ChromeDriver2 r195627 on Ubuntu + # in which we sometimes get an invalid browser session. + # This is a work-around to ensure that we get a valid session. + success = False + num_attempts = 0 + while (not success) and num_attempts < MAX_VALID_BROWSER_ATTEMPTS: + + # Get a browser session + world.browser = Browser(browser_driver) + + # Try to visit the main page + # If the browser session is invalid, this will + # raise a WebDriverException + try: + world.visit('/') + + except WebDriverException: + world.browser.quit() + num_attempts += 1 + + else: + success = True + + # If we were unable to get a valid session within the limit of attempts, + # then we cannot run the tests. + if not success: + raise IOError("Could not acquire valid ChromeDriver browser session.") @before.each_scenario def reset_data(scenario): - ''' + """ Clean out the django test database defined in the envs/acceptance.py file: mitx_all/db/test_mitx.db - ''' - logger.debug("Flushing the test database...") + """ + LOGGER.debug("Flushing the test database...") call_command('flush', interactive=False) @after.each_scenario def screenshot_on_error(scenario): - ''' - Save a screenshot to help with debugging - ''' + """ + Save a screenshot to help with debugging. + """ if scenario.failed: world.browser.driver.save_screenshot('/tmp/last_failed_scenario.png') @after.all def teardown_browser(total): - ''' - Quit the browser after executing the tests - ''' + """ + Quit the browser after executing the tests. + """ world.browser.quit() - pass diff --git a/common/djangoapps/terrain/course_helpers.py b/common/djangoapps/terrain/course_helpers.py index 9d6837ae8..cc1f77021 100644 --- a/common/djangoapps/terrain/course_helpers.py +++ b/common/djangoapps/terrain/course_helpers.py @@ -38,9 +38,11 @@ def create_user(uname): @world.absorb def log_in(username, password): - ''' - Log the user in programatically - ''' + """ + Log the user in programatically. + This will delete any existing cookies to ensure that the user + logs in to the correct session. + """ # Authenticate the user user = authenticate(username=username, password=password) @@ -60,15 +62,8 @@ def log_in(username, password): # Retrieve the sessionid and add it to the browser's cookies cookie_dict = {settings.SESSION_COOKIE_NAME: request.session.session_key} - try: - world.browser.cookies.add(cookie_dict) - - # WebDriver has an issue where we cannot set cookies - # before we make a GET request, so if we get an error, - # we load the '/' page and try again - except: - world.browser.visit(django_url('/')) - world.browser.cookies.add(cookie_dict) + world.browser.cookies.delete() + world.browser.cookies.add(cookie_dict) @world.absorb diff --git a/common/djangoapps/terrain/ui_helpers.py b/common/djangoapps/terrain/ui_helpers.py index d4d99e17b..1a8590413 100644 --- a/common/djangoapps/terrain/ui_helpers.py +++ b/common/djangoapps/terrain/ui_helpers.py @@ -53,12 +53,9 @@ def is_visible(driver): @world.absorb def css_click(css_selector): - ''' - First try to use the regular click method, - but if clicking in the middle of an element - doesn't work it might be that it thinks some other - element is on top of it there so click in the upper left - ''' + """ + Perform a click on a CSS selector, retrying if it initially fails + """ try: world.browser.find_by_css(css_selector).click() diff --git a/doc/testing.md b/doc/testing.md index 84175fee3..d6c7b7ee8 100644 --- a/doc/testing.md +++ b/doc/testing.md @@ -161,36 +161,36 @@ try running `bundle install` to install the required ruby gems. We use [Lettuce](http://lettuce.it/) for acceptance testing. Most of our tests use [Splinter](http://splinter.cobrateam.info/) to simulate UI browser interactions. Splinter, in turn, -uses [Selenium](http://docs.seleniumhq.org/) to control the browser. +uses [Selenium](http://docs.seleniumhq.org/) to control the Chrome browser. **Prerequisite**: You must have [ChromeDriver](https://code.google.com/p/selenium/wiki/ChromeDriver) -installed to run the tests in Chrome. +installed to run the tests in Chrome. The tests are confirmed to run +with Chrome (not Chromium) version 26.0.0.1410.63 with ChromeDriver +version r195636. -Before running the tests, you need to set up the test database: +To run all the acceptance tests: - rm ../db/test_mitx.db - rake django-admin[syncdb,lms,acceptance,--noinput] - rake django-admin[migrate,lms,acceptance,--noinput] + rake test_acceptance_lms + rake test_acceptance_cms -To run the acceptance tests: - -1. Start the Django server locally using the settings in **acceptance.py**: +To test only a specific feature: - rake lms[acceptance] + rake test_acceptance_lms[lms/djangoapps/courseware/features/problems.feature] -2. In another shell, run the tests: +To start the debugger on failure, add the `--pdb` option: - django-admin.py harvest --no-server --settings=lms.envs.acceptance --pythonpath=. lms/djangoapps/portal/features/ + rake test_acceptance_lms["lms/djangoapps/courseware/features/problems.feature --pdb"] -To test only a specific feature: +To run tests faster by not collecting static files, you can use +`rake fasttest_acceptance_lms` and `rake fasttest_acceptance_cms`. - django-admin.py harvest --no-server --settings=lms.envs.acceptance --pythonpath=. lms/djangoapps/courseware/features/high-level-tabs.feature **Troubleshooting**: If you get an error message that says something about harvest not being a command, you probably are missing a requirement. Try running: pip install -r requirements.txt +**Note**: The acceptance tests can *not* currently run in parallel. ## Viewing Test Coverage diff --git a/jenkins/test.sh b/jenkins/test.sh index 9734531e9..32279fe22 100755 --- a/jenkins/test.sh +++ b/jenkins/test.sh @@ -78,7 +78,7 @@ rake test_lms[false] || TESTS_FAILED=1 rake test_common/lib/capa || TESTS_FAILED=1 rake test_common/lib/xmodule || TESTS_FAILED=1 -# Run the jaavascript unit tests +# Run the javascript unit tests rake phantomjs_jasmine_lms || TESTS_FAILED=1 rake phantomjs_jasmine_cms || TESTS_FAILED=1 rake phantomjs_jasmine_common/lib/xmodule || TESTS_FAILED=1 diff --git a/jenkins/test_acceptance.sh b/jenkins/test_acceptance.sh new file mode 100755 index 000000000..1d11265d0 --- /dev/null +++ b/jenkins/test_acceptance.sh @@ -0,0 +1,39 @@ +#! /bin/bash + +set -e +set -x + +git remote prune origin + +# Reset the submodule, in case it changed +git submodule foreach 'git reset --hard HEAD' + +# Set the IO encoding to UTF-8 so that askbot will start +export PYTHONIOENCODING=UTF-8 + +if [ ! -d /mnt/virtualenvs/"$JOB_NAME" ]; then + mkdir -p /mnt/virtualenvs/"$JOB_NAME" + virtualenv /mnt/virtualenvs/"$JOB_NAME" +fi + +export PIP_DOWNLOAD_CACHE=/mnt/pip-cache + +source /mnt/virtualenvs/"$JOB_NAME"/bin/activate +rake install_prereqs +rake clobber + +TESTS_FAILED=0 + +# Assumes that Xvfb has been started by upstart +# and is capturing display :1 +# The command for this is: +# /usr/bin/Xvfb :1 -screen 0 1024x268x24 +# This allows us to run Chrome without a display +export DISPLAY=:1 + +# Run the lms and cms acceptance tests +# (the -v flag turns off color in the output) +rake test_acceptance_lms["-v 3"] || TESTS_FAILED=1 +rake test_acceptance_cms["-v 3"] || TESTS_FAILED=1 + +[ $TESTS_FAILED == '0' ] diff --git a/lms/envs/acceptance.py b/lms/envs/acceptance.py index 2c51dda5e..611c3fdac 100644 --- a/lms/envs/acceptance.py +++ b/lms/envs/acceptance.py @@ -8,13 +8,17 @@ # otherwise the browser will not render the pages correctly DEBUG = True +# Disable warnings for acceptance tests, to make the logs readable +import logging +logging.disable(logging.ERROR) + # Use the mongo store for acceptance tests modulestore_options = { 'default_class': 'xmodule.raw_module.RawDescriptor', 'host': 'localhost', 'db': 'test_xmodule', - 'collection': 'modulestore', - 'fs_root': GITHUB_REPO_ROOT, + 'collection': 'acceptance_modulestore', + 'fs_root': TEST_ROOT / "data", 'render_template': 'mitxmako.shortcuts.render_to_string', } @@ -33,7 +37,7 @@ 'ENGINE': 'xmodule.contentstore.mongo.MongoContentStore', 'OPTIONS': { 'host': 'localhost', - 'db': 'test_xcontent', + 'db': 'test_xmodule', } } @@ -43,8 +47,8 @@ DATABASES = { 'default': { 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': ENV_ROOT / "db" / "test_mitx.db", - 'TEST_NAME': ENV_ROOT / "db" / "test_mitx.db", + 'NAME': TEST_ROOT / "db" / "test_mitx.db", + 'TEST_NAME': TEST_ROOT / "db" / "test_mitx.db", } } diff --git a/rakefiles/tests.rake b/rakefiles/tests.rake index c3c7c7258..ebe8ea637 100644 --- a/rakefiles/tests.rake +++ b/rakefiles/tests.rake @@ -24,6 +24,13 @@ def run_tests(system, report_dir, stop_on_failure=true) end end +def run_acceptance_tests(system, report_dir, harvest_args) + sh(django_admin(system, 'acceptance', 'syncdb', '--noinput')) + sh(django_admin(system, 'acceptance', 'migrate', '--noinput')) + sh(django_admin(system, 'acceptance', 'harvest', '--debug-mode', '--tag -skip', harvest_args)) +end + + directory REPORT_DIR task :clean_test_files do @@ -46,6 +53,17 @@ TEST_TASK_DIRS = [] run_tests(system, report_dir, args.stop_on_failure) end + # Run acceptance tests + desc "Run acceptance tests" + task "test_acceptance_#{system}", [:harvest_args] => ["#{system}:gather_assets:acceptance", "fasttest_acceptance_#{system}"] + + desc "Run acceptance tests without collectstatic" + task "fasttest_acceptance_#{system}", [:harvest_args] => ["clean_test_files", :predjango, report_dir] do |t, args| + args.with_defaults(:harvest_args => '') + run_acceptance_tests(system, report_dir, args.harvest_args) + end + + task :fasttest => "fasttest_#{system}" TEST_TASK_DIRS << system @@ -116,4 +134,4 @@ namespace :coverage do sh("coverage xml -o #{report_dir}/coverage.xml --rcfile=#{dir}/.coveragerc") end end -end \ No newline at end of file +end