diff --git a/.cache/v/cache/lastfailed b/.cache/v/cache/lastfailed
new file mode 100644
index 00000000..08b07409
--- /dev/null
+++ b/.cache/v/cache/lastfailed
@@ -0,0 +1,3 @@
+{
+  "qtpy/tests/test_patch_qcombobox.py": true
+}
\ No newline at end of file
diff --git a/.travis.yml b/.travis.yml
index d394deec..41c06bd2 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -37,8 +37,9 @@ install:
   - source ci-helpers/travis/setup_conda_$TRAVIS_OS_NAME.sh
   - python setup.py install
 
-before_script:
-  - cd tests
-
 script:
-  - py.test
+  - python qtpy/tests/runtests.py
+
+after_success:
+  - coveralls
+
diff --git a/appveyor.yml b/appveyor.yml
index d310e75e..4c7df5c0 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -35,5 +35,5 @@ install:
 build: false
 
 test_script:
-  - "%CMD_IN_ENV% py.test"
+  - "%CMD_IN_ENV% python qtpy/tests/runtests.py"
 
diff --git a/circle.yml b/circle.yml
index 4edee2ba..6638b5cf 100644
--- a/circle.yml
+++ b/circle.yml
@@ -29,5 +29,5 @@ test:
   override:
     - conda info --json: # note the colon
         parallel: true
-    - "$PYTEST tests/": # note the colon
+    - "$PYTHON_TEST qtpy/tests/runtests.py": # note the colon
         parallel: true
diff --git a/tests/conftest.py b/qtpy/tests/conftest.py
similarity index 100%
rename from tests/conftest.py
rename to qtpy/tests/conftest.py
diff --git a/qtpy/tests/runtests.py b/qtpy/tests/runtests.py
new file mode 100755
index 00000000..b54fbb45
--- /dev/null
+++ b/qtpy/tests/runtests.py
@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# ----------------------------------------------------------------------------
+# Copyright © 2015- The Spyder Development Team
+#
+# Licensed under the terms of the MIT License
+# ----------------------------------------------------------------------------
+
+"""File for running tests programmatically."""
+
+# Standard library imports
+import sys
+
+# Third party imports
+import qtpy  # to ensure that Qt4 uses API v2
+import pytest
+
+
+def main():
+    """Run pytest tests."""
+    errno = pytest.main(['-x', 'qtpy',  '-v', '-rw', '--durations=10',
+                         '--cov=qtpy', '--cov-report=term-missing'])
+    sys.exit(errno)
+
+if __name__ == '__main__':
+    main()
diff --git a/tests/test.ui b/qtpy/tests/test.ui
similarity index 100%
rename from tests/test.ui
rename to qtpy/tests/test.ui
diff --git a/tests/test_custom.ui b/qtpy/tests/test_custom.ui
similarity index 100%
rename from tests/test_custom.ui
rename to qtpy/tests/test_custom.ui
diff --git a/tests/test_main.py b/qtpy/tests/test_main.py
similarity index 100%
rename from tests/test_main.py
rename to qtpy/tests/test_main.py
diff --git a/tests/test_patch_qcombobox.py b/qtpy/tests/test_patch_qcombobox.py
similarity index 99%
rename from tests/test_patch_qcombobox.py
rename to qtpy/tests/test_patch_qcombobox.py
index d5abf409..04a83370 100644
--- a/tests/test_patch_qcombobox.py
+++ b/qtpy/tests/test_patch_qcombobox.py
@@ -2,6 +2,7 @@
 
 from qtpy import QtGui, QtWidgets
 
+
 def get_qapp(icon_path=None):
     qapp = QtWidgets.QApplication.instance()
     if qapp is None:
diff --git a/tests/test_uic.py b/qtpy/tests/test_uic.py
similarity index 100%
rename from tests/test_uic.py
rename to qtpy/tests/test_uic.py
diff --git a/tests/__init__.py b/tests/__init__.py
deleted file mode 100644
index e69de29b..00000000