diff --git a/cabal-testsuite/README.md b/cabal-testsuite/README.md
index 1fba1d85446..b5036803bce 100644
--- a/cabal-testsuite/README.md
+++ b/cabal-testsuite/README.md
@@ -96,6 +96,15 @@ Otherwise, here is a walkthrough:
      ...
    ```
 
+   The dependencies which your test is allowed to use are listed in the
+   cabal file under the `test-runtime-deps` executable. At compile-time there is
+   a custom Setup.hs script which inspects this list and records the versions of
+   each package in a generated file. These are then used when `cabal-tests` runs
+   when it invokes `runghc` to run each test.
+   We ensure they are built and available by listing `test-runtime-deps` in the
+   build-tool-depends section of the cabal-tests executable.
+
+
 3. Run your tests using `cabal-tests` (no need to rebuild when
    you add or modify a test; it is automatically picked up).
    The first time you run a test, assuming everything else is
diff --git a/cabal-testsuite/Setup.hs b/cabal-testsuite/Setup.hs
index 2b212906a60..d83f9dc60e8 100644
--- a/cabal-testsuite/Setup.hs
+++ b/cabal-testsuite/Setup.hs
@@ -73,7 +73,7 @@ canonicalizePackageDB x = return x
 -- non-Backpack.
 cabalTestsPackages :: LocalBuildInfo -> [(OpenUnitId, ModuleRenaming)]
 cabalTestsPackages lbi =
-    case componentNameCLBIs lbi (CExeName (mkUnqualComponentName "cabal-tests")) of
+    case componentNameCLBIs lbi (CExeName (mkUnqualComponentName "test-runtime-deps")) of
         [clbi] -> -- [ (unUnitId $ unDefUnitId duid,rn) | (DefiniteUnitId duid, rn) <- componentIncludes clbi ]
                   componentIncludes clbi
         _ -> error "cabalTestsPackages"
diff --git a/cabal-testsuite/cabal-testsuite.cabal b/cabal-testsuite/cabal-testsuite.cabal
index 72221b316d5..55aa7921b52 100644
--- a/cabal-testsuite/cabal-testsuite.cabal
+++ b/cabal-testsuite/cabal-testsuite.cabal
@@ -90,6 +90,8 @@ executable cabal-tests
   main-is: cabal-tests.hs
   hs-source-dirs: main
   ghc-options: -threaded
+  -- Make sure these are built before the executable is run
+  build-tool-depends: cabal-testsuite:test-runtime-deps
   build-depends:
     , cabal-testsuite
     -- constraints inherited via lib:cabal-testsuite component
@@ -101,18 +103,6 @@ executable cabal-tests
     , transformers
     -- dependencies specific to exe:cabal-tests
     , clock                 ^>= 0.7.2 || ^>=0.8
-    -- Extra dependencies used by PackageTests.
-    --
-    -- The runner allows the tests to use extra dependencies and the custom Prelude
-    -- from 'cabal-testsuite'.
-    -- However, if the tests use a dependency, say 'directory', and there are two
-    -- packages with the same unit id available in the store, the test fails since
-    -- it doesn't know which one to pick.
-    -- By including an extra dependency to directory, we force the test runner to
-    -- use a specific version directory, fixing the test failure.
-    --
-    -- See issue description and discussion: https://github.com/haskell/cabal/issues/8356
-    , directory
 
   build-tool-depends: cabal-testsuite:setup
   default-extensions: TypeOperators
@@ -122,6 +112,32 @@ executable setup
   import: shared
   main-is: Setup.simple.hs
 
+-- This executable component is used to describe the runtime dependencies of
+-- the tests. The Main.hs file and resulting executable are not useful in any way.
+
+-- Ideally this would be an empty library, but because build-type: Custom, we can't
+-- have sublibraries.
+
+-- If you require an external dependency for a test it must be listed here.
+executable test-runtime-deps
+  build-depends: cabal-testsuite,
+                 base,
+                 directory,
+                 Cabal,
+                 Cabal-syntax,
+                 filepath,
+                 transformers,
+                 bytestring,
+                 time,
+                 process,
+                 exceptions
+  main-is: static/Main.hs
+  if !os(windows)
+    build-depends: unix
+  else
+    build-depends:
+      , Win32
+
 custom-setup
   -- we only depend on even stable releases of lib:Cabal
   -- and due to Custom complexity and ConstraintSetupCabalMaxVersion
diff --git a/cabal-testsuite/src/Test/Cabal/Script.hs b/cabal-testsuite/src/Test/Cabal/Script.hs
index a7ce082a97b..943ea784c8d 100644
--- a/cabal-testsuite/src/Test/Cabal/Script.hs
+++ b/cabal-testsuite/src/Test/Cabal/Script.hs
@@ -93,6 +93,7 @@ runnerGhcArgs senv =
   where
     ghc_options = M.mempty { ghcOptPackageDBs = runnerPackageDbStack senv
                            , ghcOptPackages   = toNubListR (runnerPackages senv)
+                           , ghcOptHideAllPackages = Flag True
                            -- Avoid picking stray module files that look
                            -- like our imports
                            , ghcOptSourcePathClear = Flag True }
diff --git a/cabal-testsuite/static/Main.hs b/cabal-testsuite/static/Main.hs
new file mode 100644
index 00000000000..de106fe48f9
--- /dev/null
+++ b/cabal-testsuite/static/Main.hs
@@ -0,0 +1,3 @@
+module Main where
+
+main = return ()