diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 8ffd59d6d7ea..5f3b12ff5af6 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -24,6 +24,9 @@ jobs:
   test:
     needs: linux-amd64
     runs-on: ubuntu-latest
+    env:
+      PASSWORD: e45432jklfdsab
+      CODE_SERVER_ADDRESS: http://localhost:8080
     steps:
       - uses: actions/checkout@v1
       - name: Download release packages
@@ -37,7 +40,7 @@ jobs:
       - uses: microsoft/playwright-github-action@v1
       - name: Install dependencies and run tests
         run: |
-          node ./release-packages/code-server*-linux-amd64 &
+          node ./release-packages/code-server*-linux-amd64 --home CODE_SERVER_ADDRESS/test-home &
           yarn --frozen-lockfile
           yarn test
           pkill node
diff --git a/ci/dev/test.sh b/ci/dev/test.sh
index 6b0acd02aaf7..14bea2a5b6ff 100755
--- a/ci/dev/test.sh
+++ b/ci/dev/test.sh
@@ -9,7 +9,8 @@ main() {
   # information. We must also run it from the root otherwise coverage will not
   # include our source files.
   cd "$OLDPWD"
-  ./test/node_modules/.bin/jest "$@"
+  # We use the same environment variables set in ci.yml in the test job
+  PASSWORD=e45432jklfdsab CODE_SERVER_ADDRESS=http://localhost:8080 ./test/node_modules/.bin/jest "$@"
 }
 
 main "$@"
diff --git a/test/goHome.test.ts b/test/goHome.test.ts
index b3faf8252b2e..3377b4c88994 100644
--- a/test/goHome.test.ts
+++ b/test/goHome.test.ts
@@ -1,21 +1,18 @@
 import { chromium, Page, Browser, BrowserContext } from "playwright"
 
-// NOTE: this is hard-coded and passed as an environment variable
-// See the test job in ci.yml
-const PASSWORD = "e45432jklfdsab"
-
 describe("login", () => {
   let browser: Browser
   let page: Page
   let context: BrowserContext
 
   beforeAll(async () => {
-    browser = await chromium.launch({ headless: false })
+    browser = await chromium.launch()
     context = await browser.newContext()
   })
 
   afterAll(async () => {
     await browser.close()
+    await context.close()
   })
 
   beforeEach(async () => {
@@ -29,22 +26,32 @@ describe("login", () => {
   })
 
   it("should see a 'Go Home' button in the Application Menu that goes to coder.com", async () => {
-    await page.goto("http://localhost:8080")
+    // waitUntil: "networkidle"
+    // In case the page takes a long time to load
+    await page.goto(process.env.CODE_SERVER_ADDRESS || "http://localhost:8080", { waitUntil: "networkidle" })
     // Type in password
-    await page.fill(".password", PASSWORD)
+    await page.fill(".password", process.env.PASSWORD || "password")
     // Click the submit button and login
     await page.click(".submit")
-    // Click the Applicaiton menu
+    // Click the Application menu
     await page.click(".menubar-menu-button[title='Application Menu']")
     // See the Go Home button
-    const goHomeButton = ".home-bar[aria-label='Home'] li"
+    const goHomeButton = "a.action-menu-item span[aria-label='Go Home']"
     expect(await page.isVisible(goHomeButton))
-    // Hover over element without clicking
-    await page.hover(goHomeButton)
-    // Click the top left corner of the element
+    // Click it and navigate to coder.com
     await page.click(goHomeButton)
-    // Note: we have to click on <li> in the Go Home button for it to work
-    // Land on coder.com
-    // expect(await page.url()).toBe("https://coder.com/")
+
+    // If there are unsaved changes it will show a dialog
+    // asking if you're sure you want to leave
+    page.on("dialog", (dialog) => dialog.accept())
+
+    // We give it a second to load in case playwright moves too quickly
+    // This resolves after 'networkidle'
+    await page.waitForLoadState("networkidle")
+
+    // We do this rather than using something like http://coder.com or an outside url
+    // so that it's guaranteed to pass everytime
+    expect(await page.url()).toBe(`${process.env.CODE_SERVER_ADDRESS}/test-home`)
+    expect(await page.title()).toBe(`404 - code-server`)
   })
 })
diff --git a/test/login.test.ts b/test/login.test.ts
index 622adddbfba2..460dbc306fb5 100644
--- a/test/login.test.ts
+++ b/test/login.test.ts
@@ -1,9 +1,5 @@
 import { chromium, Page, Browser, BrowserContext } from "playwright"
 
-// NOTE: this is hard-coded and passed as an environment variable
-// See the test job in ci.yml
-const PASSWORD = "e45432jklfdsab"
-
 describe("login", () => {
   let browser: Browser
   let page: Page
@@ -29,9 +25,9 @@ describe("login", () => {
   })
 
   it("should be able to login with the password from config.yml", async () => {
-    await page.goto("http://localhost:8080")
+    await page.goto(process.env.CODE_SERVER_ADDRESS || "http://localhost:8080")
     // Type in password
-    await page.fill(".password", PASSWORD)
+    await page.fill(".password", process.env.PASSWORD || "password")
     // Click the submit button and login
     await page.click(".submit")
     // See the editor