From 16961be0cde69589f884dc790520c7f3438c5024 Mon Sep 17 00:00:00 2001
From: Sam Macbeth <smacbeth@duckduckgo.com>
Date: Fri, 5 Feb 2021 18:01:58 +0100
Subject: [PATCH 1/4] Add crawler detection page

---
 crawler/fail.json  |  0
 crawler/index.html | 49 ++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 49 insertions(+)
 create mode 100644 crawler/fail.json
 create mode 100644 crawler/index.html
diff --git a/crawler/fail.json b/crawler/fail.json
new file mode 100644
index 0000000..e69de29
diff --git a/crawler/index.html b/crawler/index.html
new file mode 100644
index 0000000..8a55f2d
--- /dev/null
+++ b/crawler/index.html
@@ -0,0 +1,49 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta encoding='utf-8'/>
+    <title>Headless detector</title>
+  </head>
+
+  <body>
+    <table>
+        <tr>
+            <th>Test</th>
+            <th>Result</th>
+        </tr>
+        <tbody id="results"></tbody>
+    </table>
+    <template id="result-row">
+        <tr>
+            <td class="key"></td>
+            <td class="value"></td>
+        </tr>
+    </template>
+    <script>
+        (async () => {
+            const ul = document.getElementById('results');
+            const template = document.getElementById('result-row');
+            
+            function displayResult(test, result, detectValue) {
+                const listEntry = template.content.cloneNode(true);
+                listEntry.querySelector('.key').textContent = test;
+                listEntry.querySelector('.value').textContent = result ? 'fail' : 'pass';
+                ul.appendChild(listEntry)
+                if (result) {
+                    fetch(`./fail.json?test=${test}&${detectValue}`);
+                }
+            }
+
+            displayResult('userAgent', /HeadlessChrome/.test(window.navigator.userAgent), window.navigator.userAgent);
+            displayResult('plugins', navigator.plugins.length == 0, navigator.plugins.length);
+            displayResult('languages', navigator.languages == "", navigator.languages);
+            displayResult('webdriver', !!navigator.webdriver, navigator.webdriver)
+            displayResult('window.chrome', !window.chrome || !window.chrome.runtime, JSON.stringify(window.chrome))
+
+            const permissionStatus = await navigator.permissions.query({ name: 'notifications' });
+            const isHeadless = Notification.permission === 'denied' && permissionStatus.state === 'prompt'
+            displayResult('Notification.permission', isHeadless, `${Notification.permission} - ${permissionStatus.state}`)
+        })()
+    </script>
+  </body>
+</html>
\ No newline at end of file

From 3ebb34be12f57e1c359623cc8b6c9000e708daae Mon Sep 17 00:00:00 2001
From: Sam Macbeth <smacbeth@duckduckgo.com>
Date: Mon, 8 Feb 2021 12:20:35 +0100
Subject: [PATCH 2/4] Update page to match template

---
 crawler/index.html | 93 ++++++++++++++++++++++++----------------------
 1 file changed, 49 insertions(+), 44 deletions(-)

diff --git a/crawler/index.html b/crawler/index.html
index 8a55f2d..a970ec5 100644
--- a/crawler/index.html
+++ b/crawler/index.html
@@ -1,49 +1,54 @@
 <!DOCTYPE html>
-<html>
-  <head>
-    <meta encoding='utf-8'/>
-    <title>Headless detector</title>
-  </head>
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width">
+  <title>Crawler Detectability test</title>
+</head>  
 
-  <body>
-    <table>
-        <tr>
-            <th>Test</th>
-            <th>Result</th>
-        </tr>
-        <tbody id="results"></tbody>
-    </table>
-    <template id="result-row">
-        <tr>
-            <td class="key"></td>
-            <td class="value"></td>
-        </tr>
-    </template>
-    <script>
-        (async () => {
-            const ul = document.getElementById('results');
-            const template = document.getElementById('result-row');
-            
-            function displayResult(test, result, detectValue) {
-                const listEntry = template.content.cloneNode(true);
-                listEntry.querySelector('.key').textContent = test;
-                listEntry.querySelector('.value').textContent = result ? 'fail' : 'pass';
-                ul.appendChild(listEntry)
-                if (result) {
-                    fetch(`./fail.json?test=${test}&${detectValue}`);
-                }
-            }
+<body>
+<p><a href="../index.html">[Home]</a></p>
 
-            displayResult('userAgent', /HeadlessChrome/.test(window.navigator.userAgent), window.navigator.userAgent);
-            displayResult('plugins', navigator.plugins.length == 0, navigator.plugins.length);
-            displayResult('languages', navigator.languages == "", navigator.languages);
-            displayResult('webdriver', !!navigator.webdriver, navigator.webdriver)
-            displayResult('window.chrome', !window.chrome || !window.chrome.runtime, JSON.stringify(window.chrome))
+<p>This page tests against common ways of detecting headless browsers.</p>
 
-            const permissionStatus = await navigator.permissions.query({ name: 'notifications' });
-            const isHeadless = Notification.permission === 'denied' && permissionStatus.state === 'prompt'
-            displayResult('Notification.permission', isHeadless, `${Notification.permission} - ${permissionStatus.state}`)
-        })()
-    </script>
-  </body>
+<table>
+    <tr>
+        <th>Test</th>
+        <th>Result</th>
+    </tr>
+    <tbody id="results"></tbody>
+</table>
+<template id="result-row">
+    <tr>
+        <td class="key"></td>
+        <td class="value"></td>
+    </tr>
+</template>
+<script>
+  (async () => {
+    const ul = document.getElementById('results');
+    const template = document.getElementById('result-row');
+    
+    function displayResult(test, result, detectValue) {
+      const listEntry = template.content.cloneNode(true);
+      listEntry.querySelector('.key').textContent = test;
+      listEntry.querySelector('.value').textContent = result ? 'fail' : 'pass';
+      ul.appendChild(listEntry)
+      if (result) {
+        // this is used to allow the crawler to log test failures via its network log
+        fetch(`./fail.json?test=${test}&${detectValue}`);
+      }
+    }
+
+    displayResult('userAgent', /HeadlessChrome/.test(window.navigator.userAgent), window.navigator.userAgent);
+    displayResult('plugins', navigator.plugins.length == 0, navigator.plugins.length);
+    displayResult('languages', navigator.languages == "", navigator.languages);
+    displayResult('webdriver', !!navigator.webdriver, navigator.webdriver)
+    displayResult('window.chrome', !window.chrome || !window.chrome.runtime, JSON.stringify(window.chrome))
+
+    const permissionStatus = await navigator.permissions.query({ name: 'notifications' });
+    const isHeadless = Notification.permission === 'denied' && permissionStatus.state === 'prompt'
+    displayResult('Notification.permission', isHeadless, `${Notification.permission} - ${permissionStatus.state}`)
+  })()
+</script>
+</body>
 </html>
\ No newline at end of file

From 7b452a5ad43daa9b11ee76e731ac81602789c4d8 Mon Sep 17 00:00:00 2001
From: Sam Macbeth <smacbeth@duckduckgo.com>
Date: Mon, 8 Feb 2021 12:20:48 +0100
Subject: [PATCH 3/4] Add link to crawler test

---
 index.html | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/index.html b/index.html
index 7332c48..5ce392e 100644
--- a/index.html
+++ b/index.html
@@ -48,5 +48,11 @@ <h2>Privacy Protections Tests</h2>
 	<li><a href='http://privacy-test-pages.glitch.me/privacy-protections/https-upgrades/'>HTTPS upgrades</a></li>
 </ul>
 
+<h2>Other</h2>
+
+<ul>
+	<li><a href='./crawler/'>Crawler Detectability test</a></li>
+</ul>
+
 </body>
 </html>

From 131d2a6cb922518c813425a1096ea4bafd980596 Mon Sep 17 00:00:00 2001
From: Sam Macbeth <smacbeth@duckduckgo.com>
Date: Mon, 8 Feb 2021 12:24:28 +0100
Subject: [PATCH 4/4] Update breadcrumbs

---
 crawler/index.html | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/crawler/index.html b/crawler/index.html
index a970ec5..3566d12 100644
--- a/crawler/index.html
+++ b/crawler/index.html
@@ -6,10 +6,12 @@
 </head>  
 
 <body>
-<p><a href="../index.html">[Home]</a></p>
+<p><a href="../index.html">[Home]</a> ↣ <strong>[Crawler Detectability test]</strong></p>
 
 <p>This page tests against common ways of detecting headless browsers.</p>
 
+<p>A pass means that this looks like a normal Chrome browser, fail means it's Headless Chrome.</p>
+
 <table>
     <tr>
         <th>Test</th>
