Merge pull request #221 from motin/fx60-esr-init

Initial work on upgrading OpenWPM to Firefox ESR 60

.travis.yml

@@ -21,7 +21,6 @@ git:
   depth: 3
 before_install:
   - "export DISPLAY=:99.0"
-  - "npm install -g jpm"
 install:
   - echo "y" | ./install.sh
   - pip install -r requirements.txt

Dockerfile-dev

@@ -11,6 +11,7 @@ FROM openwpm
 
 ADD test /opt/OpenWPM/test/
 ADD install-dev.sh /opt/OpenWPM/
+ADD requirements-dev.txt /opt/OpenWPM/
 
 #=============================================================
 # Install requirements for OpenWPM development

README.md

@@ -377,10 +377,10 @@ dependencies, which can be installed with `install-dev.sh`.
 
 The extension instrumentation is included in `/automation/Extension/firefox/`.
 Any edits within this directory will require the extension to be re-built with
-`jpm` to produce a new `openwpm.xpi` with your updates. For more information on
+`npm run build` to produce a new `openwpm.xpi` with your updates. For more information on
 developing a Firefox extension, we recommend reading this
-[MDN introductory tutorial](https://developer.mozilla.org/en-US/Add-ons/SDK/Tutorials/Getting_Started_(jpm)),
- as well as the [jpm reference page](https://developer.mozilla.org/en-US/Add-ons/SDK/Tools/jpm).
+[MDN introductory tutorial](https://developer.mozilla.org/docs/Mozilla/Add-ons/WebExtensions),
+ as well as the [web-ext reference page](https://developer.mozilla.org/en-US/docs/Mozilla/Add-ons/WebExtensions/Getting_started_with_web-ext).
 
 ### Debugging the platform
 
@@ -392,10 +392,13 @@ continuing the crawl). We recommend using
 This utility allows manual debugging of the extension instrumentation with or
 without Selenium enabled, as well as makes it easy to launch a Selenium
 instance (without any instrumentation)
-* `python -m test.manual_test` uses `jpm` to build the current extension directory
+* `python -m test.manual_test` rebuilds the current extension directory
   and launch a Firefox instance with it.
+* `python automation/SocketIOServer.py` starts the socket.io server in
+  standalone mode (start before manual_test above to debug
+  the communication between the extension and the backend).
 * `python -m test.manual_test --selenium` launches a Firefox Selenium instance
-  after using `jpm` to automatically rebuild `openwpm.xpi`. The script then
+  after automatically rebuilding `openwpm.xpi`. The script then
   drops into an `ipython` shell where the webdriver instance is available
   through variable `driver`.
 * `python -m test.manual_test --selenium --no_extension` launches a Firefox Selenium

automation/BrowserManager.py

@@ -1,6 +1,5 @@
 from __future__ import absolute_import
 
-import errno
 import os
 import shutil
 import signal
@@ -20,7 +19,7 @@
 from .DeployBrowsers import deploy_browser
 from .Errors import BrowserConfigError, BrowserCrashError, ProfileLoadError
 from .MPLogger import loggingclient
-from .SocketInterface import clientsocket
+from .SocketIOServer import start as startSocketIOServer
 
 pickling_support.install()
 
@@ -348,34 +347,16 @@ def BrowserManager(command_queue, status_queue, browser_params,
         if prof_folder[-1] != '/':
             prof_folder += '/'
 
-        # Read the extension port -- if extension is enabled
-        # TODO: Initial communication from extension to TM should use sockets
-        if (browser_params['browser'] == 'firefox' and
-                browser_params['extension_enabled']):
-            logger.debug("BROWSER %i: Looking for extension port information "
-                         "in %s" % (browser_params['crawl_id'], prof_folder))
-            elapsed = 0
-            port = None
-            ep_filename = os.path.join(prof_folder, 'extension_port.txt')
-            while elapsed < 5:
-                try:
-                    with open(ep_filename, 'rt') as f:
-                        port = int(f.read().strip())
-                        break
-                except OSError as e:
-                    if e.errno != errno.ENOENT:
-                        raise
-                time.sleep(0.1)
-                elapsed += 0.1
-            if port is None:
-                # try one last time, allowing all exceptions to propagate
-                with open(ep_filename, 'rt') as f:
-                    port = int(f.read().strip())
-
-            logger.debug("BROWSER %i: Connecting to extension on port %i" % (
-                browser_params['crawl_id'], port))
-            extension_socket = clientsocket(serialization='json')
-            extension_socket.connect('127.0.0.1', int(port))
+        # Allow socket based communication with
+        # extension -- if extension is enabled
+        if browser_params['extension_enabled']:
+            logger.debug("BROWSER %i: Starting socket.io server for "
+                         + "communication with extension"
+                         % (browser_params['crawl_id']))
+            extension_socket = startSocketIOServer(browser_params,
+                                                   manager_params,
+                                                   log_output=True,
+                                                   daemon=False)
         else:
             extension_socket = None
 

automation/DeployBrowsers/deploy_firefox.py

@@ -1,6 +1,5 @@
 from __future__ import absolute_import
 
-import json
 import os.path
 import random
 
@@ -105,19 +104,8 @@ def deploy_firefox(status_queue, browser_params, manager_params,
         ext_loc = os.path.join(root_dir, '../Extension/firefox/openwpm.xpi')
         ext_loc = os.path.normpath(ext_loc)
         fp.add_extension(extension=ext_loc)
-        fo.set_preference("extensions.@openwpm.sdk.console.logLevel", "all")
-        extension_config = dict()
-        extension_config.update(browser_params)
-        extension_config['logger_address'] = manager_params['logger_address']
-        extension_config['sqlite_address'] = manager_params[
-            'aggregator_address']
-        if 'ldb_address' in manager_params:
-            extension_config['leveldb_address'] = manager_params['ldb_address']
-        else:
-            extension_config['leveldb_address'] = None
-        extension_config['testing'] = manager_params['testing']
-        with open(browser_profile_path + 'browser_params.json', 'w') as f:
-            json.dump(extension_config, f)
+        # TODO: Restore preference for log level in a way that works in Fx 57+
+        # fp.set_preference("extensions.@openwpm.sdk.console.logLevel", "all")
         logger.debug("BROWSER %i: OpenWPM Firefox extension loaded"
                      % browser_params['crawl_id'])