From 3c53709e1d181621dced4f82b962397f71d88f36 Mon Sep 17 00:00:00 2001 From: Skileau Date: Wed, 8 Mar 2023 18:44:04 +0100 Subject: [PATCH 1/8] Fixed firefox initialization issues --- sources/firefox/requirements.txt | 3 +- sources/firefox/setup.py | 48 +++++++++++++++++--------------- sources/install.sh | 3 +- 3 files changed, 30 insertions(+), 24 deletions(-) diff --git a/sources/firefox/requirements.txt b/sources/firefox/requirements.txt index 0aa3a3dd6..b3a74e8e5 100644 --- a/sources/firefox/requirements.txt +++ b/sources/firefox/requirements.txt @@ -1 +1,2 @@ -R2Log \ No newline at end of file +R2Log +requests diff --git a/sources/firefox/setup.py b/sources/firefox/setup.py index 4847773a3..b939bd3bd 100644 --- a/sources/firefox/setup.py +++ b/sources/firefox/setup.py @@ -5,23 +5,21 @@ # Date created : 27 February 2023 # Python Version : 3.* -import glob import json import os import re import shutil import subprocess -import time import zipfile import sqlite3 -from pathlib import Path - import requests +from pathlib import Path +from time import sleep from R2Log import logger +from glob import glob PATHNAME = "/root/.mozilla/firefox/**.Exegol/" -# pip3 install R2Log # Define addons urls urls = [ "https://addons.mozilla.org/fr/firefox/addon/foxyproxy-standard/", @@ -51,30 +49,30 @@ def download_addon(link, addon_name): logger.info(f"Downloading addon {addon_name}") addon_dl = requests.get(link) # Save xpi addon on filesystem - with open(addon_name, 'wb') as addon_file: + with open("/tmp/" + addon_name, 'wb') as addon_file: addon_file.write(addon_dl.content) -def read_manifest(addon_name): - archive = zipfile.ZipFile(addon_name, 'r') +def read_manifest(addon_path): + archive = zipfile.ZipFile(addon_path, 'r') manifest = archive.read('manifest.json').decode() # Read the id in the manifest addon_id = re.search(reid, manifest).group(1) return addon_id -def install_addons(addon_name, addon_id): +def install_addons(addon_name, addon_id, addon_path): logger.info(f"Installing addon {addon_name} with id {addon_id}") # Get the path of the Exegol profile try: - dest = glob.glob("%s" % PATHNAME)[0] + dest = glob("%s" % PATHNAME)[0] except: logger.error("Firefox profile Exegol does not exist") raise # Create the extensions folder Path(dest + "/extensions").mkdir(parents=True, exist_ok=True) # Move the addon to the extensions folder - shutil.move(addon_name, dest + "/extensions/" + addon_id + ".xpi") + shutil.move(addon_path + "/" + addon_name, dest + "/extensions/" + addon_id + ".xpi") def activate_addons(addon_list): @@ -85,7 +83,7 @@ def activate_addons(addon_list): else: logger.info(f"Enabling {addon_name}") try: - with open(Path(glob.glob("%s" % PATHNAME)[0] + "/extensions.json"), 'r+') as extensions_file: + with open(Path(glob("%s" % PATHNAME)[0] + "/extensions.json"), 'r+') as extensions_file: extensions_config = json.load(extensions_file) for addon in extensions_config["addons"]: if addon["id"] == addon_id: @@ -107,7 +105,7 @@ def activate_addons(addon_list): pass def adjust_ui(): - with open(Path(glob.glob("%s" % PATHNAME)[0] + "/prefs.js"), 'r+') as pref_js: + with open(Path(glob("%s" % PATHNAME)[0] + "/prefs.js"), 'r+') as pref_js: new_pref = re.sub(r'\\"import-button\\",', '', pref_js.read()) new_pref = re.sub(r'\\"save-to-pocket-button\\",', '', new_pref) new_pref = re.sub('"extensions.activeThemeID", "default-theme@mozilla.org"', '"extensions.activeThemeID", "firefox-compact-dark@mozilla.org"', new_pref) @@ -119,7 +117,7 @@ def import_bookmarks(): dirname = os.path.dirname(__file__) filename = os.path.join(dirname, './places.sqlite') src = sqlite3.connect(filename) - dst = sqlite3.connect(glob.glob("%s" % PATHNAME)[0] + "places.sqlite") + dst = sqlite3.connect(glob("%s" % PATHNAME)[0] + "places.sqlite") with dst: src.backup(dst) dst.close() @@ -131,7 +129,7 @@ def import_bookmarks(): logger.info("Creating Firefox profile") try: subprocess.run(["firefox", "-CreateProfile", "Exegol", "-headless"], stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL) - assert(Path(glob.glob("%s" % PATHNAME)[0]).is_dir()) + assert(Path(glob("%s" % PATHNAME)[0]).is_dir()) logger.success("Firefox profile Exegol created\n") except: logger.error("Could not create Firefox profile Exegol") @@ -147,8 +145,8 @@ def import_bookmarks(): # Download the addon download_addon(link, addon_name) # Read manifest.json in the archive - addon_id = read_manifest(addon_name) - install_addons(addon_name, addon_id) + addon_id = read_manifest("/tmp/" + addon_name) + install_addons(addon_name, addon_id, "/tmp/") logger.success(f"{addon_name} installed sucessfully\n") addon_list.append((addon_id, addon_name[0:-4], False)) @@ -158,9 +156,11 @@ def import_bookmarks(): logger.info("Initialising Firefox profile") try: p_firefox = subprocess.Popen(["firefox", "-P", "Exegol", "-headless"], stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL) - time.sleep(5) + # Wait for firefox to be initialised + while not b'sessionstore-backups' in subprocess.check_output(["ls", glob("%s" % PATHNAME)[0]]): + sleep(0.5) p_firefox.kill() - assert(Path(glob.glob("%s" % PATHNAME)[0] + "/extensions.json").is_file()) + assert(Path(glob("%s" % PATHNAME)[0] + "/extensions.json").is_file()) logger.success("Firefox profile initialised sucessfully\n") except: logger.error("Could not initialise Firefox profile") @@ -178,10 +178,11 @@ def import_bookmarks(): try: adjust_ui() # Remove existing sessions - shutil.rmtree(glob.glob("%s" % PATHNAME)[0] + "sessionstore-backups") + shutil.rmtree(glob("%s" % PATHNAME)[0] + "sessionstore-backups") logger.success("User interface successfully updated\n") except: logger.error("An error has occurred while trying to update user interface\n") + raise # Restore bookmarks logger.info("Setting up profile's bookmarks") @@ -195,7 +196,7 @@ def import_bookmarks(): # Remove backup file interfering with addons activation logger.info("Removing backup file interfering with addons activation") try: - Path(glob.glob("%s" % PATHNAME)[0] + "/addonStartup.json.lz4").unlink() + Path(glob("%s" % PATHNAME)[0] + "/addonStartup.json.lz4").unlink() logger.success("Backup file successfully removed\n") except: logger.error("Could not remove the backup file") @@ -205,8 +206,11 @@ def import_bookmarks(): logger.info("Restarting firefox to apply modifications") try: p_firefox = subprocess.Popen(["firefox", "-headless"], stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL) - time.sleep(5) + # Wait for modifications to be applied + while not b'addonStartup.json.lz4' in subprocess.check_output(["ls", glob("%s" % PATHNAME)[0]]): + sleep(0.5) p_firefox.kill() logger.success("Modifications successfully applied") except: logger.error("Could not restart firefox") + raise diff --git a/sources/install.sh b/sources/install.sh index 4dc6f6d2c..f938bb4e5 100644 --- a/sources/install.sh +++ b/sources/install.sh @@ -3270,11 +3270,12 @@ function install_ctf-party() { function install_firefox() { colorecho "Installing firefox" + fapt firefox-esr mkdir /opt/tools/firefox mv /root/sources/firefox/* /opt/tools/firefox/ python3 -m pip install -r /opt/tools/firefox/requirements.txt python3 /opt/tools/firefox/setup.py - add-test-command "firefox --version" + add-test-command "file /root/.mozilla/firefox/*.Exegol" } # Package dedicated to the basic things the env needs From 3f690955b6da05429de8571af9c39e81357b305c Mon Sep 17 00:00:00 2001 From: Skileau Date: Wed, 8 Mar 2023 18:44:51 +0100 Subject: [PATCH 2/8] Added Firefox addons in my-resources support --- sources/exegol/load_supported_setups.sh | 15 ++++ sources/firefox/user-setup.py | 110 ++++++++++++++++++++++++ 2 files changed, 125 insertions(+) create mode 100644 sources/firefox/user-setup.py diff --git a/sources/exegol/load_supported_setups.sh b/sources/exegol/load_supported_setups.sh index 2ec338501..67fd0ff9a 100644 --- a/sources/exegol/load_supported_setups.sh +++ b/sources/exegol/load_supported_setups.sh @@ -106,6 +106,18 @@ function run_user_setup() { echo "[$(date +'%d-%m-%Y_%H-%M-%S')] ==== End of custom setups loading ====" } +function deploy_firefox_addons() { + if [ -d "$MY_Setup_PATH/firefox/" ]; then + if [ -d "$MY_Setup_PATH/firefox/addons" ]; then + ADDON_FOLDER="-D $MY_Setup_PATH/firefox/addons" + fi + if [ -f "$MY_Setup_PATH/firefox/addons.txt" ]; then + ADDON_LIST="-L $MY_Setup_PATH/firefox/addons.txt" + fi + python3 /opt/tools/firefox/user-setup.py $ADDON_LIST $ADDON_FOLDER + fi +} + # Starting # This procedure is supposed to be executed only once at the first startup, using a lockfile check @@ -124,7 +136,10 @@ deploy_tmux deploy_vim deploy_apt deploy_python3 +deploy_firefox_addons run_user_setup exit 0 + +} \ No newline at end of file diff --git a/sources/firefox/user-setup.py b/sources/firefox/user-setup.py new file mode 100644 index 000000000..7a75c71dd --- /dev/null +++ b/sources/firefox/user-setup.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +# -- coding: utf-8 -- +# File name : user-setup.py +# Author : Skilo (@askilow - Alexis Marquois) +# Date created : 07 march 2023 +# Python Version : 3.* + +from setup import get_link, download_addon, read_manifest, install_addons, activate_addons +from R2Log import logger +from pathlib import Path +from glob import glob +from time import sleep +import re +import subprocess +import argparse + +PATHNAME = "/root/.mozilla/firefox/**.Exegol/" +re_links = r'https://addons\.mozilla\.org/fr/firefox/addon/[^/]+' + +def parse_args(): + arg_parser = argparse.ArgumentParser(description="Automatically installs addons from a list or folder containing .xpi files.") + arg_parser.add_argument('-L', dest="addon_links", help="txt document containing addon link (ie: https://addons.mozilla.org/fr/firefox/addon/duckduckgo-for-firefox).") + arg_parser.add_argument('-D', dest="addon_folder", help="Path to a folder containing .xpi files to install.") + args = arg_parser.parse_args() + return args + +if __name__ == "__main__": + + args = parse_args() + addon_links = args.addon_links + addon_folder = args.addon_folder + install_ok = False + + # Define a list containing all addons names and ids + addon_list = [] + + if addon_links is not None: + # Read the list input by the user + with open(addon_links, "r") as url_file: + urls = url_file.read().splitlines() + + # Iterate through addons + for url in urls: + if re.findall(re_links, url): + # Make a request to the URL + link, addon_name = get_link(url) + # Download the addon + download_addon(link, addon_name) + # Read manifest.json in the archive + addon_id = read_manifest("/tmp/" + addon_name) + install_addons(addon_name, addon_id, "/tmp/") + logger.success(f"{addon_name} installed sucessfully\n") + addon_list.append((addon_id, addon_name[0:-4], False)) + install_ok = True + logger.success("All addons from the list were installed sucessfully\n") + + if addon_folder is not None: + if glob(addon_folder + "/*.xpi"): + for addon_path in glob(addon_folder + "/*.xpi"): + addon_name = addon_path.split("/")[-1] + addon_id = read_manifest(addon_path) + install_addons(addon_name, addon_id, addon_folder) + logger.success(f"{addon_name} installed sucessfully\n") + addon_list.append((addon_id, addon_name[0:-4], False)) + install_ok = True + logger.success("All addons from the folder %s were installed sucessfully\n" % addon_folder) + else: + logger.error("No addons were found in the folder %s." % addon_folder) + + if install_ok: + # Run firefox to initialise profile + logger.info("Initialising Firefox profile") + try: + p_firefox = subprocess.Popen(["firefox", "-P", "Exegol", "-headless"], stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL) + # Wait for firefox to be initialised + while not addon_list[0][0].encode() in subprocess.check_output(["cat", glob("%s" % PATHNAME)[0] + "/extensions.json"]): + sleep(0.5) + p_firefox.kill() + assert(Path(glob("%s" % PATHNAME)[0] + "/extensions.json").is_file()) + logger.success("Firefox profile initialised sucessfully\n") + except: + logger.error("Could not initialise Firefox profile") + raise + + # Activate all addons + activate_addons(addon_list) + + # Remove backup file interfering with addons activation + logger.info("Removing backup file interfering with addons activation") + try: + Path(glob("%s" % PATHNAME)[0] + "/addonStartup.json.lz4").unlink() + logger.success("Backup file successfully removed\n") + except: + logger.error("Could not remove the backup file") + raise + + # Restart firefox to apply modifications + logger.info("Restarting firefox to apply modifications") + try: + p_firefox = subprocess.Popen(["firefox", "-headless"], stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL) + # Wait for modifications to be applied + while not b'addonStartup.json.lz4' in subprocess.check_output(["ls", glob("%s" % PATHNAME)[0]]): + sleep(0.5) + p_firefox.kill() + logger.success("Modifications successfully applied") + except: + logger.error("Could not restart firefox") + raise + else: + logger.error("No addons were found.") From ce325064875ecfee012f91dba24ab52074582d57 Mon Sep 17 00:00:00 2001 From: Skileau <83423277+Skileau@users.noreply.github.com> Date: Thu, 9 Mar 2023 00:42:17 +0100 Subject: [PATCH 3/8] Integrated suggestions Added "firefox --version" test command --- sources/install.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/sources/install.sh b/sources/install.sh index f938bb4e5..eb784657c 100644 --- a/sources/install.sh +++ b/sources/install.sh @@ -3276,6 +3276,7 @@ function install_firefox() { python3 -m pip install -r /opt/tools/firefox/requirements.txt python3 /opt/tools/firefox/setup.py add-test-command "file /root/.mozilla/firefox/*.Exegol" + add-test-command "firefox --version" } # Package dedicated to the basic things the env needs From aec58fb45c1ab337242234992f9f6e2f3e07dcec Mon Sep 17 00:00:00 2001 From: Skileau <83423277+Skileau@users.noreply.github.com> Date: Thu, 9 Mar 2023 00:49:53 +0100 Subject: [PATCH 4/8] Integrated suggestions Added steps to create the firefox/addons directories if missing --- sources/exegol/load_supported_setups.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sources/exegol/load_supported_setups.sh b/sources/exegol/load_supported_setups.sh index 67fd0ff9a..7959c584c 100644 --- a/sources/exegol/load_supported_setups.sh +++ b/sources/exegol/load_supported_setups.sh @@ -107,14 +107,19 @@ function run_user_setup() { } function deploy_firefox_addons() { + ##### firefox custom addons deployment if [ -d "$MY_Setup_PATH/firefox/" ]; then if [ -d "$MY_Setup_PATH/firefox/addons" ]; then ADDON_FOLDER="-D $MY_Setup_PATH/firefox/addons" + else + mkdir "$MY_Setup_PATH/firefox/addons" && chmod 770 "$MY_Setup_PATH/firefox/addons" fi if [ -f "$MY_Setup_PATH/firefox/addons.txt" ]; then ADDON_LIST="-L $MY_Setup_PATH/firefox/addons.txt" fi python3 /opt/tools/firefox/user-setup.py $ADDON_LIST $ADDON_FOLDER + else + mkdir --parents "$MY_Setup_PATH/firefox/addons" && chmod 770 -R "$MY_Setup_PATH/firefox/addons" fi } From 8e34c90711a4b06305e0a0612b5796be91d2b633 Mon Sep 17 00:00:00 2001 From: skileau Date: Thu, 9 Mar 2023 10:21:58 +0100 Subject: [PATCH 5/8] Added initial addons.txt file --- sources/exegol/load_supported_setups.sh | 6 +++--- sources/exegol/skel/firefox/addons.txt | 3 +++ 2 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 sources/exegol/skel/firefox/addons.txt diff --git a/sources/exegol/load_supported_setups.sh b/sources/exegol/load_supported_setups.sh index 7959c584c..012cad365 100644 --- a/sources/exegol/load_supported_setups.sh +++ b/sources/exegol/load_supported_setups.sh @@ -116,6 +116,8 @@ function deploy_firefox_addons() { fi if [ -f "$MY_Setup_PATH/firefox/addons.txt" ]; then ADDON_LIST="-L $MY_Setup_PATH/firefox/addons.txt" + else + cp --preserve=mode /.exegol/skel/firefox/addons.txt "$MY_Setup_PATH/firefox/addons.txt" fi python3 /opt/tools/firefox/user-setup.py $ADDON_LIST $ADDON_FOLDER else @@ -145,6 +147,4 @@ deploy_firefox_addons run_user_setup -exit 0 - -} \ No newline at end of file +exit 0 \ No newline at end of file diff --git a/sources/exegol/skel/firefox/addons.txt b/sources/exegol/skel/firefox/addons.txt new file mode 100644 index 000000000..c6d5abe6f --- /dev/null +++ b/sources/exegol/skel/firefox/addons.txt @@ -0,0 +1,3 @@ +# This file can be used to install addons on the Firefox instance of Exegol. +# The download links of the addons to be installed can be listed in this file (ie: https://addons.mozilla.org/fr/firefox/addon/foxyproxy-standard/). +# All addons listed below will be downloaded and installed automatically when creating a new Exegol container. \ No newline at end of file From e856c65af6637ecc00d895b9278706a86b7cbdea Mon Sep 17 00:00:00 2001 From: skileau Date: Thu, 9 Mar 2023 10:28:00 +0100 Subject: [PATCH 6/8] Added initial addons.txt file --- sources/exegol/load_supported_setups.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/sources/exegol/load_supported_setups.sh b/sources/exegol/load_supported_setups.sh index 012cad365..817040ddd 100644 --- a/sources/exegol/load_supported_setups.sh +++ b/sources/exegol/load_supported_setups.sh @@ -122,6 +122,7 @@ function deploy_firefox_addons() { python3 /opt/tools/firefox/user-setup.py $ADDON_LIST $ADDON_FOLDER else mkdir --parents "$MY_Setup_PATH/firefox/addons" && chmod 770 -R "$MY_Setup_PATH/firefox/addons" + cp --preserve=mode /.exegol/skel/firefox/addons.txt "$MY_Setup_PATH/firefox/addons.txt" fi } From 47261cd1f80fe17c853dac62a9b105b6b712a708 Mon Sep 17 00:00:00 2001 From: Skileau <83423277+Skileau@users.noreply.github.com> Date: Thu, 9 Mar 2023 14:53:28 +0100 Subject: [PATCH 7/8] Patched user-setup.py --- sources/firefox/user-setup.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/sources/firefox/user-setup.py b/sources/firefox/user-setup.py index 7a75c71dd..cc63ebe36 100644 --- a/sources/firefox/user-setup.py +++ b/sources/firefox/user-setup.py @@ -15,7 +15,7 @@ import argparse PATHNAME = "/root/.mozilla/firefox/**.Exegol/" -re_links = r'https://addons\.mozilla\.org/fr/firefox/addon/[^/]+' +re_links = r'^https://addons\.mozilla\.org/fr/firefox/addon/[^/]+' def parse_args(): arg_parser = argparse.ArgumentParser(description="Automatically installs addons from a list or folder containing .xpi files.") @@ -52,7 +52,10 @@ def parse_args(): logger.success(f"{addon_name} installed sucessfully\n") addon_list.append((addon_id, addon_name[0:-4], False)) install_ok = True + if install_ok: logger.success("All addons from the list were installed sucessfully\n") + else: + logger.error("No addons were found in the list %s.\n" % addon_links) if addon_folder is not None: if glob(addon_folder + "/*.xpi"): @@ -65,7 +68,7 @@ def parse_args(): install_ok = True logger.success("All addons from the folder %s were installed sucessfully\n" % addon_folder) else: - logger.error("No addons were found in the folder %s." % addon_folder) + logger.error("No addons were found in the folder %s.\n" % addon_folder) if install_ok: # Run firefox to initialise profile From 479f850d6731570a6b4a347a06ce5075444c6988 Mon Sep 17 00:00:00 2001 From: Skileau <83423277+Skileau@users.noreply.github.com> Date: Thu, 9 Mar 2023 15:16:16 +0100 Subject: [PATCH 8/8] Repatch user-setup.py --- sources/firefox/user-setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sources/firefox/user-setup.py b/sources/firefox/user-setup.py index cc63ebe36..6b5814d38 100644 --- a/sources/firefox/user-setup.py +++ b/sources/firefox/user-setup.py @@ -15,7 +15,7 @@ import argparse PATHNAME = "/root/.mozilla/firefox/**.Exegol/" -re_links = r'^https://addons\.mozilla\.org/fr/firefox/addon/[^/]+' +re_links = r'^https://addons\.mozilla\.org/fr/firefox/addon/[^/]+/?$' def parse_args(): arg_parser = argparse.ArgumentParser(description="Automatically installs addons from a list or folder containing .xpi files.")