From e7e9a2f3869895aa446d6a5c122d634fc4b0adcc Mon Sep 17 00:00:00 2001 From: Seth Lyles Date: Wed, 22 Jan 2025 10:17:32 -0800 Subject: [PATCH 1/6] clean up dockerfile with venv and pip --- eyeon.Dockerfile | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/eyeon.Dockerfile b/eyeon.Dockerfile index 7a5a927..bf4032e 100644 --- a/eyeon.Dockerfile +++ b/eyeon.Dockerfile @@ -7,16 +7,13 @@ ARG OUN RUN apt-get update \ && apt-get install -y \ - python3 python3-pip python3-dev python3-venv python3-build \ - libmagic1 git make wget unzip build-essential vim ssdeep jq curl \ + python3 python3-pip python3-dev python3-venv \ + libmagic1 git make wget unzip build-essential vim ssdeep jq \ && apt-get clean RUN groupadd -g $USER_ID $OUN \ && useradd -ms /bin/bash $OUN -u $USER_ID -g $USER_ID -RUN echo "alias build='python3 -m build'" >> /home/$OUN/.bashrc \ - && echo "alias clean='rm -rf /workdir/dist'" >> /home/$OUN/.bashrc \ - && echo "alias rein='build && pip uninstall -y eyeon && pip install /workdir/dist/peyeon*.whl'" >> /home/$OUN/.bashrc \ - && echo "alias eye='source /eye/bin/activate'" >> /home/$OUN/.bashrc +RUN echo "alias eye='source /eye/bin/activate'" >> /home/$OUN/.bashrc RUN wget https://github.com/Kitware/CMake/releases/download/v3.30.3/cmake-3.30.3-linux-x86_64.sh \ && chmod u+x cmake-3.30.3-linux-x86_64.sh \ @@ -30,14 +27,14 @@ RUN cd /opt && git clone https://github.com/trendmicro/tlsh.git \ && ./make.sh -RUN mkdir -p /opt/die \ - && curl -L -o /opt/die/die_3.10_Ubuntu_24.04_amd64.deb https://github.com/horsicq/DIE-engine/releases/download/3.10/die_3.10_Ubuntu_24.04_amd64.deb \ - && apt-get install -y /opt/die/die_3.10_Ubuntu_24.04_amd64.deb \ - && apt-get clean \ +RUN mkdir -p /opt/die && cd /opt/die \ + && wget https://github.com/horsicq/DIE-engine/releases/download/3.10/die_3.10_Ubuntu_24.04_amd64.deb \ + && apt-get install -y /opt/die/die_3.10_Ubuntu_24.04_amd64.deb +RUN apt-get clean \ && rm -rf /var/lib/apt/lists/* RUN python3 -m venv /eye && chown -R $OUN /eye USER $OUN - +RUN . /eye/bin/activate && pip install peyeon ENV PATH=/home/$OUN/.local/bin:$PATH From 08508a639741d66a3c443f0193b1abe634598d00 Mon Sep 17 00:00:00 2001 From: Seth Lyles Date: Wed, 22 Jan 2025 12:18:00 -0800 Subject: [PATCH 2/6] switch base to python3-slim --- eyeon.Dockerfile | 40 +++++++++++++++++++++++++++------------- 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/eyeon.Dockerfile b/eyeon.Dockerfile index bf4032e..b92b9f7 100644 --- a/eyeon.Dockerfile +++ b/eyeon.Dockerfile @@ -1,19 +1,15 @@ -FROM amd64/ubuntu:25.04 - -RUN userdel -r ubuntu # get rid of default user +FROM python:3.13.1-slim-bookworm AS builder ARG USER_ID ARG OUN +ENV DIE="3.10" + RUN apt-get update \ && apt-get install -y \ - python3 python3-pip python3-dev python3-venv \ - libmagic1 git make wget unzip build-essential vim ssdeep jq \ + git make wget unzip build-essential python3 python3-dev python3-venv \ && apt-get clean -RUN groupadd -g $USER_ID $OUN \ - && useradd -ms /bin/bash $OUN -u $USER_ID -g $USER_ID -RUN echo "alias eye='source /eye/bin/activate'" >> /home/$OUN/.bashrc RUN wget https://github.com/Kitware/CMake/releases/download/v3.30.3/cmake-3.30.3-linux-x86_64.sh \ && chmod u+x cmake-3.30.3-linux-x86_64.sh \ @@ -26,15 +22,33 @@ RUN cd /opt && git clone https://github.com/trendmicro/tlsh.git \ && cd /opt/tlsh \ && ./make.sh +RUN python3 -m venv /eye && /eye/bin/pip install peyeon RUN mkdir -p /opt/die && cd /opt/die \ - && wget https://github.com/horsicq/DIE-engine/releases/download/3.10/die_3.10_Ubuntu_24.04_amd64.deb \ - && apt-get install -y /opt/die/die_3.10_Ubuntu_24.04_amd64.deb -RUN apt-get clean \ + && wget https://github.com/horsicq/DIE-engine/releases/download/${DIE}/die_${DIE}_Ubuntu_24.04_amd64.deb + +################################################# + +FROM python:3.13.1-slim-bookworm +COPY --from=builder /opt/die/ /opt/die +COPY --from=builder /opt/tlsh/bin /opt/tlsh/bin +COPY --from=builder /eye /eye +ARG USER_ID +ARG OUN + +ENV DIE="3.10" + +RUN apt-get update \ + && apt-get install -y \ + libmagic1 ssdeep jq /opt/die/die_${DIE}_Ubuntu_24.04_amd64.deb \ + && apt-get clean \ && rm -rf /var/lib/apt/lists/* -RUN python3 -m venv /eye && chown -R $OUN /eye +RUN groupadd -g $USER_ID $OUN \ + && useradd -ms /bin/bash $OUN -u $USER_ID -g $USER_ID + +RUN chown -R $OUN /eye USER $OUN -RUN . /eye/bin/activate && pip install peyeon +ENV PATH="/eye/bin:$PATH" ENV PATH=/home/$OUN/.local/bin:$PATH From da1f9734b202a8476a95f1daacbdfcb41006fa69 Mon Sep 17 00:00:00 2001 From: Seth Lyles Date: Wed, 22 Jan 2025 12:29:09 -0800 Subject: [PATCH 3/6] added conf.py --- docs/conf.py | 94 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 docs/conf.py diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..fafde64 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,94 @@ +import os +import sys + +import requests + +if sys.version_info >= (3, 11): + import tomllib +else: + import tomli as tomllib + +sys.path.insert(0, os.path.abspath("../../src/eyeon/")) + + + +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +project = "EyeON" +# pylint: disable-next=redefined-builtin +copyright = "2024, Lawrence Livermore National Security" +author = "Seth Lyles, Wangmo Tenzing, Jack Mooney, Grant Johnson, Isabel Gardner, Grant Espe" +release = "0.0.0rc4" + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +extensions = [ + "myst_parser", + "sphinx.ext.autodoc", + "sphinx.ext.napoleon", + "sphinx.ext.viewcode", + "sphinx.ext.intersphinx", + "sphinx.ext.githubpages", + "sphinx.ext.autosummary", +] + +templates_path = ["_templates"] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", "images.toml"] + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_theme = "alabaster" +html_theme_options = { + "description": "Surfactant", + "github_user": "LLNL", + "github_repo": "Surfactant", + "github_button": "true", + "github_banner": "true", + "badge_branch": "main", + "fixed_sidebar": "false", +} + +# -- Extension configuration ------------------------------------------------- + +# Napoleon settings for NumPy and Google style docstrings +napoleon_google_docstring = True +napoleon_numpy_docstring = True +html_favicon = html_logo = "../Photo/EyeON_logo.png" +html_sidebars = {"**": ["globaltoc.html", "relations.html", "searchbox.html"]} +html_static_path = ["_static"] + + +# -- Fetch image references -------------------------------------------------- +# Download all of the image files referenced in images.toml +def download_images_from_toml(toml_file, image_dir): + with open(toml_file, "rb") as f: + data = tomllib.load(f) + + if not os.path.exists(image_dir): + os.makedirs(image_dir) + + for file_name, url in data.get("images", {}).items(): + if file_name and url: + response = requests.get(url) + if response.status_code == 200: + with open(os.path.join(image_dir, file_name), "wb") as img_file: + img_file.write(response.content) + else: + print(f"Failed to download {url}") + + +# Path to the TOML file +toml_file_path = os.path.join(os.path.dirname(__file__), "images.toml") +# Directory to save the images +image_directory = os.path.join(os.path.dirname(__file__), "img") + +# Download images +download_images_from_toml(toml_file_path, image_directory) \ No newline at end of file From 7e8e437caf956ea2fd69517b3920697bbb6b9b62 Mon Sep 17 00:00:00 2001 From: Seth Lyles Date: Wed, 22 Jan 2025 13:13:34 -0800 Subject: [PATCH 4/6] update README and installer script --- README.md | 48 +++++++++++++++++++++++++++-------------------- install-ubuntu.sh | 13 ++++++++----- 2 files changed, 36 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 67ec283..f14cc78 100644 --- a/README.md +++ b/README.md @@ -18,36 +18,44 @@ EyeON provides an automated, consistent process across users to scan software fi ## Installation Eyeon can also be run in linux or WSL. +The simplest install can be done with pip: ```bash -git clone git@github.com:LLNL/pEyeON.git +pip install peyeon ``` -or + +However, this does not install a couple key dependencies, namely libmagic, ssdeep, and tlsh. A better way to install is via the docker scripts on the github page: ```bash -git clone https://github.com/LLNL/pEyeON.git +wget ``` -### Dockerfile -This dockerfile contains all the pertinent tools specific to data extraction. The main tools needed are `ssdeep`, `libmagic`, `tlsh`, and `detect-it-easy`. There are a couple variables that need to be changed in order for it to work. -Run docker build script +### Dockerfile +This dockerfile contains all the pertinent tools specific to data extraction. The main tools needed are `ssdeep`, `libmagic`, `tlsh`, and `detect-it-easy`. We have written some convenient scripts: ```bash -./docker-build.sh +wget https://github.com/LLNL/pEyeON/blob/main/docker-build.sh \ + https://github.com/LLNL/pEyeON/blob/main/docker-run.sh \ + https://github.com/LLNL/pEyeON/blob/main/eyeon.Dockerfile +chmod +x docker-build.sh && ./docker-build.sh +chmod +x docker-run.sh && ./docker-run.sh ``` -Run docker run script +This attaches the current directory as a working directory in the container. Files that need to be scanned should go in "tests" folder. If running in a docker container, the eyeon root directory is mounted to "/workdir", so place samples in "/workdir/samples" or "/workdir/tests/samples". + +Cd into workdir directory: ```bash -./docker-run.sh +cd workdir ``` -This attaches current the code directory as a working directory in the container. Files that need to be scanned should go in "tests" folder. If running in a docker container, the eyeon root directory is mounted to "/workdir", so place samples in "/workdir/samples" or "/workdir/tests/samples". +EyeON commands should work now. -Cd into workdir directory, install EyeON, and run 'rein' alias to build python dependencies: +Alternatively, to install on a clean Ubuntu VM: ```bash -cd workdir -rein +wget https://github.com/LLNL/pEyeON/blob/main/install-ubuntu.sh +chmod +x install-ubuntu.sh && ./install-ubuntu.sh ``` -EyeON commands should work now. +To request other options for install, please create an issue on our GitHub page. + ## Usage @@ -72,20 +80,20 @@ EyeON consists of two parts - an observe call and a parse call. `observe.py` wor #### Observe -1. This CLI command calls the observe function and makes an observation of a file. +1. This CLI command calls the `observe` function and makes an observation of a file. CLI command: ```bash -eyeon observe notepad++.exe +eyeon observe demo.ipynb ``` -Init file calls observe function in observe.py +Init file calls observe function in `observe.py` ```bash -obs = eyeon.observe.Observe("./tests/binaries/x86/notepad++/notepad++.exe") +obs = eyeon.observe.Observe("demo.ipynb") ``` -The observation will output a json file containing unique identifying information such as hashes, modify date, certificate info, etc. +The observation will create a json file containing unique identifying information such as hashes, modify date, certificate info, etc. Example json file: @@ -107,7 +115,7 @@ Example json file: ``` #### Parse -parse.py calls observe recursively, returning an observation for each file in a directory. +`parse.py` calls `observe` recursively, returning an observation for each file in a directory. ```bash obs = eyeon.parse.Parse(args.dir) diff --git a/install-ubuntu.sh b/install-ubuntu.sh index 566bbf1..82111d2 100644 --- a/install-ubuntu.sh +++ b/install-ubuntu.sh @@ -2,9 +2,9 @@ export eyeon_dir=$(pwd) # dependencies -apt update +apt-get update DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC \ - apt install -y python3 python3-pip python3-dev python3-venv \ + apt-get install -y python3 python3-pip python3-dev python3-venv \ libmagic1 git make wget unzip build-essential vim ssdeep jq # cmake, have to build telfhash @@ -20,9 +20,12 @@ cd /opt && git clone https://github.com/trendmicro/tlsh.git cd /opt/tlsh ./make.sh +mkdir /opt/die && cd /opt/die +wget https://github.com/horsicq/DIE-engine/releases/download/${DIE}/die_${DIE}_Ubuntu_24.04_amd64.deb +apt-get install -y die_${DIE}_Ubuntu_24.04_amd64.deb +apt-get clean + cd $eyeon_dir # set up virtual environment python3 -m venv eye && source eye/bin/activate -pip install build sphinx -python3 -m build -pip install ./dist/eyeon-*.whl +pip install peyeon From 17f3fdfa624dc0206d7309f84c92ab69d6062c48 Mon Sep 17 00:00:00 2001 From: Seth Lyles Date: Wed, 22 Jan 2025 13:17:48 -0800 Subject: [PATCH 5/6] formatting --- README.md | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index f14cc78..33d22f0 100644 --- a/README.md +++ b/README.md @@ -18,16 +18,12 @@ EyeON provides an automated, consistent process across users to scan software fi ## Installation Eyeon can also be run in linux or WSL. -The simplest install can be done with pip: +The simplest install can be done with `pip`: ```bash pip install peyeon ``` -However, this does not install a couple key dependencies, namely libmagic, ssdeep, and tlsh. A better way to install is via the docker scripts on the github page: -```bash -wget -``` - +However, this does not install several key dependencies, namely `libmagic`, `ssdeep`, and `tlsh`. A better way to install is via the container or install scripts on the github page. ### Dockerfile This dockerfile contains all the pertinent tools specific to data extraction. The main tools needed are `ssdeep`, `libmagic`, `tlsh`, and `detect-it-easy`. We have written some convenient scripts: From bf866809cee6008fcce35b980a896ad8b86137fe Mon Sep 17 00:00:00 2001 From: Seth Lyles Date: Wed, 22 Jan 2025 13:19:45 -0800 Subject: [PATCH 6/6] change to eyeon --- docs/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index fafde64..71adfdc 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -47,9 +47,9 @@ html_theme = "alabaster" html_theme_options = { - "description": "Surfactant", + "description": "EyeON", "github_user": "LLNL", - "github_repo": "Surfactant", + "github_repo": "pEyeON", "github_button": "true", "github_banner": "true", "badge_branch": "main",