From dd353a09a759b9b70558852eb7b665a80ff308cb Mon Sep 17 00:00:00 2001 From: Jake Jepson <55201008+Jepson2k@users.noreply.github.com> Date: Thu, 2 Jan 2025 21:58:21 -0500 Subject: [PATCH 1/6] changing the order improved dockers ability to cache the large download section and building as well as expecting a license files to be included --- ofrak-angr.yml | 2 +- ofrak-binary-ninja.yml | 2 +- ofrak-core-dev.yml | 2 +- ofrak-dev.yml | 7 ++++--- ofrak-ghidra.yml | 7 ++++--- ofrak-tutorial.yml | 15 ++++++++------- 6 files changed, 19 insertions(+), 16 deletions(-) diff --git a/ofrak-angr.yml b/ofrak-angr.yml index 0b001ae1f..aa8838fb5 100644 --- a/ofrak-angr.yml +++ b/ofrak-angr.yml @@ -3,9 +3,9 @@ base_image_name: "angr-base" image_name: "angr" packages_paths: [ + "ofrak_patch_maker", # Patch maker dockerstub includes lots of downloads. Keeping it first helps with docker caching. "ofrak_type", "ofrak_io", - "ofrak_patch_maker", "ofrak_core", "disassemblers/ofrak_angr", "disassemblers/ofrak_capstone", diff --git a/ofrak-binary-ninja.yml b/ofrak-binary-ninja.yml index 1aa31f32f..6e33b531c 100644 --- a/ofrak-binary-ninja.yml +++ b/ofrak-binary-ninja.yml @@ -3,9 +3,9 @@ base_image_name: "binary-ninja-base" image_name: "binary-ninja" packages_paths: [ + "ofrak_patch_maker", # Patch maker dockerstub includes lots of downloads. Keeping it first helps with docker caching. "ofrak_type", "ofrak_io", - "ofrak_patch_maker", "ofrak_core", "disassemblers/ofrak_binary_ninja", "disassemblers/ofrak_capstone", diff --git a/ofrak-core-dev.yml b/ofrak-core-dev.yml index b9466dd09..86424660a 100644 --- a/ofrak-core-dev.yml +++ b/ofrak-core-dev.yml @@ -3,9 +3,9 @@ base_image_name: "core-dev-base" image_name: "core-dev" packages_paths: [ + "ofrak_patch_maker", # Patch maker dockerstub includes lots of downloads. Keeping it first helps with docker caching. "ofrak_type", "ofrak_io", - "ofrak_patch_maker", "ofrak_core", "frontend", ] diff --git a/ofrak-dev.yml b/ofrak-dev.yml index e41ec347a..32af104da 100644 --- a/ofrak-dev.yml +++ b/ofrak-dev.yml @@ -3,9 +3,9 @@ base_image_name: "dev-base" image_name: "dev" packages_paths: [ + "ofrak_patch_maker", # Patch maker dockerstub includes lots of downloads. Keeping it first helps with docker caching. "ofrak_type", "ofrak_io", - "ofrak_patch_maker", "ofrak_core", "disassemblers/ofrak_ghidra", "disassemblers/ofrak_binary_ninja", @@ -21,5 +21,6 @@ extra_build_args: ] entrypoint: | nginx \ - & python3 -m ofrak_ghidra.server start \ - & python3 -m ofrak gui -H 0.0.0.0 -p 8877 --backend ghidra + && ofrak license -l /ofrak.license --i-agree \ + && python3 -m ofrak_ghidra.server start \ + && ofrak gui -H 0.0.0.0 -p 8877 --backend ghidra diff --git a/ofrak-ghidra.yml b/ofrak-ghidra.yml index 491bdcc59..a4f74ed8c 100644 --- a/ofrak-ghidra.yml +++ b/ofrak-ghidra.yml @@ -3,14 +3,15 @@ base_image_name: "ghidra-base" image_name: "ghidra" packages_paths: [ + "ofrak_patch_maker", # Patch maker dockerstub includes lots of downloads. Keeping it first helps with docker caching. "ofrak_type", "ofrak_io", - "ofrak_patch_maker", "ofrak_core", "disassemblers/ofrak_ghidra", "frontend", ] entrypoint: | nginx \ - & python3 -m ofrak_ghidra.server start \ - & python3 -m ofrak gui -H 0.0.0.0 -p 8877 --backend ghidra + && ofrak license -l /ofrak.license --i-agree \ + && python3 -m ofrak_ghidra.server start \ + && ofrak gui -H 0.0.0.0 -p 8877 --backend ghidra diff --git a/ofrak-tutorial.yml b/ofrak-tutorial.yml index c463176ba..cc95cbb7e 100644 --- a/ofrak-tutorial.yml +++ b/ofrak-tutorial.yml @@ -3,9 +3,9 @@ base_image_name: "tutorial-base" image_name: "tutorial" packages_paths: [ + "ofrak_patch_maker", # Patch maker dockerstub includes lots of downloads. Keeping it first helps with docker caching. "ofrak_type", "ofrak_io", - "ofrak_patch_maker", "ofrak_core", "disassemblers/ofrak_ghidra", "examples", @@ -13,9 +13,10 @@ packages_paths: "ofrak_tutorial" ] entrypoint: | - python -m ofrak_ghidra.server start \ - && jupyter notebook \ - --no-browser \ - --allow-root \ - --ip 0.0.0.0 \ - --notebook-dir "/ofrak_tutorial/notebooks" + ofrak license -l ofrak.license --i-agree \ + && python -m ofrak_ghidra.server start \ + && jupyter notebook \ + --no-browser \ + --allow-root \ + --ip 0.0.0.0 \ + --notebook-dir "/ofrak_tutorial/notebooks" From bf9fa01a09d40f30041bf67bd088452db31e068f Mon Sep 17 00:00:00 2001 From: Jake Jepson <55201008+Jepson2k@users.noreply.github.com> Date: Tue, 7 Jan 2025 20:51:40 +0000 Subject: [PATCH 2/6] Add Makefile targets for creating and starting docker images --- Makefile | 243 +++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 235 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 581a19a97..0ad6dd890 100644 --- a/Makefile +++ b/Makefile @@ -1,20 +1,247 @@ .PHONY: check-black -check-black: +check-black: ## Run black in check + diff mode black . --check --diff .PHONY: autoflake -autoflake: +autoflake: ## Remove all unused imports recursively autoflake --in-place --remove-all-unused-imports --ignore-init-module-imports -r . -c .PHONY: inspect -inspect: autoflake check-black +inspect: ## Runs autoflake then check-black + autoflake + make check-black +# For compatibility, this target calls the pattern rule for "ofrak-core-dev" +# so "make image" continues to work as before. .PHONY: image -image: - python3 build_image.py --config ofrak-core-dev.yml --base --finish +image: ofrak-core-dev ## Build ofrak-core-dev image -tutorial-image: - python3 build_image.py --config ofrak-tutorial.yml --base --finish +# For compatibility, this target calls the pattern rule for "ofrak-tutorial" +# so "make tutorial-image" continues to work as before. +.PHONY: tutorial-image +tutorial-image: ofrak-tutorial ## Build OFRAK tutorial Docker image -tutorial-run: +.PHONY: tutorial-run +tutorial-run: ## Run the tutorial inside ofrak_tutorial make -C ofrak_tutorial run + +# ---------------------------------------------------------------------------- +# Ensure OFRAK license is present +# ---------------------------------------------------------------------------- + +# We'll store the license locally in ofrak.license +license_file = $(PWD)/ofrak.license + +.PHONY: ensure-ofrak-license +ensure-ofrak-license: + @if [ ! -f "$(license_file)" ]; then \ + echo "No local 'ofrak.license' found."; \ + echo "Launching container to obtain license from IMAGE: 'redballoonsecurity/ofrak/$(image_name):latest'..."; \ + docker run --name ofrak-license-check \ + -it \ + --entrypoint '' \ + redballoonsecurity/ofrak/$(image_name):latest \ + ofrak license; \ + if [ "$$?" -eq 0 ]; then \ + echo "License accepted. Copying license file out of container..."; \ + docker cp ofrak-license-check:/ofrak_core/ofrak/license/license.json "$(license_file)"; \ + else \ + echo "License was NOT accepted or command failed. Exiting..."; \ + docker rm ofrak-license-check >/dev/null 2>&1 || true; \ + exit 1; \ + fi; \ + docker rm ofrak-license-check >/dev/null 2>&1 || true; \ + fi + +# ---------------------------------------------------------------------------- +# BUILD VARIOUS OFRAK IMAGES WITH A PATTERN RULE +# ---------------------------------------------------------------------------- + +# Pattern rule for building images named "ofrak-". +# Usage examples: +# make ofrak-dev -> builds with ofrak-dev.yml +# make ofrak-angr -> builds with ofrak-angr.yml +# make ofrak-binary-ninja-> builds with ofrak-binary-ninja.yml +# make ofrak-core-dev -> builds with ofrak-core-dev.yml +# make ofrak-ghidra -> builds with ofrak-ghidra.yml +# make ofrak-tutorial -> builds with ofrak-tutorial.yml +.PHONY: ofrak-% +ofrak-%: ## Build OFRAK image using ofrak- + @echo "Building OFRAK image using config: ofrak-$*.yml" + python3 build_image.py --config ofrak-$*.yml --base --finish + +# ---------------------------------------------------------------------------- +# START VARIOUS OFRAK IMAGES WITH A PATTERN RULE +# ---------------------------------------------------------------------------- + +# Pattern rule for starting images named "ofrak-". +# Usage examples: +# make start-ofrak-dev -> starts ofrak-dev image +# make start-ofrak-angr -> starts ofrak-angr image +# make start-ofrak-binary-ninja-> starts ofrak-binary-ninja image +# make start-ofrak-core-dev -> starts ofrak-core-dev image +# make start-ofrak-ghidra -> starts ofrak-ghidra image +# make start-ofrak-tutorial -> starts ofrak-tutorial image +.PHONY: start-ofrak-% +start-ofrak-%: ## Start OFRAK image using ofrak- + make ensure-ofrak-license image_name=$* + @echo "Starting OFRAK image using config: ofrak-$*.yml..." + docker run \ + --rm \ + --detach \ + --hostname ofrak \ + --name ofrak-$* \ + --interactive \ + --tty \ + --publish 8877:80 \ + --volume $(pwd)/ofrak.license:/ofrak.license \ + redballoonsecurity/ofrak/$*:latest + +define check_for_binja_license + $(shell python3 -c "import os; license_path = os.path.join(os.getcwd(), 'license.dat'); vol_args = f'--mount type=bind,source={license_path},target=/root/.binaryninja/license.dat' if os.path.isfile(license_path) else ''; print(vol_args)") +endef + +# Yes this is actually how you input a newline in a makefile +define newline + + +endef + +# Function to get package paths from YAML +# $(1) - config file path +# $(shell grep -A100 "packages_paths:" $(1) | grep -v "packages_paths:" | grep -B100 "extra_build_args\|$$" | grep "^[[:space:]]*-" | sed 's/[[:space:]]*-[[:space:]]*//') +define get_packages +$(shell grep -A100 "packages_paths:" ofrak-ghidra.yml | grep -v "packages_paths:" | grep -B100 "]" | sed -n 's/.*"\([^"]*\)".*/\1/p') +endef + +# Function to generate volume mounts +# $(1) - list of packages +define volume_mounts +$(foreach pkg,$(1),--volume "$$(pwd)"/$(pkg):/$(shell basename $(pkg)) \$(newline)) +endef + +.PHONY: super-start-ofrak-% +super-start-ofrak-%: IMAGE_NAME = redballoonsecurity/ofrak/$*:latest +super-start-ofrak-%: ensure-ofrak-license ## Start OFRAK image with mounted volumes, profiling tools, misc dev utils, and a modified entrypoint + @echo "Starting OFRAK "super" image using config: ofrak-$*.yml..." + $(eval CONFIG_FILE := ofrak-$*.yml) + $(eval PACKAGES := $(call get_packages,$(CONFIG_FILE))) + docker run \ + --rm \ + --detach \ + --hostname ofrak \ + --name super-ofrak-$* \ + --interactive \ + --tty \ + --publish 8877:80 \ + --cap-add SYS_PTRACE \ + $(call check_for_binja_license,) \ + $(call volume_mounts,$(PACKAGES)) --entrypoint bash \ + redballoonsecurity/ofrak/dev:latest \ + -c 'make develop \ + && ofrak license --community --i-agree \ + && (sleep infinity)' + + @echo "Install common profiling packages (py-spy, yappi) and other utilities..." + docker exec -it super-ofrak-$* bash -c "python3 -m pip install py-spy yappi" + docker exec -it super-ofrak-$* bash -c "apt-get install -y vim nano less" + +.PHONY: use-local-in-setup-py +use-local-in-setup-py: ## Back up and modify setup.py references to local OFRAK packages + @echo "Backing up and modifying all setup.py files..." + find . -name "setup.py" ! -name "*.bak" -exec sh -c '[ -f "{}.bak" ] || cp "{}" "{}.bak"' \; + find . -name "setup.py" -exec sed -i -E 's/(ofrak_[A-Za-z0-9_]+)~=[0-9]+\.[0-9]+/\1 @ file:\/\/\/\1/g' {} \; + @echo "Done. All setup.py references updated to local paths." + +.PHONY: restore-setup-py +restore-setup-py: ## Restore setup.py from backups + @echo "Restoring any setup.py from its backup..." + find . -name "setup.py.bak" -exec sh -c 'f="{}"; mv "$$f" "$${f%.bak}"' \; + @echo "Done. All setup.py files have been restored." + +# ---------------------------------------------------------------------------- +# TEST TARGETS SIMILAR TO THE GITHUB CI JOBS +# ---------------------------------------------------------------------------- + +.PHONY: test-ofrak-ghidra +test-ofrak-ghidra: IMAGE_NAME = redballoonsecurity/ofrak/ghidra:latest +test-ofrak-ghidra: ## Build the Ghidra Docker image and run documentation + main OFRAK tests + @echo "=== Building the Ghidra image (similar to GH Actions) ===" + python3 -m pip install --quiet PyYAML + python3 build_image.py \ + --config ofrak-ghidra.yml \ + --base \ + --finish + + @echo "=== Testing documentation build in ofrak-ghidra ===" + docker run \ + --interactive \ + --rm \ + --entrypoint bash \ + --volume "$(PWD)":/ofrak \ + redballoonsecurity/ofrak/ghidra:latest \ + -c "cd /ofrak && mkdocs build --site-dir /tmp/docs" + + @echo "=== Testing OFRAK components in ofrak-ghidra ===" + docker run \ + --interactive \ + --rm \ + --entrypoint bash \ + redballoonsecurity/ofrak/ghidra:latest \ + -c "python -m ofrak_ghidra.server start \ + && ofrak license --community --i-agree \ + && make test" + +.PHONY: test-ofrak-angr +test-ofrak-angr: IMAGE_NAME = redballoonsecurity/ofrak/angr:latest +test-ofrak-angr: ## Build the angr Docker image and run angr/capstone tests + @echo "=== Building the angr image (similar to GH Actions) ===" + python3 -m pip install --quiet PyYAML + python3 build_image.py \ + --config ofrak-angr.yml \ + --base \ + --finish + + @echo "=== Testing OFRAK angr + capstone components ===" + docker run \ + --interactive \ + --rm \ + --entrypoint bash \ + --volume "$(PWD)":/ofrak \ + redballoonsecurity/ofrak/angr:latest \ + -c "ofrak license --community --i-agree \ + && make -C /ofrak_angr test \ + && make -C /ofrak_capstone test" + +.PHONY: test-ofrak-tutorial +test-ofrak-tutorial: IMAGE_NAME = redballoonsecurity/ofrak/tutorial:latest +test-ofrak-tutorial: ## Build the tutorial Docker image and run the examples + tutorial tests + @echo "=== Building the tutorial image (similar to GH Actions) ===" + python3 -m pip install --quiet PyYAML + python3 build_image.py \ + --config ofrak-tutorial.yml \ + --base \ + --finish + + @echo "=== Testing OFRAK tutorial notebooks and examples ===" + docker run \ + --interactive \ + --rm \ + --entrypoint bash \ + redballoonsecurity/ofrak/tutorial:latest \ + -c "python -m ofrak_ghidra.server start \ + && ofrak license --community --i-agree \ + && make -C /examples test \ + && make -C /ofrak_tutorial test" + +.PHONY: test-all +test-all: test-ofrak-ghidra test-ofrak-angr test-ofrak-tutorial ## Run all tests (Ghidra, angr, tutorial) + @echo "=== All tests completed! ===" + +.PHONY: help +help: ## Display this help message. + @echo "Usage: make [target]" + @echo "" + @echo "Targets:" + @echo "" + @awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_%-]+:.*?## / {printf " %-25s %s\n", $$1, $$2}' $(MAKEFILE_LIST) From 6f35ea3eb2a4ca299de6725d4a4dc32be538f820 Mon Sep 17 00:00:00 2001 From: Jake Jepson <55201008+Jepson2k@users.noreply.github.com> Date: Fri, 3 Jan 2025 15:29:18 -0500 Subject: [PATCH 3/6] bug fix to change super dev start up --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 0ad6dd890..02de9f52a 100644 --- a/Makefile +++ b/Makefile @@ -121,8 +121,8 @@ $(foreach pkg,$(1),--volume "$$(pwd)"/$(pkg):/$(shell basename $(pkg)) \$(newlin endef .PHONY: super-start-ofrak-% -super-start-ofrak-%: IMAGE_NAME = redballoonsecurity/ofrak/$*:latest -super-start-ofrak-%: ensure-ofrak-license ## Start OFRAK image with mounted volumes, profiling tools, misc dev utils, and a modified entrypoint +super-start-ofrak-%: ## Start OFRAK image with mounted volumes, profiling tools, misc dev utils, and a modified entrypoint + make ensure-ofrak-license image_name=$* @echo "Starting OFRAK "super" image using config: ofrak-$*.yml..." $(eval CONFIG_FILE := ofrak-$*.yml) $(eval PACKAGES := $(call get_packages,$(CONFIG_FILE))) From afd5545f762fbdfd11369a4ef6a7a6df4e88dcaa Mon Sep 17 00:00:00 2001 From: Jake Jepson <55201008+Jepson2k@users.noreply.github.com> Date: Fri, 3 Jan 2025 15:32:13 -0500 Subject: [PATCH 4/6] more bug fix --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 02de9f52a..e423d0db8 100644 --- a/Makefile +++ b/Makefile @@ -137,7 +137,7 @@ super-start-ofrak-%: ## Start OFRAK image with mounted volumes, profiling tools, --cap-add SYS_PTRACE \ $(call check_for_binja_license,) \ $(call volume_mounts,$(PACKAGES)) --entrypoint bash \ - redballoonsecurity/ofrak/dev:latest \ + redballoonsecurity/ofrak/$*:latest \ -c 'make develop \ && ofrak license --community --i-agree \ && (sleep infinity)' From 3175f7ce0c9869d340aea8bd8b30a794a4dae361 Mon Sep 17 00:00:00 2001 From: Albert Zhang Date: Tue, 7 Jan 2025 20:53:13 +0000 Subject: [PATCH 5/6] Change $(pwd) to $(PWD) to fix typo in binding license file --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e423d0db8..eec991e04 100644 --- a/Makefile +++ b/Makefile @@ -94,7 +94,7 @@ start-ofrak-%: ## Start OFRAK image using ofrak- --interactive \ --tty \ --publish 8877:80 \ - --volume $(pwd)/ofrak.license:/ofrak.license \ + --volume $(PWD)/ofrak.license:/ofrak.license \ redballoonsecurity/ofrak/$*:latest define check_for_binja_license From 5cd87a7b9fd5625927d2ca7d6b23d6ee3a61177e Mon Sep 17 00:00:00 2001 From: Jake Jepson <55201008+Jepson2k@users.noreply.github.com> Date: Wed, 22 Jan 2025 12:16:59 -0500 Subject: [PATCH 6/6] unable to reproduce docker caching issues -> undoing those changes. --- ofrak-angr.yml | 2 +- ofrak-binary-ninja.yml | 2 +- ofrak-core-dev.yml | 2 +- ofrak-dev.yml | 2 +- ofrak-ghidra.yml | 2 +- ofrak-tutorial.yml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ofrak-angr.yml b/ofrak-angr.yml index aa8838fb5..0b001ae1f 100644 --- a/ofrak-angr.yml +++ b/ofrak-angr.yml @@ -3,9 +3,9 @@ base_image_name: "angr-base" image_name: "angr" packages_paths: [ - "ofrak_patch_maker", # Patch maker dockerstub includes lots of downloads. Keeping it first helps with docker caching. "ofrak_type", "ofrak_io", + "ofrak_patch_maker", "ofrak_core", "disassemblers/ofrak_angr", "disassemblers/ofrak_capstone", diff --git a/ofrak-binary-ninja.yml b/ofrak-binary-ninja.yml index 6e33b531c..1aa31f32f 100644 --- a/ofrak-binary-ninja.yml +++ b/ofrak-binary-ninja.yml @@ -3,9 +3,9 @@ base_image_name: "binary-ninja-base" image_name: "binary-ninja" packages_paths: [ - "ofrak_patch_maker", # Patch maker dockerstub includes lots of downloads. Keeping it first helps with docker caching. "ofrak_type", "ofrak_io", + "ofrak_patch_maker", "ofrak_core", "disassemblers/ofrak_binary_ninja", "disassemblers/ofrak_capstone", diff --git a/ofrak-core-dev.yml b/ofrak-core-dev.yml index 86424660a..b9466dd09 100644 --- a/ofrak-core-dev.yml +++ b/ofrak-core-dev.yml @@ -3,9 +3,9 @@ base_image_name: "core-dev-base" image_name: "core-dev" packages_paths: [ - "ofrak_patch_maker", # Patch maker dockerstub includes lots of downloads. Keeping it first helps with docker caching. "ofrak_type", "ofrak_io", + "ofrak_patch_maker", "ofrak_core", "frontend", ] diff --git a/ofrak-dev.yml b/ofrak-dev.yml index 32af104da..3c9798058 100644 --- a/ofrak-dev.yml +++ b/ofrak-dev.yml @@ -3,9 +3,9 @@ base_image_name: "dev-base" image_name: "dev" packages_paths: [ - "ofrak_patch_maker", # Patch maker dockerstub includes lots of downloads. Keeping it first helps with docker caching. "ofrak_type", "ofrak_io", + "ofrak_patch_maker", "ofrak_core", "disassemblers/ofrak_ghidra", "disassemblers/ofrak_binary_ninja", diff --git a/ofrak-ghidra.yml b/ofrak-ghidra.yml index a4f74ed8c..d7fab4161 100644 --- a/ofrak-ghidra.yml +++ b/ofrak-ghidra.yml @@ -3,9 +3,9 @@ base_image_name: "ghidra-base" image_name: "ghidra" packages_paths: [ - "ofrak_patch_maker", # Patch maker dockerstub includes lots of downloads. Keeping it first helps with docker caching. "ofrak_type", "ofrak_io", + "ofrak_patch_maker", "ofrak_core", "disassemblers/ofrak_ghidra", "frontend", diff --git a/ofrak-tutorial.yml b/ofrak-tutorial.yml index cc95cbb7e..073dbf5c5 100644 --- a/ofrak-tutorial.yml +++ b/ofrak-tutorial.yml @@ -3,9 +3,9 @@ base_image_name: "tutorial-base" image_name: "tutorial" packages_paths: [ - "ofrak_patch_maker", # Patch maker dockerstub includes lots of downloads. Keeping it first helps with docker caching. "ofrak_type", "ofrak_io", + "ofrak_patch_maker", "ofrak_core", "disassemblers/ofrak_ghidra", "examples",