Skip to content

Commit

Permalink
2.2.0 (#1069)
Browse files Browse the repository at this point in the history
* local files, nginx

* api test code

* vt/maxmind error, update notebook

* python3.12 - go1.22 - ws updates

* #1030 update - #947 method 1

* update libs

* upgrade tree view

* fix download dump error

* keep pip for volatility for shared install path

* improve plugin install, regipy plugins

* fix admin upload plugins

* #1068

* update changelog

---------
  • Loading branch information
dadokkio authored Mar 26, 2024
1 parent d16f127 commit 016bbd4
Show file tree
Hide file tree
Showing 61 changed files with 10,451 additions and 1,974 deletions.
1 change: 1 addition & 0 deletions .env.sample
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
VERSION=latest
25 changes: 24 additions & 1 deletion .envs/.local/.django
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,14 @@ IPYTHONDIR=/app/.ipython
# ------------------------------------------------------------------------------
ALLOWED_HOSTS=*

# Smtp
# ------------------------------------------------------------------------------
EMAIL_HOST=mailpit
EMAIL_PORT=1025
EMAIL_USE_TLS=False
EMAIL_HOST_USER=
EMAIL_HOST_PASSWORD=

# Redis
# ------------------------------------------------------------------------------
REDIS_URL=redis://redis:6379/0
Expand All @@ -29,6 +37,21 @@ AWESOME_PATH=https://raw.githubusercontent.com/InQuest/awesome-yara/master/READM
LOCAL_YARA_PATH=/yara
DEFAULT_YARA_RULE_PATH=/yara/default.yara
VOLATILITY_SYMBOL_DOWNLOAD_PATH=https://downloads.volatilityfoundation.org/volatility3/symbols
LOCAL_UPLOAD_PATH=/uploads
# $ from regipy.plugins.plugin import PLUGINS
# $ [plugin.NAME for plugin in PLUGINS]
# ['amcache', 'terminal_services_history', 'bootkey', 'last_logon_plugin', 'winscp_saved_sessions',
# 'ras_tracing', 'active_control_set', 'timezone_data', 'safeboot_configuration', 'uac_plugin',
# 'user_assist', 'word_wheel_query', 'print_demon_plugin', 'shimcache', 'usbstor_plugin', 'wdigest',
# 'installed_programs_ntuser', 'software_classes_installer', 'installed_programs_software', 'background_activity_moderator',
# 'ntuser_classes_installer', 'typed_urls', 'usrclass_shellbag_plugin', 'network_data', 'image_file_execution_options',
# 'domain_sid', 'routes', 'boot_entry_list', 'ntuser_shellbag_plugin', 'typed_paths', 'services', 'local_sid',
# 'host_domain_name', 'profilelist_plugin', 'winrar_plugin', 'ntuser_persistence', 'computer_name', 'network_drives_plugin', 'software_plugin']
REGIPY_PLUGINS=computer_name,host_domain_name,network_drives_plugin

# Debug
# ------------------------------------------------------------------------------
DEBUG_LEVEL=WARNING

# Ldap
# ------------------------------------------------------------------------------
Expand All @@ -47,4 +70,4 @@ no_proxy=

# CSRF_TRUSTED_ORIGINS
# ------------------------------------------------------------------------------
CSRF_TRUSTED_ORIGINS=
CSRF_TRUSTED_ORIGINS=https://orochi.dev,https://localhost
7 changes: 6 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -271,7 +271,12 @@ compose/local/clamav/freshclam.conf

# History
.history

# Temporary test files
orochi/utils/test.py
docker-compose__tmp.yml

# Maxmind
compose/local/maxmind/GeoLite2-ASN.mmdb
compose/local/maxmind/GeoLite2-City.mmdb
compose/local/maxmind/GeoLite2-Country.mmdb
compose/local/maxmind/GeoLite2-Country.mmdb
2 changes: 1 addition & 1 deletion .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ repos:
- id: check-yaml

- repo: https://github.com/psf/black
rev: 24.1.1
rev: 24.3.0
hooks:
- id: black

Expand Down
17 changes: 15 additions & 2 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,10 +1,23 @@
## Changelog


<details open>
<summary><b>OROCHI 2.2.0</b></summary>
<summary><b>OROCHI 2.2.0 [2024/03/26]</b></summary>

* Upload ntoskrnl.exe and generate symbol [[#1020](https://github.com/LDO-CERT/orochi/issues/1020)]
* evaluate possibility to switch from daphne to uvicorn for asgi [[#982](https://github.com/LDO-CERT/orochi/issues/982)]
* Improve path flexibility for local import [[#451](https://github.com/LDO-CERT/orochi/issues/451)]
* uv for installing requirements [[#1030](https://github.com/LDO-CERT/orochi/issues/1030)]
* Read only users for educational. [[#947](https://github.com/LDO-CERT/orochi/issues/947)]
* Add use case example with API. [[#248](https://github.com/LDO-CERT/orochi/issues/248)]
* put custom plugins under volatility3 /plugins/ [[#1068](https://github.com/LDO-CERT/orochi/issues/1068)]
* Improve tree rendered plugins
</details>

<details>
<summary><b>OROCHI 2.1.1 [2024/02/13]</b></summary>

* ADD more info on foreign addr in netstat [[#494](https://github.com/LDO-CERT/orochi/issues/494)]
* Expand/Collapse folders [[#1006](https://github.com/LDO-CERT/orochi/issues/1006)]
</details>

<details open>
Expand Down
10 changes: 8 additions & 2 deletions compose/local/dask/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
FROM daskdev/dask:2024.1.1-py3.11
FROM daskdev/dask:2024.3.1-py3.12
ENV DEBIAN_FRONTEND noninteractive

ARG local_folder=/uploads

RUN apt-get --allow-releaseinfo-change update \
# dependencies for building Python packages
&& apt-get install -y --no-install-recommends build-essential \
Expand Down Expand Up @@ -45,14 +47,18 @@ RUN python setup.py build \
# Workers should have similar reqs as django
WORKDIR /
COPY ./requirements /requirements
RUN pip install -r /requirements/base.txt --ignore-installed ruamel.yaml
RUN pip install uv==0.1.21 -e git+https://github.com/volatilityfoundation/volatility3.git@55dd39f2ba60ffdd2126b7ea011940f0df42815a#egg=volatility3 \
&& uv pip install --no-cache --system -r /requirements/base.txt

COPY ./compose/local/dask/prepare.sh /usr/bin/prepare.sh
RUN chmod +x /usr/bin/prepare.sh

COPY ./compose/local/dask/daskworkerinit.py /opt/daskworkerinit.py
COPY ./compose/local/__init__.py /src/volatility3/volatility3/framework/constants/__init__.py

RUN mkdir -p $local_folder /maxmind
COPY ./compose/local/maxmind /maxmind

WORKDIR /app
COPY . .
ENV PYTHONPATH "${PYTHONPATH}:/app"
Expand Down
15 changes: 8 additions & 7 deletions compose/local/django/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
FROM python:3.11-slim-buster as common-base
FROM python:3.12-slim-bookworm as common-base

ENV DJANGO_SETTINGS_MODULE config.settings.local
ENV PYTHONUNBUFFERED 1
ENV PYTHONDONTWRITEBYTECODE 1

ARG local_folder=/uploads

RUN apt-get update \
# dependencies for building Python packages
&& apt-get install -y --no-install-recommends build-essential \
Expand Down Expand Up @@ -54,22 +56,21 @@ RUN python setup.py build

FROM base-builder as go-builder
WORKDIR /
RUN curl https://dl.google.com/go/go1.15.2.linux-amd64.tar.gz --output go1.15.2.linux-amd64.tar.gz \
&& tar -C /usr/local -xzf go1.15.2.linux-amd64.tar.gz
RUN curl https://dl.google.com/go/go1.15.2.linux-amd64.tar.gz --output go1.15.2.linux-amd64.tar.gz \
&& tar -C /usr/local -xzf go1.15.2.linux-amd64.tar.gz
RUN curl https://dl.google.com/go/go1.22.0.linux-amd64.tar.gz --output go1.22.0.linux-amd64.tar.gz \
&& tar -C /usr/local -xzf go1.22.0.linux-amd64.tar.gz
RUN git clone https://github.com/volatilityfoundation/dwarf2json.git
WORKDIR /dwarf2json
RUN /usr/local/go/bin/go build

FROM common-base
WORKDIR /
COPY ./requirements /requirements
RUN pip install -r /requirements/local.txt --ignore-installed ruamel.yaml
RUN pip install uv==0.1.21 -e git+https://github.com/volatilityfoundation/volatility3.git@55dd39f2ba60ffdd2126b7ea011940f0df42815a#egg=volatility3 \
&& uv pip install --no-cache --system -r /requirements/base.txt

COPY ./compose/local/__init__.py /src/volatility3/volatility3/framework/constants/__init__.py

RUN mkdir /maxmind
RUN mkdir -p $local_folder /maxmind
COPY ./compose/local/maxmind /maxmind

WORKDIR /app
Expand Down
2 changes: 1 addition & 1 deletion compose/local/django/start
Original file line number Diff line number Diff line change
Expand Up @@ -7,4 +7,4 @@ set -o nounset
python manage.py makemigrations
python manage.py migrate
python manage.py collectstatic --skip-checks --noinput --clear
daphne -b 0.0.0.0 -p 8000 config.asgi:application
gunicorn config.wsgi:application --workers=2 --timeout 1800 -b :8000
19 changes: 19 additions & 0 deletions compose/local/nginx/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
FROM alpine:latest as alpine
ARG DOMAIN_NAME=orochi.dev
ARG DAYS_VALID=365

RUN apk add --no-cache openssl
RUN echo "Creating self-signed certificate valid for ${DAYS_VALID} days for domain ${DOMAIN_NAME}" && \
openssl \
req -x509 \
-nodes \
-subj "/CN=${DOMAIN_NAME}" \
-addext "subjectAltName=DNS:${DOMAIN_NAME}" \
-days ${DAYS_VALID} \
-newkey rsa:2048 -keyout /tmp/self-signed.key \
-out /tmp/self-signed.crt

# Prepare an NGINX-based image with the certificate created above.
FROM nginx:latest as nginx
COPY --from=alpine /tmp/self-signed.key /etc/ssl/private
COPY --from=alpine /tmp/self-signed.crt /etc/ssl/certs
57 changes: 57 additions & 0 deletions compose/local/nginx/conf/default.conf
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
upstream django_wsgi {
server django_wsgi:8000;
}

upstream django_asgi {
server django_asgi:9000;
}

map $http_upgrade $connection_upgrade {
default upgrade;
'' close;
}

server {
listen 80;
listen [::]:80;
server_name _;
return 301 https://$host$request_uri;
}

server {
listen 443 ssl;
listen [::]:443 ssl;

proxy_connect_timeout 50000;
proxy_read_timeout 50000;
proxy_send_timeout 50000;

server_name orochi.dev;

location = /favicon.ico { access_log off; log_not_found off; }

ssl_certificate /etc/ssl/certs/self-signed.crt;
ssl_certificate_key /etc/ssl/private/self-signed.key;

location / {
proxy_pass http://django_wsgi;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Host $host;
proxy_redirect off;

client_max_body_size 1000M;
}

location /ws/notify/ {
proxy_pass http://django_asgi;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $connection_upgrade;

proxy_redirect off;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Host $server_name;
}
}
10 changes: 6 additions & 4 deletions config/routing.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,15 @@
import orochi.website.routing
from django.core.asgi import get_asgi_application
from channels.auth import AuthMiddlewareStack
from channels.routing import ProtocolTypeRouter, URLRouter
from channels.security.websocket import AllowedHostsOriginValidator
from django.core.asgi import get_asgi_application

import orochi.website.routing

application = ProtocolTypeRouter(
{
"http": get_asgi_application(),
"websocket": AuthMiddlewareStack(
URLRouter(orochi.website.routing.websocket_urlpatterns)
"websocket": AllowedHostsOriginValidator(
AuthMiddlewareStack(URLRouter(orochi.website.routing.websocket_urlpatterns))
),
}
)
15 changes: 10 additions & 5 deletions config/settings/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -228,6 +228,7 @@

# LOGGING
# ------------------------------------------------------------------------------
DEBUG_LEVEL = env("DEBUG_LEVEL", default="WARNING")
LOGGING = {
"version": 1,
"disable_existing_loggers": False,
Expand All @@ -239,14 +240,14 @@
},
"handlers": {
"console": {
"level": "INFO",
"level": DEBUG_LEVEL,
"class": "logging.StreamHandler",
"formatter": "verbose",
}
},
"root": {"level": "INFO", "handlers": ["console"]},
"root": {"level": DEBUG_LEVEL, "handlers": ["console"]},
"loggers": {
"distributed": {"level": "INFO", "handlers": ["console"]},
"distributed": {"level": DEBUG_LEVEL, "handlers": ["console"]},
},
}

Expand Down Expand Up @@ -275,7 +276,7 @@
ASGI_APPLICATION = "config.routing.application"
CHANNEL_LAYERS = {
"default": {
"BACKEND": "channels_redis.pubsub.RedisPubSubChannelLayer",
"BACKEND": "channels_redis.core.RedisChannelLayer",
"CONFIG": {
"hosts": [(env("REDIS_SERVER"), env("REDIS_PORT"))],
},
Expand Down Expand Up @@ -329,8 +330,12 @@
RULES_ANALYSIS_INDEX = "rules_analysis"
# local path of volatility folder
VOLATILITY_SYMBOL_PATH = "/src/volatility3/volatility3/symbols"
VOLATILITY_PLUGIN_PATH = "/src/volatility3/volatility3/framework/plugins/custom"
VOLATILITY_PLUGIN_PATH = "/src/volatility3/volatility3/plugins/custom"
# local path of dwarg2json executable
DWARF2JSON = "/dwarf2json/./dwarf2json"
# online path of volatility symbols
VOLATILITY_SYMBOL_DOWNLOAD_PATH = env("VOLATILITY_SYMBOL_DOWNLOAD_PATH")
# path of a remote folder with already uploaded files
LOCAL_UPLOAD_PATH = env("LOCAL_UPLOAD_PATH")
# Regipy plugins
REGIPY_PLUGINS = env.list("REGIPY_PLUGINS")
2 changes: 1 addition & 1 deletion config/settings/local.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@

# EMAIL
# ------------------------------------------------------------------------------
EMAIL_HOST = env("EMAIL_HOST", default="mailhog")
EMAIL_HOST = env("EMAIL_HOST", default="mailpit")
EMAIL_PORT = 1025
EMAIL_USE_TLS = env("EMAIL_USE_TLS", default=True)
EMAIL_HOST_USER = env("EMAIL_HOST_USER", default=None)
Expand Down
6 changes: 3 additions & 3 deletions docker-compose-swarm.yml
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ services:
volumes:
- media_path:/media
- symbols_path:/src/volatility3/volatility3/symbols
- plugin_path:/src/volatility3/volatility3/framework/plugins/custom
- plugin_path:/src/volatility3/volatility3/plugins/custom
- yara_path:/yara
- cache_path:/root/.cache/volatility3
- ./compose/local/__init__.py:/src/volatility3/volatility3/framework/constants/__init__.py
Expand Down Expand Up @@ -178,7 +178,7 @@ services:
volumes:
- media_path:/media
- symbols_path:/src/volatility3/volatility3/symbols
- plugin_path:/src/volatility3/volatility3/framework/plugins/custom
- plugin_path:/src/volatility3/volatility3/plugins/custom
- yara_path:/yara
- cache_path:/root/.cache/volatility3

Expand Down Expand Up @@ -213,7 +213,7 @@ services:
volumes:
- media_path:/media
- symbols_path:/src/volatility3/volatility3/symbols
- plugin_path:/src/volatility3/volatility3/framework/plugins/custom
- plugin_path:/src/volatility3/volatility3/plugins/custom
- yara_path:/yara
- cache_path:/root/.cache/volatility3
- ./compose/local/__init__.py:/src/volatility3/volatility3/framework/constants/__init__.py
Expand Down
Loading

0 comments on commit 016bbd4

Please sign in to comment.