diff --git a/compose/local/dask/Dockerfile b/compose/local/dask/Dockerfile index f6a395ae..6e0f3784 100644 --- a/compose/local/dask/Dockerfile +++ b/compose/local/dask/Dockerfile @@ -27,7 +27,7 @@ RUN freshclam # Workers should have similar reqs as django WORKDIR / COPY ./requirements /requirements -RUN pip install uv==0.4.2 -e git+https://github.com/dadokkio/volatility3.git@d56cd83510e64f9f875ff1dad11d8c8cbf5995f5#egg=volatility3 \ +RUN pip install uv==0.4.2 -e git+https://github.com/dadokkio/volatility3.git@e76d51251be922bb364f10f3fc04f7bfe06c759a#egg=volatility3 \ && uv pip install --no-cache --system -r /requirements/base.txt COPY ./compose/local/dask/prepare.sh /usr/bin/prepare.sh diff --git a/compose/local/django/Dockerfile b/compose/local/django/Dockerfile index 2e2ddf62..d59eb49a 100644 --- a/compose/local/django/Dockerfile +++ b/compose/local/django/Dockerfile @@ -44,7 +44,7 @@ RUN /usr/local/go/bin/go build FROM common-base WORKDIR / COPY ./requirements /requirements -RUN pip install uv==0.4.2 -e git+https://github.com/dadokkio/volatility3.git@d56cd83510e64f9f875ff1dad11d8c8cbf5995f5#egg=volatility3 \ +RUN pip install uv==0.4.2 -e git+https://github.com/dadokkio/volatility3.git@e76d51251be922bb364f10f3fc04f7bfe06c759a#egg=volatility3 \ && uv pip install --no-cache --system -r /requirements/base.txt COPY ./compose/local/__init__.py /src/volatility3/volatility3/framework/constants/__init__.py diff --git a/orochi/utils/elk_migrate.py b/orochi/utils/elk_migrate.py new file mode 100644 index 00000000..5d078c35 --- /dev/null +++ b/orochi/utils/elk_migrate.py @@ -0,0 +1,45 @@ +import os + +from elasticsearch import Elasticsearch +from elasticsearch_dsl import Search + +from orochi.website.defaults import RESULT_STATUS_ERROR, RESULT_STATUS_SUCCESS +from orochi.website.models import Result, Value +from orochi.ya.models import Rule + +es_client = Elasticsearch([os.environ["ELASTICSEARCH_URL"]]) + +rules = Rule.objects.filter(rule__isnull=True) +for rule in rules: + try: + with open(rule.path, "rb") as f: + rule.rule = f.read().decode("utf8", "replace")[:65000] + rule.save() + except Exception as e: + print(e) + + +results = Result.objects.filter(result__in=[RESULT_STATUS_SUCCESS, RESULT_STATUS_ERROR]) +for result in results: + if values := Value.objects.filter(result=result): + continue + s = Search( + using=es_client, index=f"{result.dump.index}_{result.plugin.name.lower()}" + ) + vals = s.execute() + info = [hit.to_dict() for hit in vals if hit.meta.index.split("_")[0] != ".kibana"] + values = [] + for item in info: + tmp = { + k: v + for k, v in item.items() + if k + not in [ + "orochi_createdAt", + "orochi_os", + "orochi_plugin", + "down_path", + ] + } + values.append(Value(result=result, value=tmp)) + Value.objects.bulk_create(values) diff --git a/orochi/utils/elk_migrate.todo b/orochi/utils/elk_migrate.todo new file mode 100644 index 00000000..820d76ab --- /dev/null +++ b/orochi/utils/elk_migrate.todo @@ -0,0 +1,13 @@ +1 - start old elastic service + docker compose --profile migration up -d es01 + +2 - add ELASTICSEARCH_URL environment varible + export ELASTICSEARCH_URL=http://es01:9200 + +3 - install elasticsearch python dependencies + pip install elasticsearch elasticsearch_dsl + +4 - open python terminal + python manage.py shell + +5 - copy and execute code from utils > elk_migrate.py diff --git a/orochi/website/views.py b/orochi/website/views.py index b3cc9646..d9dfc792 100644 --- a/orochi/website/views.py +++ b/orochi/website/views.py @@ -339,13 +339,15 @@ def generate(request): tmp["actions"] = render_to_string( "website/file_download.html", { - "down_path": item["down_path"], + "down_path": item["value"]["down_path"], "misp_configured": misp_configured, - "regipy": Path(f"{item['down_path']}.regipy.json").exists(), + "regipy": Path( + f"{item['value']['down_path']}.regipy.json" + ).exists(), "vt": ( # if empty read is false - open(f"{item['down_path']}.vt.json").read() - if Path(f"{item['down_path']}.vt.json").exists() + open(f"{item['value']['down_path']}.vt.json").read() + if Path(f"{item['value']['down_path']}.vt.json").exists() else None ), },