fixes, elk migration

compose/local/dask/Dockerfile

@@ -27,7 +27,7 @@ RUN freshclam
 # Workers should have similar reqs as django
 WORKDIR /
 COPY ./requirements /requirements
-RUN pip install uv==0.4.2 -e git+https://github.com/dadokkio/volatility3.git@d56cd83510e64f9f875ff1dad11d8c8cbf5995f5#egg=volatility3 \
+RUN pip install uv==0.4.2 -e git+https://github.com/dadokkio/volatility3.git@e76d51251be922bb364f10f3fc04f7bfe06c759a#egg=volatility3 \
  && uv pip install --no-cache --system -r /requirements/base.txt
 
 COPY ./compose/local/dask/prepare.sh /usr/bin/prepare.sh

compose/local/django/Dockerfile

@@ -44,7 +44,7 @@ RUN /usr/local/go/bin/go build
 FROM common-base
 WORKDIR /
 COPY ./requirements /requirements
-RUN pip install uv==0.4.2 -e git+https://github.com/dadokkio/volatility3.git@d56cd83510e64f9f875ff1dad11d8c8cbf5995f5#egg=volatility3 \
+RUN pip install uv==0.4.2 -e git+https://github.com/dadokkio/volatility3.git@e76d51251be922bb364f10f3fc04f7bfe06c759a#egg=volatility3 \
  && uv pip install --no-cache --system -r /requirements/base.txt
 
 COPY ./compose/local/__init__.py /src/volatility3/volatility3/framework/constants/__init__.py

orochi/utils/elk_migrate.py

@@ -0,0 +1,45 @@
+import os
+
+from elasticsearch import Elasticsearch
+from elasticsearch_dsl import Search
+
+from orochi.website.defaults import RESULT_STATUS_ERROR, RESULT_STATUS_SUCCESS
+from orochi.website.models import Result, Value
+from orochi.ya.models import Rule
+
+es_client = Elasticsearch([os.environ["ELASTICSEARCH_URL"]])
+
+rules = Rule.objects.filter(rule__isnull=True)
+for rule in rules:
+ try:
+ with open(rule.path, "rb") as f:
+ rule.rule = f.read().decode("utf8", "replace")[:65000]
+ rule.save()
+ except Exception as e:
+ print(e)
+
+
+results = Result.objects.filter(result__in=[RESULT_STATUS_SUCCESS, RESULT_STATUS_ERROR])
+for result in results:
+ if values := Value.objects.filter(result=result):
+ continue
+ s = Search(
+ using=es_client, index=f"{result.dump.index}_{result.plugin.name.lower()}"
+ )
+ vals = s.execute()
+ info = [hit.to_dict() for hit in vals if hit.meta.index.split("_")[0] != ".kibana"]
+ values = []
+ for item in info:
+ tmp = {
+ k: v
+ for k, v in item.items()
+ if k
+ not in [
+ "orochi_createdAt",
+ "orochi_os",
+ "orochi_plugin",
+ "down_path",
+ ]
+ }
+ values.append(Value(result=result, value=tmp))
+ Value.objects.bulk_create(values)

orochi/utils/elk_migrate.todo

@@ -0,0 +1,13 @@
+1 - start old elastic service
+ docker compose --profile migration up -d es01
+
+2 - add ELASTICSEARCH_URL environment varible
+ export ELASTICSEARCH_URL=http://es01:9200
+
+3 - install elasticsearch python dependencies
+ pip install elasticsearch elasticsearch_dsl
+
+4 - open python terminal
+ python manage.py shell
+
+5 - copy and execute code from utils > elk_migrate.py

orochi/website/views.py

@@ -339,13 +339,15 @@ def generate(request):
  tmp["actions"] = render_to_string(
  "website/file_download.html",
  {
- "down_path": item["down_path"],
+ "down_path": item["value"]["down_path"],
  "misp_configured": misp_configured,
- "regipy": Path(f"{item['down_path']}.regipy.json").exists(),
+ "regipy": Path(
+ f"{item['value']['down_path']}.regipy.json"
+ ).exists(),
  "vt": (
  # if empty read is false
- open(f"{item['down_path']}.vt.json").read()
- if Path(f"{item['down_path']}.vt.json").exists()
+ open(f"{item['value']['down_path']}.vt.json").read()
+ if Path(f"{item['value']['down_path']}.vt.json").exists()
  else None
  ),
  },