From 8e4188ef79afcef4705362d33554ea24a0228b65 Mon Sep 17 00:00:00 2001
From: Davide Arcuri <davide.arcuri@leonardo.com>
Date: Wed, 11 Sep 2024 16:38:04 +0200
Subject: [PATCH] fixes, elk migration

---
 compose/local/dask/Dockerfile   |  2 +-
 compose/local/django/Dockerfile |  2 +-
 orochi/utils/elk_migrate.py     | 45 +++++++++++++++++++++++++++++++++
 orochi/utils/elk_migrate.todo   | 13 ++++++++++
 orochi/website/views.py         | 10 +++++---
 5 files changed, 66 insertions(+), 6 deletions(-)
 create mode 100644 orochi/utils/elk_migrate.py
 create mode 100644 orochi/utils/elk_migrate.todo

diff --git a/compose/local/dask/Dockerfile b/compose/local/dask/Dockerfile
index f6a395ae..6e0f3784 100644
--- a/compose/local/dask/Dockerfile
+++ b/compose/local/dask/Dockerfile
@@ -27,7 +27,7 @@ RUN freshclam
 # Workers should have similar reqs as django
 WORKDIR /
 COPY ./requirements /requirements
-RUN pip install uv==0.4.2 -e git+https://github.com/dadokkio/volatility3.git@d56cd83510e64f9f875ff1dad11d8c8cbf5995f5#egg=volatility3 \
+RUN pip install uv==0.4.2 -e git+https://github.com/dadokkio/volatility3.git@e76d51251be922bb364f10f3fc04f7bfe06c759a#egg=volatility3 \
   && uv pip install --no-cache --system -r /requirements/base.txt
 
 COPY ./compose/local/dask/prepare.sh /usr/bin/prepare.sh
diff --git a/compose/local/django/Dockerfile b/compose/local/django/Dockerfile
index 2e2ddf62..d59eb49a 100644
--- a/compose/local/django/Dockerfile
+++ b/compose/local/django/Dockerfile
@@ -44,7 +44,7 @@ RUN /usr/local/go/bin/go build
 FROM common-base
 WORKDIR /
 COPY ./requirements /requirements
-RUN pip install uv==0.4.2 -e git+https://github.com/dadokkio/volatility3.git@d56cd83510e64f9f875ff1dad11d8c8cbf5995f5#egg=volatility3 \
+RUN pip install uv==0.4.2 -e git+https://github.com/dadokkio/volatility3.git@e76d51251be922bb364f10f3fc04f7bfe06c759a#egg=volatility3 \
   && uv pip install --no-cache --system -r /requirements/base.txt
 
 COPY ./compose/local/__init__.py /src/volatility3/volatility3/framework/constants/__init__.py
diff --git a/orochi/utils/elk_migrate.py b/orochi/utils/elk_migrate.py
new file mode 100644
index 00000000..5d078c35
--- /dev/null
+++ b/orochi/utils/elk_migrate.py
@@ -0,0 +1,45 @@
+import os
+
+from elasticsearch import Elasticsearch
+from elasticsearch_dsl import Search
+
+from orochi.website.defaults import RESULT_STATUS_ERROR, RESULT_STATUS_SUCCESS
+from orochi.website.models import Result, Value
+from orochi.ya.models import Rule
+
+es_client = Elasticsearch([os.environ["ELASTICSEARCH_URL"]])
+
+rules = Rule.objects.filter(rule__isnull=True)
+for rule in rules:
+    try:
+        with open(rule.path, "rb") as f:
+            rule.rule = f.read().decode("utf8", "replace")[:65000]
+            rule.save()
+    except Exception as e:
+        print(e)
+
+
+results = Result.objects.filter(result__in=[RESULT_STATUS_SUCCESS, RESULT_STATUS_ERROR])
+for result in results:
+    if values := Value.objects.filter(result=result):
+        continue
+    s = Search(
+        using=es_client, index=f"{result.dump.index}_{result.plugin.name.lower()}"
+    )
+    vals = s.execute()
+    info = [hit.to_dict() for hit in vals if hit.meta.index.split("_")[0] != ".kibana"]
+    values = []
+    for item in info:
+        tmp = {
+            k: v
+            for k, v in item.items()
+            if k
+            not in [
+                "orochi_createdAt",
+                "orochi_os",
+                "orochi_plugin",
+                "down_path",
+            ]
+        }
+        values.append(Value(result=result, value=tmp))
+    Value.objects.bulk_create(values)
diff --git a/orochi/utils/elk_migrate.todo b/orochi/utils/elk_migrate.todo
new file mode 100644
index 00000000..820d76ab
--- /dev/null
+++ b/orochi/utils/elk_migrate.todo
@@ -0,0 +1,13 @@
+1 - start old elastic service
+	docker compose --profile migration up -d es01
+
+2 - add ELASTICSEARCH_URL environment varible
+	export ELASTICSEARCH_URL=http://es01:9200
+
+3 - install elasticsearch python dependencies
+	pip install elasticsearch elasticsearch_dsl
+
+4 - open python terminal
+	python manage.py shell
+
+5 - copy and execute code from utils > elk_migrate.py
diff --git a/orochi/website/views.py b/orochi/website/views.py
index b3cc9646..d9dfc792 100644
--- a/orochi/website/views.py
+++ b/orochi/website/views.py
@@ -339,13 +339,15 @@ def generate(request):
             tmp["actions"] = render_to_string(
                 "website/file_download.html",
                 {
-                    "down_path": item["down_path"],
+                    "down_path": item["value"]["down_path"],
                     "misp_configured": misp_configured,
-                    "regipy": Path(f"{item['down_path']}.regipy.json").exists(),
+                    "regipy": Path(
+                        f"{item['value']['down_path']}.regipy.json"
+                    ).exists(),
                     "vt": (
                         # if empty read is false
-                        open(f"{item['down_path']}.vt.json").read()
-                        if Path(f"{item['down_path']}.vt.json").exists()
+                        open(f"{item['value']['down_path']}.vt.json").read()
+                        if Path(f"{item['value']['down_path']}.vt.json").exists()
                         else None
                     ),
                 },