From d559040da5075ae66dae856fe745fcb260929607 Mon Sep 17 00:00:00 2001
From: Arid Hasan <arid.hasan.h@gmail.com>
Date: Mon, 9 Sep 2024 16:00:57 -0300
Subject: [PATCH] add mistral asset NE

---
 .../ne/QA/MultiNativQA_Mistral_7b_ZeroShot.py | 61 +++++++++++++++++++
 1 file changed, 61 insertions(+)
 create mode 100644 assets/ne/QA/MultiNativQA_Mistral_7b_ZeroShot.py

diff --git a/assets/ne/QA/MultiNativQA_Mistral_7b_ZeroShot.py b/assets/ne/QA/MultiNativQA_Mistral_7b_ZeroShot.py
new file mode 100644
index 00000000..c43560fe
--- /dev/null
+++ b/assets/ne/QA/MultiNativQA_Mistral_7b_ZeroShot.py
@@ -0,0 +1,61 @@
+import json
+
+from llmebench.datasets import MultiNativQADataset
+from llmebench.models import AzureModel
+from llmebench.tasks import MultiNativQATask
+
+
+def metadata():
+    return {
+        "author": "Arabic Language Technologies, QCRI, HBKU",
+        "model": "Mistral 7b",
+        "description": "Deployed on Azure.",
+        "scores": {},
+    }
+
+
+def config():
+    return {
+        "dataset": MultiNativQADataset,
+        "task": MultiNativQATask,
+        "model": AzureModel,
+        "general_args": {"test_split": "nepali_np"},
+    }
+
+
+def prompt(input_sample):
+
+    # Define the question prompt
+    question_prompt = f"""
+    Please use your expertise to answer the following Nepali question. Answer in Nepali and rate your confidence level from 1 to 10.
+    Provide your response in the following JSON format: {{"answer": "your answer", "score": your confidence score}}. 
+    Please provide JSON output only. No additional text. Answer should be limited to less or equal to {input_sample['length']} words.
+
+    Question: {input_sample['question']}
+    """
+
+    # Define the assistant prompt
+    assistant_prompt = """
+    You are a Nepali AI assistant specialized in providing detailed and accurate answers across various fields.
+    Your task is to deliver clear, concise, and relevant information. 
+    """
+
+    return [
+        # {
+        #     "role": "assistant",
+        #     "content": assistant_prompt,
+        # },
+        {
+            "role": "user",
+            "content": question_prompt,
+        },
+    ]
+
+
+def post_process(response):
+    data = response["output"]
+    if "\n\n" in data:
+        data = data.split("\n\n")[0]
+    response = json.loads(data)
+    answer = response["answer"]
+    return answer