Skip to content

Commit

Permalink
integration modifications
Browse files Browse the repository at this point in the history
  • Loading branch information
@KunalTiwary
KunalTiwary committed Jan 10, 2024
1 parent 475e717 commit 0880781
Show file tree
Hide file tree
Showing 10 changed files with 136 additions and 129 deletions.
29 changes: 29 additions & 0 deletions backend/dataset/migrations/0047_alter_instruction_meta_info_language.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
# Generated by Django 3.2.14 on 2024-01-10 07:16

from django.db import migrations, models


class Migration(migrations.Migration):
dependencies = [
("dataset", "0046_auto_20240104_0617"),
]

operations = [
migrations.AlterField(
model_name="instruction",
name="meta_info_language",
field=models.CharField(
blank=True,
choices=[
("1", "English(Any script)"),
("2", "Indic(Indic script)"),
("3", "Indic(Latin script)"),
("4", "Indic/English(Latin script)"),
],
help_text="Language of the instruction",
max_length=20,
null=True,
verbose_name="Meta Info Language",
),
),
]
11 changes: 9 additions & 2 deletions backend/dataset/models.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,13 @@
("Urdu", "Urdu"),
]

LANGUAGE_CHOICES_INSTRUCTIONS = (
("1", "English(Any script)"),
("2", "Indic(Indic script)"),
("3", "Indic(Latin script)"),
("4", "Indic/English(Latin script)"),
)

LLM_CHOICES = (("GPT3.5", "GPT3.5"), ("GPT4", "GPT4"), ("LLAMA2", "LLAMA2"))


Expand Down Expand Up @@ -684,7 +691,7 @@ class Instruction(DatasetBase):
)
meta_info_language = models.CharField(
max_length=20,
choices=LANGUAGE_CHOICES,
choices=LANGUAGE_CHOICES_INSTRUCTIONS,
verbose_name="Meta Info Language",
null=True,
blank=True,
Expand All @@ -695,7 +702,7 @@ class Instruction(DatasetBase):
hint = models.TextField(verbose_name="Hint")

def __str__(self):
return f"{self.id} - {self.instruction}"
return f"{self.id} - {self.instruction_data}"


class Interaction(DatasetBase):
Expand Down
24 changes: 24 additions & 0 deletions backend/projects/migrations/0054_alter_project_project_type.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
# Generated by Django 3.2.14 on 2024-01-10 03:10

from django.db import migrations, models


class Migration(migrations.Migration):
dependencies = [
("projects", "0053_alter_project_project_type"),
]

operations = [
migrations.AlterField(
model_name="project",
name="project_type",
field=models.CharField(
choices=[
("ModelOutputEvaluation", "ModelOutputEvaluation"),
("InstructionDrivenChat", "InstructionDrivenChat"),
],
help_text="Project Type indicating the annotation task",
max_length=100,
),
),
]
2 changes: 2 additions & 0 deletions backend/projects/project_registry.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@ Chat:
class: Interaction
save_type: new_record
fields:
copy_from_input:
id: instruction_id
annotations:
- interactions_json
- no_of_turns
Expand Down
3 changes: 2 additions & 1 deletion backend/projects/tasks.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -190,7 +190,8 @@ def create_tasks_from_dataitems(items, project):
data = dataset_models.DatasetBase.objects.get(pk=data_id)

# Remove data id because it's not needed in task.data
del item["id"]
if "id" in item:
del item["id"]
task = Task(data=item, project_id=project, input_data=data)
"""
if is_translation_project or dataset_type1 == "TranslationPair":
Expand Down
98 changes: 47 additions & 51 deletions backend/tasks/views.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1344,12 +1344,11 @@ def partial_update(self, request, pk=None):
annotation_obj.task.project_id.project_type
== "InstructionDrivenChat"
):
if len(request.data["result"]) > 0:
annotation_obj.result = get_llm_output(
request.data["result"],
annotation_obj.task,
annotation_obj,
)
annotation_obj.result = get_llm_output(
request.data["result"],
annotation_obj.task,
annotation_obj,
)
else:
annotation_obj.result = request.data["result"]
if "annotation_notes" in dict(request.data):
Expand Down Expand Up @@ -1407,12 +1406,11 @@ def partial_update(self, request, pk=None):
annotation_obj.task.project_id.project_type
== "InstructionDrivenChat"
):
if len(request.data["result"]) > 0:
request.data["result"] = get_llm_output(
request.data["result"],
annotation_obj.task,
annotation_obj,
)
request.data["result"] = get_llm_output(
request.data["result"],
annotation_obj.task,
annotation_obj,
)
annotation_response = super().partial_update(request)
annotation_id = annotation_response.data["id"]
annotation = Annotation.objects.get(pk=annotation_id)
Expand Down Expand Up @@ -1473,12 +1471,11 @@ def partial_update(self, request, pk=None):
annotation_obj.task.project_id.project_type
== "InstructionDrivenChat"
):
if len(request.data["result"]) > 0:
annotation_obj.result = get_llm_output(
request.data["result"],
annotation_obj.task,
annotation_obj,
)
annotation_obj.result = get_llm_output(
request.data["result"],
annotation_obj.task,
annotation_obj,
)
else:
annotation_obj.result = request.data["result"]
if "review_notes" in dict(request.data):
Expand Down Expand Up @@ -1575,12 +1572,11 @@ def partial_update(self, request, pk=None):
annotation_obj.task.project_id.project_type
== "InstructionDrivenChat"
):
if len(request.data["result"]) > 0:
request.data["result"] = get_llm_output(
request.data["result"],
annotation_obj.task,
annotation_obj,
)
request.data["result"] = get_llm_output(
request.data["result"],
annotation_obj.task,
annotation_obj,
)
annotation_response = super().partial_update(request)
annotation_id = annotation_response.data["id"]
annotation = Annotation.objects.get(pk=annotation_id)
Expand Down Expand Up @@ -1668,12 +1664,11 @@ def partial_update(self, request, pk=None):
annotation_obj.task.project_id.project_type
== "InstructionDrivenChat"
):
if len(request.data["result"]) > 0:
annotation_obj.result = get_llm_output(
request.data["result"],
annotation_obj.task,
annotation_obj,
)
annotation_obj.result = get_llm_output(
request.data["result"],
annotation_obj.task,
annotation_obj,
)
else:
annotation_obj.result = request.data["result"]
if "supercheck_notes" in dict(request.data):
Expand Down Expand Up @@ -1761,12 +1756,11 @@ def partial_update(self, request, pk=None):
annotation_obj.task.project_id.project_type
== "InstructionDrivenChat"
):
if len(request.data["result"]) > 0:
request.data["result"] = get_llm_output(
request.data["result"],
annotation_obj.task,
annotation_obj,
)
request.data["result"] = get_llm_output(
request.data["result"],
annotation_obj.task,
annotation_obj,
)
annotation_response = super().partial_update(request)
annotation_id = annotation_response.data["id"]
annotation = Annotation.objects.get(pk=annotation_id)
Expand Down Expand Up @@ -2080,26 +2074,28 @@ def calculate_bleu_score(self, request):
)


def get_llm_output(prompt, task, annotation):
##############
user_language = "hin"
def get_llm_output(prompt, task, annotation, complete_checking):
# CHECKS
intent = task["data"]["meta_info_intent"]
domain = task["data"]["meta_info_domain"]
lang_type = task["data"]["meta_info_language"]
if intent and domain:
intent_check, domain_check, reason = evaluate_prompt_alignment(
prompt, domain, intent
)
if user_language and lang_type:
lang_check = prompt_lang_check(user_language, prompt, lang_type)
dup_check, message = duplicate_check(annotation, prompt)
ann_result = json.loads(annotation.result)
if len(ann_result) == 0 and complete_checking:
if intent and domain:
intent_check, domain_check, reason = evaluate_prompt_alignment(
prompt, domain, intent
)
if lang_type:
lang_check = prompt_lang_check(prompt, lang_type)
if len(ann_result) >= 0:
dup_check, message = duplicate_check(ann_result, prompt)

# GET MODEL OUTPUT
##############
history = json.loads(annotation.result)
history = ann_result
model = task["data"]["meta_info_model"]
output = get_model_output(prompt, history, model)
existing_result = json.loads(annotation.result)
existing_result.append({"prompt": prompt, "output": output})
return existing_result
get_model_output(
"You are a very kind and helpful assistant!", prompt, history, model
)
ann_result.append({"prompt": prompt, "output": output})
return ann_result
57 changes: 0 additions & 57 deletions backend/users/migrations/0031_auto_20231227_1055.py
View file

This file was deleted.

13 changes: 6 additions & 7 deletions backend/utils/llm_checks.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ def get_lid(text):
return languages[1]


def prompt_lang_check(user_lang, prompt, lang_type):
def prompt_lang_check(prompt, lang_type):
"""
Checks if the given prompt matches the specified language and script criteria.
Expand Down Expand Up @@ -142,9 +142,8 @@ def evaluate_prompt_alignment(prompt, target_domain, target_intent):
return intent, domain, resp_dict["reason"]


def duplicate_check(annotation, prompt):
existingData = json.loads(annotation.result)
if prompt in existingData:
return False, "Duplicate prompt"
else:
return True, "Original prompt"
def duplicate_check(ann_result, prompt):
for r in ann_result:
if r["prompt"] == prompt:
return False
return True
Loading

0 comments on commit 0880781

Please sign in to comment.