From 7fbf2c7ac7d38069f66d8150510799fe7725301e Mon Sep 17 00:00:00 2001
From: fangao <1227796539@qq.com>
Date: Sun, 10 Nov 2024 21:55:00 +0800
Subject: [PATCH] upload data loader testing

---
 demo_data.jsonl => data/demo_data.jsonl       |  0
 .../function_calling_demo.jsonl               |  0
 data/test_data_loading.py                     | 39 +++++++++++++++++++
 3 files changed, 39 insertions(+)
 rename demo_data.jsonl => data/demo_data.jsonl (100%)
 rename function_calling_demo.jsonl => data/function_calling_demo.jsonl (100%)
 create mode 100644 data/test_data_loading.py

diff --git a/demo_data.jsonl b/data/demo_data.jsonl
similarity index 100%
rename from demo_data.jsonl
rename to data/demo_data.jsonl
diff --git a/function_calling_demo.jsonl b/data/function_calling_demo.jsonl
similarity index 100%
rename from function_calling_demo.jsonl
rename to data/function_calling_demo.jsonl
diff --git a/data/test_data_loading.py b/data/test_data_loading.py
new file mode 100644
index 0000000..594537c
--- /dev/null
+++ b/data/test_data_loading.py
@@ -0,0 +1,39 @@
+import pytest
+import json
+
+
+@pytest.fixture
+def load_data():
+    with open("function_calling_demo.jsonl", "r", encoding="utf8") as f:
+        data_list = f.readlines()
+    return data_list
+
+
+def test_data_loading(load_data):
+    # Verify data was loaded correctly
+    assert load_data is not None, "Data should be loaded"
+    assert len(load_data) > 0, "Data list should not be empty"
+
+    # Verify each line is a valid JSON with required fields
+    for line in load_data:
+        try:
+            data = json.loads(line)
+            assert isinstance(
+                data, dict
+            ), "Each line should be a JSON object (dictionary)"
+
+            # Check for "conversations" key
+            assert (
+                "conversations" in data
+            ), "'conversations' key is missing in the JSON object"
+            assert isinstance(
+                data["conversations"], list
+            ), "'conversations' should be a list"
+
+            # Check for 'user' and 'assistant' roles within "conversations"
+            roles = {conv["role"] for conv in data["conversations"] if "role" in conv}
+            assert "user" in roles, "Role 'user' is missing in conversations"
+            assert "assistant" in roles, "Role 'assistant' is missing in conversations"
+
+        except json.JSONDecodeError:
+            pytest.fail("Each line in the data file should be valid JSON")