diff --git a/.changeset/tidy-bulldogs-accept.md b/.changeset/tidy-bulldogs-accept.md
new file mode 100644
index 00000000..d13c3de2
--- /dev/null
+++ b/.changeset/tidy-bulldogs-accept.md
@@ -0,0 +1,8 @@
+---
+"@empiricalrun/scorer": minor
+"@empiricalrun/types": minor
+"@empiricalrun/cli": minor
+"web": minor
+---
+
+feat: add support for merging inputs and add multi-turn chat example
diff --git a/apps/web/app/page.tsx b/apps/web/app/page.tsx
index 12f0b9d7..132db468 100644
--- a/apps/web/app/page.tsx
+++ b/apps/web/app/page.tsx
@@ -44,10 +44,6 @@ export default function Page(): JSX.Element {
     () => (dataset?.samples || [])?.map((s) => s.id),
     [dataset],
   );
-  const datasetInputNames = useMemo(
-    () => Object.keys(dataset?.samples?.[0]?.inputs || {}),
-    [dataset],
-  );
   const runColumnHeaders = useMemo(
     () => tableHeaders.filter((h) => h.type == "completion"),
     [tableHeaders],
@@ -202,7 +198,6 @@ export default function Page(): JSX.Element {
                   <div className="flex flex-1 min-w-[500px] overflow-hidden">
                     <SampleCard
                       sample={inputSample!}
-                      inputTabs={datasetInputNames}
                       onSampleAdd={(sample) => addDatasetSample(sample)}
                       onSampleInputUpdate={updateDatasetSampleInput}
                       onSampleRemove={(sample) => {
diff --git a/apps/web/components/json-as-tab.tsx b/apps/web/components/json-as-tab.tsx
index 4e7208e4..fbc8898d 100644
--- a/apps/web/components/json-as-tab.tsx
+++ b/apps/web/components/json-as-tab.tsx
@@ -49,12 +49,6 @@ export function JsonAsTab({
   const { activeTab: remoteActiveTab, onChangeTab: remoteOnChangeTab } =
     useSyncedTabs(tabs, storeKey);
   const [activeTab, setActiveTab] = useState<string | undefined>();
-  const activeTabValue = useMemo(() => {
-    if (activeTab && data) {
-      return data[activeTab];
-    }
-    return undefined;
-  }, [activeTab, data]);
 
   useEffect(() => {
     if (remoteActiveTab && data[remoteActiveTab]) {
@@ -67,6 +61,8 @@ export function JsonAsTab({
   const onChangeTab = useCallback(
     (tab: string) => {
       setActiveTab(tab);
+      console.log(JSON.stringify(data));
+      console.log("setting remote active tab", tab);
       remoteOnChangeTab(tab);
     },
     [remoteOnChangeTab],
@@ -76,7 +72,7 @@ export function JsonAsTab({
     <>
       <div className="flex flex-row space-x-2 justify-end">
         <>
-          {activeTabValue && showExpandOption && (
+          {showExpandOption && (
             <Sheet>
               <SheetTrigger asChild>
                 <Button
@@ -90,15 +86,11 @@ export function JsonAsTab({
               </SheetTrigger>
               <SheetContent className="w-[700px] sm:w-[540px]">
                 <SheetHeader>
-                  <SheetTitle>{activeTab}</SheetTitle>
+                  <SheetTitle>Inputs</SheetTitle>
                 </SheetHeader>
                 <div className="py-4 h-full">
                   <CodeViewer
-                    value={
-                      typeof activeTabValue === "string"
-                        ? activeTabValue
-                        : JSON.stringify(activeTabValue, null, 2)
-                    }
+                    value={JSON.stringify(data, null, 2)}
                     readOnly // expand sheet is readonly
                     scrollable
                     language="json"
diff --git a/apps/web/components/sample-card.tsx b/apps/web/components/sample-card.tsx
index 10df8ced..f6ed66b5 100644
--- a/apps/web/components/sample-card.tsx
+++ b/apps/web/components/sample-card.tsx
@@ -5,7 +5,6 @@ import { JsonAsTab } from "./json-as-tab";
 
 export default function SampleCard({
   sample,
-  inputTabs,
   onSampleRemove,
   onSampleAdd,
   onSampleInputUpdate,
@@ -13,7 +12,6 @@ export default function SampleCard({
   hasMissingCompletion,
 }: {
   sample: DatasetSample;
-  inputTabs?: string[];
   hasMissingCompletion: boolean;
   onSampleRemove?: (sample: DatasetSample) => void;
   onSampleAdd?: (sample: DatasetSample) => void;
@@ -29,7 +27,6 @@ export default function SampleCard({
         <JsonAsTab
           storeKey="input"
           data={sample?.inputs}
-          defaultTabs={inputTabs}
           showRunButton={hasMissingCompletion}
           onSampleAdd={() => onSampleAdd?.(sample)}
           onSampleRemove={() => onSampleRemove?.(sample)}
diff --git a/apps/web/hooks/useSyncedTab.ts b/apps/web/hooks/useSyncedTab.ts
index 5e6f4dd4..50c9bb21 100644
--- a/apps/web/hooks/useSyncedTab.ts
+++ b/apps/web/hooks/useSyncedTab.ts
@@ -39,23 +39,8 @@ export function useSyncedTabs(tabList: string[], tabStoreKey: string) {
   );
 
   useEffect(() => {
-    if ((tabs || []).length < tabList.length) {
-      // find the missing tab
-      const missingTabs = tabList.filter(
-        // @ts-ignore
-        (tab) => !tabs.includes(tab),
-      );
-      setActiveTab(missingTabs[0]);
-    }
     setTabs(tabList);
-  }, [tabs, setTabs, tabList, activeTab]);
-
-  useEffect(() => {
-    //@ts-ignore
-    if (!tabs.includes(activeTab || "")) {
-      setActiveTab(tabs[0]);
-    }
-  }, [activeTab, tabs]);
+  }, [setTabs, tabList]);
 
   return {
     tabs,
diff --git a/examples/multi-turn-chat/.gitignore b/examples/multi-turn-chat/.gitignore
new file mode 100644
index 00000000..b14a4890
--- /dev/null
+++ b/examples/multi-turn-chat/.gitignore
@@ -0,0 +1,3 @@
+
+# Ignore outputs from Empirical
+.empiricalrun
diff --git a/examples/multi-turn-chat/README.md b/examples/multi-turn-chat/README.md
new file mode 100644
index 00000000..458b0405
--- /dev/null
+++ b/examples/multi-turn-chat/README.md
@@ -0,0 +1,29 @@
+# Evaluating multi-turn chat
+This example illustrates how to score outputs for multi-turn chat scenarios.
+
+### Dataset
+The dataset configured is configured in a [Google Sheet](https://docs.google.com/spreadsheets/d/1fZ_3FFj94SiucglQOTrCHQTZVhrWJwWqfSk-vEIp8_I/edit#gid=0). 
+
+### Run Configuration
+The run is implemented using python script. The run configuration mentions `chat.py` as part of configuration.
+Essentially `chat.py` implements multi-turn conversation.
+
+### Scorer Configuration
+The scoring mechanism is implemented through a Python script named `score.py`.
+
+## Steps to run
+To execute the example:
+1. Install dependencies:
+    ```
+    poetry install
+    ```
+
+1. Evaluate multi-turn chat using Empirical:
+    ```
+    npx @empiricalrun/cli run --python-path `poetry env info -e`
+    ```
+    >Note: Ensure `OPENAI_API_KEY` is exported before running above command.
+1. Visualize the output:
+    ```
+    npx @empiricalrun/cli ui
+    ```
\ No newline at end of file
diff --git a/examples/multi-turn-chat/chat.py b/examples/multi-turn-chat/chat.py
new file mode 100644
index 00000000..01c90b3e
--- /dev/null
+++ b/examples/multi-turn-chat/chat.py
@@ -0,0 +1,27 @@
+from openai import AsyncOpenAI
+
+
+async def execute(inputs, parameters):
+    openai = AsyncOpenAI()
+    messages = []
+    for input in inputs:
+        input.get("user_query")
+        messages.append({"role": "user", "content": input.get("user_query")})
+        chat_completion = await openai.chat.completions.create(
+            messages=messages,
+            model="gpt-3.5-turbo",
+        )
+        messages.append(
+            {
+                "role": chat_completion.choices[0].message.role,
+                "content": chat_completion.choices[0].message.content,
+            }
+        )
+        openai.chat.completions.create
+    thread_length = len(messages)
+    return {
+        # setting the last response as the final output of the conversation
+        "value": messages[thread_length - 1].get("content", ""),
+        # saving the thread in metadata for eyeball and scoring output
+        "metadata": {"messages": messages},
+    }
diff --git a/examples/multi-turn-chat/empiricalrc.json b/examples/multi-turn-chat/empiricalrc.json
new file mode 100644
index 00000000..367962e5
--- /dev/null
+++ b/examples/multi-turn-chat/empiricalrc.json
@@ -0,0 +1,20 @@
+{
+  "$schema": "https://assets.empirical.run/config/schema/latest.json",
+  "runs": [
+    {
+      "type": "py-script",
+      "path": "chat.py"
+    }
+  ],
+  "dataset": {
+    "path": "https://docs.google.com/spreadsheets/d/1fZ_3FFj94SiucglQOTrCHQTZVhrWJwWqfSk-vEIp8_I/edit#gid=0",
+    "group_by": "conv_id"
+  },
+  "scorers": [
+    {
+      "name": "llm-evaluation",
+      "type": "py-script",
+      "path": "score.py"
+    }
+  ]
+}
\ No newline at end of file
diff --git a/examples/multi-turn-chat/poetry.lock b/examples/multi-turn-chat/poetry.lock
new file mode 100644
index 00000000..9f071d78
--- /dev/null
+++ b/examples/multi-turn-chat/poetry.lock
@@ -0,0 +1,312 @@
+# This file is automatically @generated by Poetry 1.8.0 and should not be changed by hand.
+
+[[package]]
+name = "annotated-types"
+version = "0.6.0"
+description = "Reusable constraint types to use with typing.Annotated"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "annotated_types-0.6.0-py3-none-any.whl", hash = "sha256:0641064de18ba7a25dee8f96403ebc39113d0cb953a01429249d5c7564666a43"},
+    {file = "annotated_types-0.6.0.tar.gz", hash = "sha256:563339e807e53ffd9c267e99fc6d9ea23eb8443c08f112651963e24e22f84a5d"},
+]
+
+[[package]]
+name = "anyio"
+version = "4.3.0"
+description = "High level compatibility layer for multiple asynchronous event loop implementations"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "anyio-4.3.0-py3-none-any.whl", hash = "sha256:048e05d0f6caeed70d731f3db756d35dcc1f35747c8c403364a8332c630441b8"},
+    {file = "anyio-4.3.0.tar.gz", hash = "sha256:f75253795a87df48568485fd18cdd2a3fa5c4f7c5be8e5e36637733fce06fed6"},
+]
+
+[package.dependencies]
+idna = ">=2.8"
+sniffio = ">=1.1"
+
+[package.extras]
+doc = ["Sphinx (>=7)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme"]
+test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (>=0.17)"]
+trio = ["trio (>=0.23)"]
+
+[[package]]
+name = "certifi"
+version = "2024.2.2"
+description = "Python package for providing Mozilla's CA Bundle."
+optional = false
+python-versions = ">=3.6"
+files = [
+    {file = "certifi-2024.2.2-py3-none-any.whl", hash = "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"},
+    {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"},
+]
+
+[[package]]
+name = "colorama"
+version = "0.4.6"
+description = "Cross-platform colored terminal text."
+optional = false
+python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7"
+files = [
+    {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"},
+    {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"},
+]
+
+[[package]]
+name = "distro"
+version = "1.9.0"
+description = "Distro - an OS platform information API"
+optional = false
+python-versions = ">=3.6"
+files = [
+    {file = "distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2"},
+    {file = "distro-1.9.0.tar.gz", hash = "sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed"},
+]
+
+[[package]]
+name = "h11"
+version = "0.14.0"
+description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761"},
+    {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"},
+]
+
+[[package]]
+name = "httpcore"
+version = "1.0.5"
+description = "A minimal low-level HTTP client."
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "httpcore-1.0.5-py3-none-any.whl", hash = "sha256:421f18bac248b25d310f3cacd198d55b8e6125c107797b609ff9b7a6ba7991b5"},
+    {file = "httpcore-1.0.5.tar.gz", hash = "sha256:34a38e2f9291467ee3b44e89dd52615370e152954ba21721378a87b2960f7a61"},
+]
+
+[package.dependencies]
+certifi = "*"
+h11 = ">=0.13,<0.15"
+
+[package.extras]
+asyncio = ["anyio (>=4.0,<5.0)"]
+http2 = ["h2 (>=3,<5)"]
+socks = ["socksio (==1.*)"]
+trio = ["trio (>=0.22.0,<0.26.0)"]
+
+[[package]]
+name = "httpx"
+version = "0.27.0"
+description = "The next generation HTTP client."
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "httpx-0.27.0-py3-none-any.whl", hash = "sha256:71d5465162c13681bff01ad59b2cc68dd838ea1f10e51574bac27103f00c91a5"},
+    {file = "httpx-0.27.0.tar.gz", hash = "sha256:a0cb88a46f32dc874e04ee956e4c2764aba2aa228f650b06788ba6bda2962ab5"},
+]
+
+[package.dependencies]
+anyio = "*"
+certifi = "*"
+httpcore = "==1.*"
+idna = "*"
+sniffio = "*"
+
+[package.extras]
+brotli = ["brotli", "brotlicffi"]
+cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"]
+http2 = ["h2 (>=3,<5)"]
+socks = ["socksio (==1.*)"]
+
+[[package]]
+name = "idna"
+version = "3.7"
+description = "Internationalized Domain Names in Applications (IDNA)"
+optional = false
+python-versions = ">=3.5"
+files = [
+    {file = "idna-3.7-py3-none-any.whl", hash = "sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0"},
+    {file = "idna-3.7.tar.gz", hash = "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc"},
+]
+
+[[package]]
+name = "openai"
+version = "1.26.0"
+description = "The official Python library for the openai API"
+optional = false
+python-versions = ">=3.7.1"
+files = [
+    {file = "openai-1.26.0-py3-none-any.whl", hash = "sha256:884ced523fb0225780f8b0e0ed6f7e014049c32d049a41ad0ac962869f1055d1"},
+    {file = "openai-1.26.0.tar.gz", hash = "sha256:642e857b60855702ee6ff665e8fa80946164f77b92e58fd24e01b545685b8405"},
+]
+
+[package.dependencies]
+anyio = ">=3.5.0,<5"
+distro = ">=1.7.0,<2"
+httpx = ">=0.23.0,<1"
+pydantic = ">=1.9.0,<3"
+sniffio = "*"
+tqdm = ">4"
+typing-extensions = ">=4.7,<5"
+
+[package.extras]
+datalib = ["numpy (>=1)", "pandas (>=1.2.3)", "pandas-stubs (>=1.1.0.11)"]
+
+[[package]]
+name = "pydantic"
+version = "2.7.1"
+description = "Data validation using Python type hints"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "pydantic-2.7.1-py3-none-any.whl", hash = "sha256:e029badca45266732a9a79898a15ae2e8b14840b1eabbb25844be28f0b33f3d5"},
+    {file = "pydantic-2.7.1.tar.gz", hash = "sha256:e9dbb5eada8abe4d9ae5f46b9939aead650cd2b68f249bb3a8139dbe125803cc"},
+]
+
+[package.dependencies]
+annotated-types = ">=0.4.0"
+pydantic-core = "2.18.2"
+typing-extensions = ">=4.6.1"
+
+[package.extras]
+email = ["email-validator (>=2.0.0)"]
+
+[[package]]
+name = "pydantic-core"
+version = "2.18.2"
+description = "Core functionality for Pydantic validation and serialization"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "pydantic_core-2.18.2-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:9e08e867b306f525802df7cd16c44ff5ebbe747ff0ca6cf3fde7f36c05a59a81"},
+    {file = "pydantic_core-2.18.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f0a21cbaa69900cbe1a2e7cad2aa74ac3cf21b10c3efb0fa0b80305274c0e8a2"},
+    {file = "pydantic_core-2.18.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0680b1f1f11fda801397de52c36ce38ef1c1dc841a0927a94f226dea29c3ae3d"},
+    {file = "pydantic_core-2.18.2-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:95b9d5e72481d3780ba3442eac863eae92ae43a5f3adb5b4d0a1de89d42bb250"},
+    {file = "pydantic_core-2.18.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c4fcf5cd9c4b655ad666ca332b9a081112cd7a58a8b5a6ca7a3104bc950f2038"},
+    {file = "pydantic_core-2.18.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b5155ff768083cb1d62f3e143b49a8a3432e6789a3abee8acd005c3c7af1c74"},
+    {file = "pydantic_core-2.18.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:553ef617b6836fc7e4df130bb851e32fe357ce36336d897fd6646d6058d980af"},
+    {file = "pydantic_core-2.18.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b89ed9eb7d616ef5714e5590e6cf7f23b02d0d539767d33561e3675d6f9e3857"},
+    {file = "pydantic_core-2.18.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:75f7e9488238e920ab6204399ded280dc4c307d034f3924cd7f90a38b1829563"},
+    {file = "pydantic_core-2.18.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:ef26c9e94a8c04a1b2924149a9cb081836913818e55681722d7f29af88fe7b38"},
+    {file = "pydantic_core-2.18.2-cp310-none-win32.whl", hash = "sha256:182245ff6b0039e82b6bb585ed55a64d7c81c560715d1bad0cbad6dfa07b4027"},
+    {file = "pydantic_core-2.18.2-cp310-none-win_amd64.whl", hash = "sha256:e23ec367a948b6d812301afc1b13f8094ab7b2c280af66ef450efc357d2ae543"},
+    {file = "pydantic_core-2.18.2-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:219da3f096d50a157f33645a1cf31c0ad1fe829a92181dd1311022f986e5fbe3"},
+    {file = "pydantic_core-2.18.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:cc1cfd88a64e012b74e94cd00bbe0f9c6df57049c97f02bb07d39e9c852e19a4"},
+    {file = "pydantic_core-2.18.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:05b7133a6e6aeb8df37d6f413f7705a37ab4031597f64ab56384c94d98fa0e90"},
+    {file = "pydantic_core-2.18.2-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:224c421235f6102e8737032483f43c1a8cfb1d2f45740c44166219599358c2cd"},
+    {file = "pydantic_core-2.18.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b14d82cdb934e99dda6d9d60dc84a24379820176cc4a0d123f88df319ae9c150"},
+    {file = "pydantic_core-2.18.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2728b01246a3bba6de144f9e3115b532ee44bd6cf39795194fb75491824a1413"},
+    {file = "pydantic_core-2.18.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:470b94480bb5ee929f5acba6995251ada5e059a5ef3e0dfc63cca287283ebfa6"},
+    {file = "pydantic_core-2.18.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:997abc4df705d1295a42f95b4eec4950a37ad8ae46d913caeee117b6b198811c"},
+    {file = "pydantic_core-2.18.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:75250dbc5290e3f1a0f4618db35e51a165186f9034eff158f3d490b3fed9f8a0"},
+    {file = "pydantic_core-2.18.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:4456f2dca97c425231d7315737d45239b2b51a50dc2b6f0c2bb181fce6207664"},
+    {file = "pydantic_core-2.18.2-cp311-none-win32.whl", hash = "sha256:269322dcc3d8bdb69f054681edff86276b2ff972447863cf34c8b860f5188e2e"},
+    {file = "pydantic_core-2.18.2-cp311-none-win_amd64.whl", hash = "sha256:800d60565aec896f25bc3cfa56d2277d52d5182af08162f7954f938c06dc4ee3"},
+    {file = "pydantic_core-2.18.2-cp311-none-win_arm64.whl", hash = "sha256:1404c69d6a676245199767ba4f633cce5f4ad4181f9d0ccb0577e1f66cf4c46d"},
+    {file = "pydantic_core-2.18.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:fb2bd7be70c0fe4dfd32c951bc813d9fe6ebcbfdd15a07527796c8204bd36242"},
+    {file = "pydantic_core-2.18.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6132dd3bd52838acddca05a72aafb6eab6536aa145e923bb50f45e78b7251043"},
+    {file = "pydantic_core-2.18.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7d904828195733c183d20a54230c0df0eb46ec746ea1a666730787353e87182"},
+    {file = "pydantic_core-2.18.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c9bd70772c720142be1020eac55f8143a34ec9f82d75a8e7a07852023e46617f"},
+    {file = "pydantic_core-2.18.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2b8ed04b3582771764538f7ee7001b02e1170223cf9b75dff0bc698fadb00cf3"},
+    {file = "pydantic_core-2.18.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e6dac87ddb34aaec85f873d737e9d06a3555a1cc1a8e0c44b7f8d5daeb89d86f"},
+    {file = "pydantic_core-2.18.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ca4ae5a27ad7a4ee5170aebce1574b375de390bc01284f87b18d43a3984df72"},
+    {file = "pydantic_core-2.18.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:886eec03591b7cf058467a70a87733b35f44707bd86cf64a615584fd72488b7c"},
+    {file = "pydantic_core-2.18.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ca7b0c1f1c983e064caa85f3792dd2fe3526b3505378874afa84baf662e12241"},
+    {file = "pydantic_core-2.18.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4b4356d3538c3649337df4074e81b85f0616b79731fe22dd11b99499b2ebbdf3"},
+    {file = "pydantic_core-2.18.2-cp312-none-win32.whl", hash = "sha256:8b172601454f2d7701121bbec3425dd71efcb787a027edf49724c9cefc14c038"},
+    {file = "pydantic_core-2.18.2-cp312-none-win_amd64.whl", hash = "sha256:b1bd7e47b1558ea872bd16c8502c414f9e90dcf12f1395129d7bb42a09a95438"},
+    {file = "pydantic_core-2.18.2-cp312-none-win_arm64.whl", hash = "sha256:98758d627ff397e752bc339272c14c98199c613f922d4a384ddc07526c86a2ec"},
+    {file = "pydantic_core-2.18.2-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:9fdad8e35f278b2c3eb77cbdc5c0a49dada440657bf738d6905ce106dc1de439"},
+    {file = "pydantic_core-2.18.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:1d90c3265ae107f91a4f279f4d6f6f1d4907ac76c6868b27dc7fb33688cfb347"},
+    {file = "pydantic_core-2.18.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:390193c770399861d8df9670fb0d1874f330c79caaca4642332df7c682bf6b91"},
+    {file = "pydantic_core-2.18.2-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:82d5d4d78e4448683cb467897fe24e2b74bb7b973a541ea1dcfec1d3cbce39fb"},
+    {file = "pydantic_core-2.18.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4774f3184d2ef3e14e8693194f661dea5a4d6ca4e3dc8e39786d33a94865cefd"},
+    {file = "pydantic_core-2.18.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d4d938ec0adf5167cb335acb25a4ee69a8107e4984f8fbd2e897021d9e4ca21b"},
+    {file = "pydantic_core-2.18.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e0e8b1be28239fc64a88a8189d1df7fad8be8c1ae47fcc33e43d4be15f99cc70"},
+    {file = "pydantic_core-2.18.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:868649da93e5a3d5eacc2b5b3b9235c98ccdbfd443832f31e075f54419e1b96b"},
+    {file = "pydantic_core-2.18.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:78363590ef93d5d226ba21a90a03ea89a20738ee5b7da83d771d283fd8a56761"},
+    {file = "pydantic_core-2.18.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:852e966fbd035a6468fc0a3496589b45e2208ec7ca95c26470a54daed82a0788"},
+    {file = "pydantic_core-2.18.2-cp38-none-win32.whl", hash = "sha256:6a46e22a707e7ad4484ac9ee9f290f9d501df45954184e23fc29408dfad61350"},
+    {file = "pydantic_core-2.18.2-cp38-none-win_amd64.whl", hash = "sha256:d91cb5ea8b11607cc757675051f61b3d93f15eca3cefb3e6c704a5d6e8440f4e"},
+    {file = "pydantic_core-2.18.2-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:ae0a8a797a5e56c053610fa7be147993fe50960fa43609ff2a9552b0e07013e8"},
+    {file = "pydantic_core-2.18.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:042473b6280246b1dbf530559246f6842b56119c2926d1e52b631bdc46075f2a"},
+    {file = "pydantic_core-2.18.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1a388a77e629b9ec814c1b1e6b3b595fe521d2cdc625fcca26fbc2d44c816804"},
+    {file = "pydantic_core-2.18.2-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e25add29b8f3b233ae90ccef2d902d0ae0432eb0d45370fe315d1a5cf231004b"},
+    {file = "pydantic_core-2.18.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f459a5ce8434614dfd39bbebf1041952ae01da6bed9855008cb33b875cb024c0"},
+    {file = "pydantic_core-2.18.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:eff2de745698eb46eeb51193a9f41d67d834d50e424aef27df2fcdee1b153845"},
+    {file = "pydantic_core-2.18.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a8309f67285bdfe65c372ea3722b7a5642680f3dba538566340a9d36e920b5f0"},
+    {file = "pydantic_core-2.18.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f93a8a2e3938ff656a7c1bc57193b1319960ac015b6e87d76c76bf14fe0244b4"},
+    {file = "pydantic_core-2.18.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:22057013c8c1e272eb8d0eebc796701167d8377441ec894a8fed1af64a0bf399"},
+    {file = "pydantic_core-2.18.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:cfeecd1ac6cc1fb2692c3d5110781c965aabd4ec5d32799773ca7b1456ac636b"},
+    {file = "pydantic_core-2.18.2-cp39-none-win32.whl", hash = "sha256:0d69b4c2f6bb3e130dba60d34c0845ba31b69babdd3f78f7c0c8fae5021a253e"},
+    {file = "pydantic_core-2.18.2-cp39-none-win_amd64.whl", hash = "sha256:d9319e499827271b09b4e411905b24a426b8fb69464dfa1696258f53a3334641"},
+    {file = "pydantic_core-2.18.2-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:a1874c6dd4113308bd0eb568418e6114b252afe44319ead2b4081e9b9521fe75"},
+    {file = "pydantic_core-2.18.2-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:ccdd111c03bfd3666bd2472b674c6899550e09e9f298954cfc896ab92b5b0e6d"},
+    {file = "pydantic_core-2.18.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e18609ceaa6eed63753037fc06ebb16041d17d28199ae5aba0052c51449650a9"},
+    {file = "pydantic_core-2.18.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e5c584d357c4e2baf0ff7baf44f4994be121e16a2c88918a5817331fc7599d7"},
+    {file = "pydantic_core-2.18.2-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:43f0f463cf89ace478de71a318b1b4f05ebc456a9b9300d027b4b57c1a2064fb"},
+    {file = "pydantic_core-2.18.2-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:e1b395e58b10b73b07b7cf740d728dd4ff9365ac46c18751bf8b3d8cca8f625a"},
+    {file = "pydantic_core-2.18.2-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:0098300eebb1c837271d3d1a2cd2911e7c11b396eac9661655ee524a7f10587b"},
+    {file = "pydantic_core-2.18.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:36789b70d613fbac0a25bb07ab3d9dba4d2e38af609c020cf4d888d165ee0bf3"},
+    {file = "pydantic_core-2.18.2-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:3f9a801e7c8f1ef8718da265bba008fa121243dfe37c1cea17840b0944dfd72c"},
+    {file = "pydantic_core-2.18.2-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:3a6515ebc6e69d85502b4951d89131ca4e036078ea35533bb76327f8424531ce"},
+    {file = "pydantic_core-2.18.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:20aca1e2298c56ececfd8ed159ae4dde2df0781988c97ef77d5c16ff4bd5b400"},
+    {file = "pydantic_core-2.18.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:223ee893d77a310a0391dca6df00f70bbc2f36a71a895cecd9a0e762dc37b349"},
+    {file = "pydantic_core-2.18.2-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2334ce8c673ee93a1d6a65bd90327588387ba073c17e61bf19b4fd97d688d63c"},
+    {file = "pydantic_core-2.18.2-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:cbca948f2d14b09d20268cda7b0367723d79063f26c4ffc523af9042cad95592"},
+    {file = "pydantic_core-2.18.2-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:b3ef08e20ec49e02d5c6717a91bb5af9b20f1805583cb0adfe9ba2c6b505b5ae"},
+    {file = "pydantic_core-2.18.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:c6fdc8627910eed0c01aed6a390a252fe3ea6d472ee70fdde56273f198938374"},
+    {file = "pydantic_core-2.18.2.tar.gz", hash = "sha256:2e29d20810dfc3043ee13ac7d9e25105799817683348823f305ab3f349b9386e"},
+]
+
+[package.dependencies]
+typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0"
+
+[[package]]
+name = "sniffio"
+version = "1.3.1"
+description = "Sniff out which async library your code is running under"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2"},
+    {file = "sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc"},
+]
+
+[[package]]
+name = "tqdm"
+version = "4.66.4"
+description = "Fast, Extensible Progress Meter"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "tqdm-4.66.4-py3-none-any.whl", hash = "sha256:b75ca56b413b030bc3f00af51fd2c1a1a5eac6a0c1cca83cbb37a5c52abce644"},
+    {file = "tqdm-4.66.4.tar.gz", hash = "sha256:e4d936c9de8727928f3be6079590e97d9abfe8d39a590be678eb5919ffc186bb"},
+]
+
+[package.dependencies]
+colorama = {version = "*", markers = "platform_system == \"Windows\""}
+
+[package.extras]
+dev = ["pytest (>=6)", "pytest-cov", "pytest-timeout", "pytest-xdist"]
+notebook = ["ipywidgets (>=6)"]
+slack = ["slack-sdk"]
+telegram = ["requests"]
+
+[[package]]
+name = "typing-extensions"
+version = "4.11.0"
+description = "Backported and Experimental Type Hints for Python 3.8+"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "typing_extensions-4.11.0-py3-none-any.whl", hash = "sha256:c1f94d72897edaf4ce775bb7558d5b79d8126906a14ea5ed1635921406c0387a"},
+    {file = "typing_extensions-4.11.0.tar.gz", hash = "sha256:83f085bd5ca59c80295fc2a82ab5dac679cbe02b9f33f7d83af68e241bea51b0"},
+]
+
+[metadata]
+lock-version = "2.0"
+python-versions = "^3.12"
+content-hash = "008cd170541cd28ee8f72c3fa582cb9d55cc490dc6b37bb8d101dbd3820c263a"
diff --git a/examples/multi-turn-chat/pyproject.toml b/examples/multi-turn-chat/pyproject.toml
new file mode 100644
index 00000000..4077945a
--- /dev/null
+++ b/examples/multi-turn-chat/pyproject.toml
@@ -0,0 +1,16 @@
+[tool.poetry]
+name = "multi-turn-chat"
+version = "0.1.0"
+description = ""
+authors = ["Empirical Team"]
+readme = "README.md"
+package-mode = false
+
+[tool.poetry.dependencies]
+python = "^3.12"
+openai = "^1.26.0"
+
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
diff --git a/examples/multi-turn-chat/score.py b/examples/multi-turn-chat/score.py
new file mode 100644
index 00000000..d115c2e2
--- /dev/null
+++ b/examples/multi-turn-chat/score.py
@@ -0,0 +1,87 @@
+from typing import TypedDict
+from openai import AsyncOpenAI
+import json
+from enum import Enum
+
+
+class ScorerResultEnum(Enum):
+    YES = "Yes"
+    NO = "No"
+
+
+class ScorerResult(TypedDict):
+    result: ScorerResultEnum
+    reason: str
+
+
+async def llm_score(output, criteria) -> ScorerResult:
+    system_prompt = "You are an expert evaluator who grades an output string based on a criteria. The output must fulfil the criteria to pass the evaluation."
+    openai = AsyncOpenAI()
+    completion = await openai.chat.completions.create(
+        messages=[
+            {"role": "system", "content": system_prompt},
+            {"role": "user", "content": f"Criteria: {criteria}\n\nOutput: {output}"},
+        ],
+        model="gpt-3.5-turbo",
+        temperature=0.1,
+        tools=[
+            {
+                "type": "function",
+                "function": {
+                    "name": "set_evaluator_response",
+                    "description": "Sets the response of the evaluation",
+                    "parameters": {
+                        "type": "object",
+                        "properties": {
+                            "reason": {
+                                "type": "string",
+                                "description": "Reasoning for the evaluation, shared as a step-by-step chain of thought",
+                            },
+                            "result": {
+                                "type": "string",
+                                "enum": [
+                                    ScorerResultEnum.YES.value,
+                                    ScorerResultEnum.NO.value,
+                                ],
+                            },
+                        },
+                        "required": ["reason", "result"],
+                    },
+                },
+            },
+        ],
+        tool_choice={
+            "type": "function",
+            "function": {"name": "set_evaluator_response"},
+        },
+    )
+    response = completion.choices[0].message.tool_calls[0]
+    return ScorerResult(**json.loads(response.function.arguments))
+
+
+async def evaluate(output, inputs):
+    thread: list[object] = output.get("metadata", {}).get("messages", [])
+    assistant_responses: list[object] = [
+        obj for obj in thread if obj["role"] == "assistant"
+    ]
+    success = 0
+    message = ""
+    total = len(inputs)
+    for idx, assistant_response in enumerate(assistant_responses):
+        score = await llm_score(
+            assistant_response.get("content"),
+            inputs[idx].get("acceptable_response", ""),
+        )
+        if score["result"] == ScorerResultEnum.YES.value:
+            success = success + 1
+
+        message = (
+            f"{message} [{idx}]: {score['reason']}"
+            if message != ""
+            else f"[{idx}]: {score['reason']}"
+        )
+
+    return {
+        "score": success / total,
+        "message": message,
+    }
diff --git a/packages/cli/src/bin/dataset/index.ts b/packages/cli/src/bin/dataset/index.ts
index 1c2a58a2..51809c4e 100644
--- a/packages/cli/src/bin/dataset/index.ts
+++ b/packages/cli/src/bin/dataset/index.ts
@@ -67,9 +67,26 @@ export async function loadDataset(config: DatasetConfig): Promise<Dataset> {
     }));
   } else if ("path" in config) {
     contents = await fetchContents(config.path); // wrap and throw if fetching fails
-    const parsed = await parseDataset(config.path, contents);
-    if (parsed) {
-      samples = parsed;
+    samples = (await parseDataset(config.path, contents)) || samples;
+    if (samples && config.group_by) {
+      const updatedSamples: DatasetSample[] = [];
+      for (const idx in samples) {
+        const sample = samples[idx]!;
+        const updateIdx = updatedSamples.findIndex((s) => {
+          return (
+            s.inputs[0][config.group_by!] === sample.inputs[config.group_by!]
+          );
+        });
+        if (updateIdx < 0) {
+          updatedSamples.push({
+            ...sample,
+            inputs: [sample.inputs],
+          });
+        } else {
+          (updatedSamples[updateIdx]?.inputs as any[]).push(sample.inputs);
+        }
+      }
+      samples = updatedSamples;
     }
   } else {
     throw new DatasetError(
diff --git a/packages/scorer/src/provider/deterministic/script.ts b/packages/scorer/src/provider/deterministic/script.ts
index 5b022c21..c566ca32 100644
--- a/packages/scorer/src/provider/deterministic/script.ts
+++ b/packages/scorer/src/provider/deterministic/script.ts
@@ -74,7 +74,7 @@ export const scoreWithPythonScript: ScoringFn = async ({
     });
 
     shell.on("pythonError", function (message) {
-      console.error(message);
+      console.error(message.traceback);
       runOutput.push(
         JSON.stringify([
           {
diff --git a/packages/scorer/src/python/wrapper.py b/packages/scorer/src/python/wrapper.py
index 684a763b..f651772f 100644
--- a/packages/scorer/src/python/wrapper.py
+++ b/packages/scorer/src/python/wrapper.py
@@ -1,10 +1,29 @@
 import json
 import sys
 import importlib
+import inspect
+import asyncio
 
 
-sys.path.append(sys.argv[1])
-user_module = importlib.import_module(sys.argv[2])
+async def evaluator():
+    sys.path.append(sys.argv[1])
+    user_module = importlib.import_module(sys.argv[2])
+    if hasattr(user_module, "evaluate"):
+        if inspect.iscoroutinefunction(user_module.evaluate):
+            result = await user_module.evaluate(
+                json.loads(sys.argv[3]), json.loads(sys.argv[4])
+            )
+        else:
+            result = user_module.evaluate(
+                json.loads(sys.argv[3]), json.loads(sys.argv[4])
+            )
+    else:
+        raise KeyError(
+            "Error: failed to find `evaluate` method in the python script provided."
+        )
+    if result is None:
+        raise KeyError("Error: `evaluate` function did not return a result")
+    print("scorer_output:", json.dumps(result))
 
-result = user_module.evaluate(json.loads(sys.argv[3]), json.loads(sys.argv[4]))
-print("scorer_output:", json.dumps(result))
+
+asyncio.run(evaluator())
diff --git a/packages/types/src/index.ts b/packages/types/src/index.ts
index 725a3250..6b82e15c 100644
--- a/packages/types/src/index.ts
+++ b/packages/types/src/index.ts
@@ -160,7 +160,7 @@ export interface RunCompletion {
   created_at: Date;
 }
 
-export type DatasetSampleInputs = { [key: string]: unknown };
+export type DatasetSampleInputs = any;
 
 export type DatasetSample = {
   id: string;
@@ -185,6 +185,7 @@ export type DatasetConfig =
     }
   | {
       path: string;
+      group_by?: string;
     };
 
 // TODO: fix types. text generation and others how does that show up ?