diff --git a/.gitignore b/.gitignore index 73315cbdaa..ed95cd72e6 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,6 @@ coverage.xml # devtools pyrightconfig.json + +# Demo data +tutorials/internal/demo_llama_index/*.json diff --git a/src/phoenix/trace/fixtures.py b/src/phoenix/trace/fixtures.py index f969aeef1f..f55ea19528 100644 --- a/src/phoenix/trace/fixtures.py +++ b/src/phoenix/trace/fixtures.py @@ -82,6 +82,26 @@ class TracesFixture: dataset_fixtures: Iterable[DatasetFixture] = () +demo_llama_index_rag_fixture = TracesFixture( + name="demo_llama_index_rag", + description="Traces and evaluations of a RAG chatbot using LlamaIndex.", + file_name="demo_llama_index_rag.parquet", + evaluation_fixtures=( + EvaluationFixture( + evaluation_name="Q&A Correctness", + file_name="demo_llama_index_rag_qa_correctness_eval.parquet", + ), + EvaluationFixture( + evaluation_name="Hallucination", + file_name="demo_llama_index_rag_hallucination_eval.parquet", + ), + DocumentEvaluationFixture( + evaluation_name="Relevance", + file_name="demo_llama_index_rag_doc_relevance_eval.parquet", + ), + ), +) + llama_index_rag_fixture = TracesFixture( name="llama_index_rag", description="Traces from running the llama_index on a RAG use case.", @@ -169,6 +189,7 @@ class TracesFixture: ) TRACES_FIXTURES: List[TracesFixture] = [ + demo_llama_index_rag_fixture, llama_index_rag_fixture, llama_index_rag_fixture_with_davinci, langchain_rag_stuff_document_chain_fixture, @@ -269,6 +290,7 @@ def send_dataset_fixtures( def get_evals_from_fixture(fixture_name: str) -> Iterator[pb.Evaluation]: fixture = get_trace_fixture_by_name(fixture_name) for eval_fixture in fixture.evaluation_fixtures: + print(eval_fixture) yield from _read_eval_fixture(eval_fixture) diff --git a/tutorials/internal/.gitignore b/tutorials/internal/.gitignore deleted file mode 100644 index bccc1450f2..0000000000 --- a/tutorials/internal/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*.parquet \ No newline at end of file diff --git a/tutorials/internal/demo_llama_index/.gitignore b/tutorials/internal/demo_llama_index/.gitignore new file mode 100644 index 0000000000..c07349dca6 --- /dev/null +++ b/tutorials/internal/demo_llama_index/.gitignore @@ -0,0 +1,2 @@ +*.json +*.parquet diff --git a/tutorials/internal/demo_llama_index/demo_llama_index_rag.ipynb b/tutorials/internal/demo_llama_index/demo_llama_index_rag.ipynb new file mode 100644 index 0000000000..96e9bcb9fe --- /dev/null +++ b/tutorials/internal/demo_llama_index/demo_llama_index_rag.ipynb @@ -0,0 +1,1324 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "wYET8LC-3koX" + }, + "source": [ + "## Setup" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Install libraries" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "E3qLx9JwjMuj" + }, + "outputs": [], + "source": [ + "!pip install -qq arize-phoenix llama-index \"openai>=1\" gcsfs nest_asyncio langchain langchain-community cohere llama-index-postprocessor-cohere-rerank" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "EejzP1ov3wgw" + }, + "source": [ + "Set up environment variables\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "SfHrgDaHjkFP", + "outputId": "4f28de04-c240-42bb-e49a-a8abe892bd10" + }, + "outputs": [], + "source": [ + "import os\n", + "from getpass import getpass\n", + "\n", + "if not (openai_api_key := os.getenv(\"OPENAI_API_KEY\")):\n", + " openai_api_key = getpass(\"πŸ”‘ Enter your OpenAI API key: \")\n", + "os.environ[\"OPENAI_API_KEY\"] = openai_api_key\n", + "\n", + "if not (cohere_api_key := os.getenv(\"COHERE_API_KEY\")):\n", + " cohere_api_key = getpass(\"πŸ”‘ Enter your Cohere API key: \")\n", + "os.environ[\"COHERE_API_KEY\"] = cohere_api_key" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "nrRN-Tur32Qd" + }, + "source": [ + "## Launch Phoenix and Instrumentation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "RDmiCpAlK3Wd" + }, + "outputs": [], + "source": [ + "import phoenix as px" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "session = px.launch_app()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Ig1vG4l5co7-" + }, + "outputs": [], + "source": [ + "from openinference.instrumentation.llama_index import LlamaIndexInstrumentor\n", + "from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter\n", + "from opentelemetry.sdk.trace import TracerProvider\n", + "from opentelemetry.sdk.trace.export import SimpleSpanProcessor\n", + "\n", + "endpoint = \"http://127.0.0.1:6006/v1/traces\"\n", + "tracer_provider = TracerProvider()\n", + "tracer_provider.add_span_processor(SimpleSpanProcessor(OTLPSpanExporter(endpoint)))\n", + "\n", + "LlamaIndexInstrumentor().instrument(tracer_provider=tracer_provider)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "7OP5ELmg4LeS" + }, + "source": [ + "## Parse Phoenix Documentation into Llama-Index Documents" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "w6cVWlJGjsMO", + "outputId": "649293f3-48a6-470e-f9e1-0de741667746" + }, + "outputs": [], + "source": [ + "# The nest_asyncio module enables the nesting of asynchronous functions within an already running async loop.\n", + "# This is necessary because Jupyter notebooks inherently operate in an asynchronous loop.\n", + "# By applying nest_asyncio, we can run additional async functions within this existing loop without conflicts.\n", + "import json\n", + "import logging\n", + "import sys\n", + "import time\n", + "\n", + "import nest_asyncio\n", + "\n", + "nest_asyncio.apply()\n", + "\n", + "import pandas as pd\n", + "from langchain.document_loaders import GitbookLoader\n", + "from llama_index.core import Document, VectorStoreIndex\n", + "from llama_index.llms.openai import OpenAI" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "PWDCQJN1co7-" + }, + "source": [ + "Enable Phoenix tracing via `LlamaIndexInstrumentor`. Phoenix uses OpenInference traces - an open-source standard for capturing and storing LLM application traces that enables LLM applications to seamlessly integrate with LLM observability solutions such as Phoenix." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "F9mFzfnlj2rV", + "outputId": "b52d862f-bb3c-41f6-b07a-3ad0e18c5cc7" + }, + "outputs": [], + "source": [ + "\"\"\"\n", + "Fetches the Arize documentation from Gitbook and serializes it into LangChain format.\n", + "\"\"\"\n", + "\n", + "\n", + "def load_gitbook_docs(docs_url: str):\n", + " \"\"\"Loads documents from a Gitbook URL.\n", + "\n", + " Args:\n", + " docs_url (str): URL to Gitbook docs.\n", + "\n", + " Returns:\n", + " List[LangChainDocument]: List of documents in LangChain format.\n", + " \"\"\"\n", + " loader = GitbookLoader(\n", + " docs_url,\n", + " load_all_paths=True,\n", + " )\n", + " return loader.load()\n", + "\n", + "\n", + "logging.basicConfig(level=logging.INFO, stream=sys.stdout)\n", + "\n", + "# fetch documentation\n", + "docs_url = \"https://docs.arize.com/phoenix\"\n", + "embedding_model_name = \"text-embedding-ada-002\"\n", + "docs = load_gitbook_docs(docs_url)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "B5KT9icyk_Lh" + }, + "outputs": [], + "source": [ + "documents = []\n", + "for doc in docs:\n", + " documents.append(Document(metadata=doc.metadata, text=doc.page_content))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "CemnegejgI7f", + "outputId": "1f7e434f-4e1b-4937-9478-aa1f0cedc2e7" + }, + "outputs": [], + "source": [ + "documents[0].metadata" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "lAPNcG-RHWL2" + }, + "outputs": [], + "source": [ + "# Convert documents to a JSON serializable format (if needed)\n", + "documents_json = [doc.to_dict() for doc in documents]\n", + "\n", + "# Save to a JSON file\n", + "with open(\"llama_index_documents.json\", \"w\") as file:\n", + " json.dump(documents_json, file, indent=4)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "a3pYhJwq4VwX" + }, + "source": [ + "## Set Up VectorStore and Query Engine" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "4S8pfGvFluIg" + }, + "outputs": [], + "source": [ + "from llama_index.core.node_parser import SentenceSplitter\n", + "from llama_index.postprocessor.cohere_rerank import CohereRerank\n", + "\n", + "# Define an LLM\n", + "llm = OpenAI(model=\"gpt-4\")\n", + "\n", + "# Build index with a chunk_size of 1024\n", + "splitter = SentenceSplitter(chunk_size=1024, chunk_overlap=250)\n", + "nodes = splitter.get_nodes_from_documents(documents)\n", + "vector_index = VectorStoreIndex(nodes)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "cOmyBVoSco7-" + }, + "source": [ + "Build a QueryEngine and start querying." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "3OB49Cghco7-", + "outputId": "95c7afd2-4d6c-41dc-f1ce-e636e1516a24" + }, + "outputs": [], + "source": [ + "cohere_api_key = os.environ[\"COHERE_API_KEY\"]\n", + "cohere_rerank = CohereRerank(api_key=cohere_api_key, top_n=2)\n", + "\n", + "query_engine = vector_index.as_query_engine(\n", + " similarity_top_k=5,\n", + " node_postprocessors=[cohere_rerank],\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "wmlw_irPco7_" + }, + "source": [ + "## Import Questions" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "21s8Kk3znqJ-" + }, + "outputs": [], + "source": [ + "questions_df = pd.read_parquet(\"fixtures/demo_llama_index_rag_questions.parquet\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 424 + }, + "id": "qKpbdBFwsKuC", + "outputId": "cd5feb82-1643-440b-dd21-9294cdb09394" + }, + "outputs": [], + "source": [ + "questions_df" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "jc7hbkyu5zr3" + }, + "source": [ + "## Generate Answers for all of the questions" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "4HmTkFYJsLq4", + "outputId": "c6c18359-d7e1-4180-f194-2d67b27b6c0f" + }, + "outputs": [], + "source": [ + "# loop over the questions and generate the answers\n", + "for i, row in questions_df.iterrows():\n", + " if i in [25, 50, 75]:\n", + " time.sleep(30)\n", + " question = row[\"Prompt/ Question\"]\n", + " response_vector = query_engine.query(question)\n", + " print(f\"Question: {question}\\nAnswer: {response_vector.response}\\n\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Jd6tXZk-59VT" + }, + "source": [ + "## Phoenix Evals" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 542 + }, + "id": "6iwW7y2Bco8H", + "outputId": "49df3b47-4999-4a52-b4e6-d46da4e1654f" + }, + "outputs": [], + "source": [ + "from phoenix.session.evaluation import get_retrieved_documents\n", + "\n", + "retrieved_documents_df = get_retrieved_documents(px.Client())\n", + "retrieved_documents_df" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 455 + }, + "id": "QzKMthtAxzsU", + "outputId": "a65f625e-b1c5-4e51-ac9c-00ff0f3efbb9" + }, + "outputs": [], + "source": [ + "from phoenix.session.evaluation import get_qa_with_reference\n", + "\n", + "queries_df = get_qa_with_reference(px.active_session())\n", + "queries_df" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "AFECnz04co8H" + }, + "source": [ + "Let's now use Phoenix's LLM Evals to evaluate the relevance of the retrieved documents with regards to the query. Note, we've turned on `explanations` which prompts the LLM to explain it's reasoning. This can be useful for debugging and for figuring out potential corrective actions." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 49, + "referenced_widgets": [ + "8648201ffcab4023a1a54dde1b964b8b", + "df7e8713256a48c2992f3dce61c702d1", + "d9af102cb64142b3ac7ea6a75027e6a7", + "59a894868a0344789e8458549f8aa894", + "2c6eb911b6354343a46fca08b0b85f03", + "88978e8ddc704d8e9d4048095047615a", + "44f75a3f6c48404990251849a9be33ca", + "ae22d487bb1b442cad9018897a53e623", + "c6aab3035f8247b6a3e276b2e86fe79b", + "99e5a5eeb1ef41d68f41bbd3cb98d544", + "d946d51d35344047add12d14c8445f46" + ] + }, + "id": "Hq4sr_zIco8H", + "outputId": "176a349e-727e-4a4c-c305-176fc34ae23a" + }, + "outputs": [], + "source": [ + "from phoenix.evals import (\n", + " HallucinationEvaluator,\n", + " OpenAIModel,\n", + " QAEvaluator,\n", + " RelevanceEvaluator,\n", + " run_evals,\n", + ")\n", + "\n", + "eval_model = OpenAIModel(model=\"gpt-4\")\n", + "relevance_evaluator = RelevanceEvaluator(eval_model)\n", + "hallucination_evaluator = HallucinationEvaluator(eval_model)\n", + "qa_evaluator = QAEvaluator(eval_model)\n", + "\n", + "retrieved_documents_relevance_df = run_evals(\n", + " evaluators=[relevance_evaluator],\n", + " dataframe=retrieved_documents_df,\n", + " provide_explanation=True,\n", + " concurrency=20,\n", + ")[0]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 49, + "referenced_widgets": [ + "3363dcebe07847be99e7b41b3840d042", + "90412f8febda4ab1bb22fe56e2392274", + "f7654cf0da3f47da801c21b947fda6da", + "82e8f507e44841f88be5c66c91b27f48", + "8aeca790868b44babe415849afba77db", + "5f8024635af64b0a99855ae3adef558d", + "aed1f2822d114ce4be7dcddfbc2dfc20", + "aea78f53909e44b195de5886df74fad1", + "fcf5b6ce6ca94b48b80ed2aa8ced55d5", + "d10322222cb3435fafdbbdc4d98e5059", + "d1d54e8e26a54881b1380b02159203ad" + ] + }, + "id": "Pe9J4-zYwuCl", + "outputId": "b3424ecb-9536-4230-92ce-debaf19e690b" + }, + "outputs": [], + "source": [ + "hallucination_eval_df, qa_eval_df = run_evals(\n", + " dataframe=queries_df,\n", + " evaluators=[hallucination_evaluator, qa_evaluator],\n", + " provide_explanation=True,\n", + " concurrency=20,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 455 + }, + "id": "RaZwCSFico8I", + "outputId": "14ef12b7-af63-4ab1-ec14-c9b7d198ed54" + }, + "outputs": [], + "source": [ + "retrieved_documents_relevance_df = retrieved_documents_relevance_df.reset_index().set_index(\n", + " \"context.span_id\"\n", + ")\n", + "retrieved_documents_relevance_df" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 238 + }, + "id": "cCPWGtd0zPIl", + "outputId": "cc6e820f-7b58-4f84-dd9e-2a926f48b163" + }, + "outputs": [], + "source": [ + "hallucination_eval_df.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "zbH5A6cS6fd-" + }, + "source": [ + "## Log the Evals into Phoenix" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Xbs-D5W-zkDZ" + }, + "outputs": [], + "source": [ + "from phoenix.trace import SpanEvaluations\n", + "\n", + "px.Client().log_evaluations(\n", + " SpanEvaluations(eval_name=\"Hallucination\", dataframe=hallucination_eval_df),\n", + " SpanEvaluations(eval_name=\"QA Correctness\", dataframe=qa_eval_df),\n", + " SpanEvaluations(eval_name=\"Retrieval Relevance\", dataframe=retrieved_documents_relevance_df),\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "id": "O2-3FxRc4ukO", + "outputId": "1193ef27-1de3-400c-ed40-901784b879d1" + }, + "outputs": [], + "source": [ + "session.view()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "I5IS5dvf6lkD" + }, + "source": [ + "## Save the Trace and Evals" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true, + "id": "qG-RDMTsY5a9" + }, + "outputs": [], + "source": [ + "import os\n", + "\n", + "# Specify and Create the Directory for Trace Dataset\n", + "directory = \"fixtures\"\n", + "os.makedirs(directory, exist_ok=True)\n", + "\n", + "# Save the Trace Dataset\n", + "trace_id = px.Client().get_trace_dataset().save(directory=directory)" + ] + } + ], + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.19" + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "2c6eb911b6354343a46fca08b0b85f03": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "3363dcebe07847be99e7b41b3840d042": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_90412f8febda4ab1bb22fe56e2392274", + "IPY_MODEL_f7654cf0da3f47da801c21b947fda6da", + "IPY_MODEL_82e8f507e44841f88be5c66c91b27f48" + ], + "layout": "IPY_MODEL_8aeca790868b44babe415849afba77db" + } + }, + "44f75a3f6c48404990251849a9be33ca": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "59a894868a0344789e8458549f8aa894": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_99e5a5eeb1ef41d68f41bbd3cb98d544", + "placeholder": "​", + "style": "IPY_MODEL_d946d51d35344047add12d14c8445f46", + "value": " 500/500 (100.0%) | ⏳ 02:06<00:00 |  1.63it/s" + } + }, + "5f8024635af64b0a99855ae3adef558d": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "82e8f507e44841f88be5c66c91b27f48": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_d10322222cb3435fafdbbdc4d98e5059", + "placeholder": "​", + "style": "IPY_MODEL_d1d54e8e26a54881b1380b02159203ad", + "value": " 200/200 (100.0%) | ⏳ 01:13<00:00 |  2.21it/s" + } + }, + "8648201ffcab4023a1a54dde1b964b8b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_df7e8713256a48c2992f3dce61c702d1", + "IPY_MODEL_d9af102cb64142b3ac7ea6a75027e6a7", + "IPY_MODEL_59a894868a0344789e8458549f8aa894" + ], + "layout": "IPY_MODEL_2c6eb911b6354343a46fca08b0b85f03" + } + }, + "88978e8ddc704d8e9d4048095047615a": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "8aeca790868b44babe415849afba77db": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "90412f8febda4ab1bb22fe56e2392274": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_5f8024635af64b0a99855ae3adef558d", + "placeholder": "​", + "style": "IPY_MODEL_aed1f2822d114ce4be7dcddfbc2dfc20", + "value": "run_evals " + } + }, + "99e5a5eeb1ef41d68f41bbd3cb98d544": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "ae22d487bb1b442cad9018897a53e623": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "aea78f53909e44b195de5886df74fad1": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "aed1f2822d114ce4be7dcddfbc2dfc20": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "c6aab3035f8247b6a3e276b2e86fe79b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "d10322222cb3435fafdbbdc4d98e5059": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "d1d54e8e26a54881b1380b02159203ad": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "d946d51d35344047add12d14c8445f46": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "d9af102cb64142b3ac7ea6a75027e6a7": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_ae22d487bb1b442cad9018897a53e623", + "max": 500, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_c6aab3035f8247b6a3e276b2e86fe79b", + "value": 500 + } + }, + "df7e8713256a48c2992f3dce61c702d1": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_88978e8ddc704d8e9d4048095047615a", + "placeholder": "​", + "style": "IPY_MODEL_44f75a3f6c48404990251849a9be33ca", + "value": "run_evals " + } + }, + "f7654cf0da3f47da801c21b947fda6da": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_aea78f53909e44b195de5886df74fad1", + "max": 200, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_fcf5b6ce6ca94b48b80ed2aa8ced55d5", + "value": 200 + } + }, + "fcf5b6ce6ca94b48b80ed2aa8ced55d5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + } + } + } + }, + "nbformat": 4, + "nbformat_minor": 0 +}