From 6ffb5c1b376e5167544a4254b284ca7d5d4105da Mon Sep 17 00:00:00 2001 From: OceanLi <122793010+ohdearquant@users.noreply.github.com> Date: Tue, 26 Mar 2024 11:58:47 -0400 Subject: [PATCH 1/2] majority voting --- lionagi/core/direct/score.py | 79 ++++++++- lionagi/core/direct/vote.py | 29 ++++ lionagi/version.py | 2 +- notebooks/lion_agent_with_tool.ipynb | 2 +- notebooks/try_DSPy.ipynb | 241 +-------------------------- 5 files changed, 113 insertions(+), 240 deletions(-) create mode 100644 lionagi/core/direct/vote.py diff --git a/lionagi/core/direct/score.py b/lionagi/core/direct/score.py index dc8bce716..ff1d9aa98 100644 --- a/lionagi/core/direct/score.py +++ b/lionagi/core/direct/score.py @@ -1,5 +1,5 @@ from pydantic import Field - +import numpy as np from lionagi.libs import func_call, convert from ..prompt.prompt_template import ScoredTemplate from ..branch import Branch @@ -55,7 +55,7 @@ def __init__( "precision": num_digit if num_digit != 0 else None, } -async def score( +async def _score( sentence, instruction=None, score_range=(1, 10), @@ -80,6 +80,11 @@ async def score( tool_manager=None, **kwargs, ): + + if "temperature" not in kwargs: + kwargs["temperature"] = 0.1 + + instruction = instruction or "" branch = Branch( name=branch_name, @@ -116,3 +121,73 @@ async def score( ) return _template + +# async def group_score(sentence, *args, num_instances=5, **kwargs): +# sentences = [sentence for _ in range(num_instances)] + +# outs_ = await func_call.alcall(sentences, _score, *args, **kwargs) + +# return np.mean([i.answer for i in outs_]) + + +async def score( + sentence, + num_instances=1, + instruction=None, + score_range=(1, 10), + inclusive=True, + num_digit=0, + confidence_score=False, + reason=False, + retries=2, + delay=0.5, + backoff_factor=2, + default_value=None, + timeout=None, + branch_name=None, + system=None, + messages=None, + service=None, + sender=None, + llmconfig=None, + tools=None, + datalogger=None, + persist_path=None, + tool_manager=None, + return_template=True, + **kwargs, +): + async def _inner(i=0): + return await _score( + sentence=sentence, + instruction=instruction, + score_range=score_range, + inclusive=inclusive, + num_digit=num_digit, + confidence_score=confidence_score, + reason=reason, + retries=retries, + delay=delay, + backoff_factor=backoff_factor, + default_value=default_value, + timeout=timeout, + branch_name=branch_name, + system=system, + messages=messages, + service=service, + sender=sender, + llmconfig=llmconfig, + tools=tools, + datalogger=datalogger, + persist_path=persist_path, + tool_manager=tool_manager, + **kwargs + ) + + if num_instances == 1: + _out = await _inner() + return _out if return_template else _out.answer + + elif num_instances > 1: + _outs = await func_call.alcall(range(num_instances), _inner) + return _outs if return_template else np.mean([i.answer for i in _outs]) diff --git a/lionagi/core/direct/vote.py b/lionagi/core/direct/vote.py new file mode 100644 index 000000000..638af362b --- /dev/null +++ b/lionagi/core/direct/vote.py @@ -0,0 +1,29 @@ +from lionagi.libs import func_call +import numpy as np +from .score import score + +# for example, directive=predict + +async def vote(sentence, directive, num_generations=5, num_output=1, num_scorer=5, score_range=(0,100), num_digit=2, scorer_instruction=None, **kwargs): + + async def _inner(i): + out_ = await directive(sentence, **kwargs) + score_ = await score( + out_.answer, + context=sentence, + instruction=scorer_instruction, + score_range=score_range, + num_digit=num_digit, + num_instances=num_scorer, + return_template=False + ) + + out_.__setattr__('score', score_) + return out_ + + _outs = await func_call.alcall(list(range(num_generations)), _inner) + + top_index = np.argsort([i.score for i in _outs])[-num_output:] + final_output = list(np.array(_outs)[top_index]) + + return final_output[0] if len(final_output) == 1 else final_output diff --git a/lionagi/version.py b/lionagi/version.py index 1981e9572..e783a0b31 100644 --- a/lionagi/version.py +++ b/lionagi/version.py @@ -1 +1 @@ -__version__ = "0.0.310" \ No newline at end of file +__version__ = "0.0.311" \ No newline at end of file diff --git a/notebooks/lion_agent_with_tool.ipynb b/notebooks/lion_agent_with_tool.ipynb index 1444844aa..d32db3db9 100644 --- a/notebooks/lion_agent_with_tool.ipynb +++ b/notebooks/lion_agent_with_tool.ipynb @@ -330,7 +330,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.6" + "version": "3.12.1" } }, "nbformat": 4, diff --git a/notebooks/try_DSPy.ipynb b/notebooks/try_DSPy.ipynb index 88d75ff01..c0a50c385 100644 --- a/notebooks/try_DSPy.ipynb +++ b/notebooks/try_DSPy.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -14,7 +14,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -70,165 +70,9 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2024-03-25 18:40:16,426 - INFO - Starting to generate expert outputs...\n", - "2024-03-25 18:40:16,426 - INFO - Generating output for Expert 1...\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Generating expert outputs...\n", - "Generating output for Expert 1/5...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2024-03-25 18:40:36,472 - ERROR - Error generating output for Expert 1: field output with value Given the constraints of the max_tokens limit, a comprehensive evaluation of all four technologies in the depth required is not feasible. However, a brief overview can be provided. \n", - "\n", - "Artificial Intelligence (AI): Currently, AI is experiencing rapid growth with significant advancements in machine learning, natural language processing, and computer vision. Major players include Google, IBM, and OpenAI. The technology's applications across various industries, including healthcare, finance, and automotive, offer substantial efficiency improvements and cost savings. Challenges include ethical concerns and data privacy issues.\n", - "\n", - "Blockchain: This technology is still in its early stages but has potential beyond cryptocurrencies, such as in supply chain management and secure transactions. Companies like IBM and Ripple are at the forefront. Blockchain could enhance transparency and security for Acme's operations but faces scalability and regulatory hurdles.\n", - "\n", - "Quantum Computing: It's an emerging field with the promise to revolutionize computing by performing complex calculations much faster than current computers. Key players include Google, IBM, and D-Wave. Its impact could be transformative for cryptography and complex simulations but is currently limited by technological maturity and high costs.\n", - "\n", - "Biotechnology: Advancements in gene editing, synthetic biology, and personalized medicine are driving growth in this sector. Companies like CRISPR Therapeutics and Moderna are leading the way. Biotechnology offers significant opportunities for healthcare innovation but faces ethical, regulatory, and technical challenges.\n", - "\n", - "Recommendation: Acme Corporation should prioritize AI and Biotechnology based on their current development stage and potential impact on the company's core industries. These areas also offer clearer paths to market and integration into existing operations. A strategic allocation of resources towards these technologies, with a focus on partnerships and talent acquisition, could position Acme for future leadership in its industry. is not valid\n", - "2024-03-25 18:40:36,473 - INFO - Generating output for Expert 2...\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Generating output for Expert 2/5...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2024-03-25 18:40:50,555 - ERROR - Error generating output for Expert 2: field output with value Given the constraints and the depth of analysis required for each technology, a comprehensive evaluation cannot be fully provided within the 100 max tokens limit. However, here's a brief overview: \n", - "\n", - "Artificial Intelligence (AI): AI technology has seen rapid development, with major players including Google, IBM, and Microsoft. Its application in data analysis, automation, and customer service presents significant opportunities for Acme. Challenges include ethical concerns and regulatory scrutiny.\n", - "\n", - "Blockchain: Beyond cryptocurrency, blockchain offers secure, transparent transaction frameworks. It's in earlier stages compared to AI but has potential in supply chain management and secure transactions. Key players include IBM, Ethereum, and Hyperledger.\n", - "\n", - "Quantum Computing: Still in an experimental phase, quantum computing promises to revolutionize computing by drastically increasing processing power. Companies like Google, IBM, and startups like Rigetti are at the forefront. Its impact could be transformative but is also uncertain and long-term.\n", - "\n", - "Biotechnology: Advances in biotechnology could lead to breakthroughs in healthcare, agriculture, and materials science. It's a broad field with significant investment from both startups and established companies like Roche and Novartis. Regulatory hurdles and ethical debates are notable challenges.\n", - "\n", - "In summary, AI and Biotechnology are recommended for near-term investment due to their clearer path to market applications and existing impact on Acme's industries. Blockchain and Quantum Computing, while promising, present greater uncertainties and may benefit from a watch-and-wait approach with strategic investments in foundational research and partnerships. is not valid\n", - "2024-03-25 18:40:50,555 - INFO - Generating output for Expert 3...\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Generating output for Expert 3/5...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2024-03-25 18:41:07,898 - ERROR - Error generating output for Expert 3: field output with value Given the constraints of max_tokens, a comprehensive analysis of all four technologies cannot be fully provided. Here is an abbreviated assessment focusing on key insights:\n", - "\n", - "Artificial Intelligence (AI):\n", - "- Current State: Rapidly advancing, with significant investment across sectors.\n", - "- Major Players: Google, IBM, Microsoft.\n", - "- Market Growth: Expected to reach $190.61 billion by 2025.\n", - "- Applications: Predictive analytics, automation, customer service.\n", - "- Recommendation: High priority due to its wide applicability and potential to enhance operational efficiency.\n", - "\n", - "Blockchain:\n", - "- Current State: Beyond initial hype, finding niche applications.\n", - "- Major Players: IBM, Ethereum, Ripple.\n", - "- Market Growth: Projected to grow to $39.7 billion by 2025.\n", - "- Applications: Supply chain transparency, smart contracts.\n", - "- Recommendation: Medium priority, consider for specific use cases in supply chain and data integrity.\n", - "\n", - "Quantum Computing:\n", - "- Current State: Still in early experimental phase.\n", - "- Major Players: Google, IBM, Rigetti.\n", - "- Market Growth: Nascent, with significant potential disruptions in the long term.\n", - "- Applications: Complex problem solving, cryptography.\n", - "- Recommendation: Low priority for immediate investment, monitor developments for future opportunities.\n", - "\n", - "Biotechnology:\n", - "- Current State: Rapid advancements in genetic engineering and CRISPR technology.\n", - "- Major Players: CRISPR Therapeutics, Editas Medicine, Illumina.\n", - "- Market Growth: Expected to exceed $775 billion by 2024.\n", - "- Applications: Drug discovery, personalized medicine.\n", - "- Recommendation: High priority, especially for investments in health-related industries.\n", - "\n", - "Acme should prioritize AI and Biotechnology for their broad applicability and potential to offer competitive advantages. Blockchain and Quantum Computing should be monitored for future investment opportunities. is not valid\n", - "2024-03-25 18:41:07,899 - INFO - Generating output for Expert 4...\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Generating output for Expert 4/5...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2024-03-25 18:41:11,284 - ERROR - Error generating output for Expert 4: field output with value Due to the max_tokens limit, a comprehensive analysis of all four technologies cannot be provided in a single response. Please adjust the parameters or request analysis for one technology at a time. is not valid\n", - "2024-03-25 18:41:11,285 - INFO - Generating output for Expert 5...\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Generating output for Expert 5/5...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2024-03-25 18:41:36,475 - ERROR - Error generating output for Expert 5: field output with value Given the constraints of the prompt and the detailed analysis required for each of the four emerging technologies - artificial intelligence (AI), blockchain, quantum computing, and biotechnology - a comprehensive evaluation within this response's limit is challenging. However, a high-level overview can be provided: \n", - "\n", - "1. **Artificial Intelligence (AI)**: Currently at an advanced stage of development with significant investments from both startups and established tech giants. AI has broad applications across industries including automation, data analysis, and customer service enhancements. Investment in AI can provide Acme with tools for predictive analytics, process optimization, and personalized customer experiences. Key challenges include ethical considerations and data privacy.\n", - "\n", - "2. **Blockchain**: Beyond its initial financial applications, blockchain technology offers potential for secure, transparent transactions and data management systems. It is particularly relevant for supply chain management, intellectual property rights, and secure voting systems. The technology is still evolving, with scalability and regulatory acceptance as major hurdles.\n", - "\n", - "3. **Quantum Computing**: Although in its nascent stages, quantum computing promises to revolutionize industries by making calculations and data processing exponentially faster than current computers. Its potential impact on cryptography, material science, and complex system simulations is immense. The technology is, however, years away from commercial viability and requires significant investment in research and development.\n", - "\n", - "4. **Biotechnology**: Offers transformative potential in healthcare, agriculture, and environmental conservation. Advances in genetic engineering, personalized medicine, and biofuels are rapidly evolving. The industry faces regulatory hurdles and ethical debates but holds significant promise for addressing global challenges.\n", - "\n", - "Given the current state of these technologies, Acme Corporation should consider prioritizing investment in AI and biotechnology for immediate applicability and market opportunities. Blockchain and quantum computing, while promising, represent longer-term investments. Acme should allocate resources towards building expertise and partnerships in AI and biotechnology, while keeping a close watch on developments in blockchain and quantum computing for future investment opportunities. is not valid\n", - "2024-03-25 18:41:36,477 - ERROR - Error selecting the most relevant expert: 'experts_outputs'\n" - ] - }, - { - "ename": "ValueError", - "evalue": "too many values to unpack (expected 3)", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[3], line 228\u001b[0m\n\u001b[1;32m 220\u001b[0m moe \u001b[38;5;241m=\u001b[39m MixtureOfExperts(\n\u001b[1;32m 221\u001b[0m num_experts\u001b[38;5;241m=\u001b[39mnum_experts,\n\u001b[1;32m 222\u001b[0m min_iterations\u001b[38;5;241m=\u001b[39mmin_iterations,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 225\u001b[0m exploration_rate\u001b[38;5;241m=\u001b[39mexploration_rate\n\u001b[1;32m 226\u001b[0m )\n\u001b[1;32m 227\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m iteration \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(moe\u001b[38;5;241m.\u001b[39mmin_iterations):\n\u001b[0;32m--> 228\u001b[0m final_output, selected_expert_index, intrinsic_reward \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m moe\u001b[38;5;241m.\u001b[39mgenerate_expert_outputs(context, prompt, max_tokens, guidance)\n\u001b[1;32m 230\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mIteration \u001b[39m\u001b[38;5;132;01m{\u001b[39;00miteration\u001b[38;5;241m+\u001b[39m\u001b[38;5;241m1\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m - Selected Expert: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mselected_expert_index\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m, Intrinsic Reward: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mintrinsic_reward\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 231\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mExpert Values:\u001b[39m\u001b[38;5;124m\"\u001b[39m, moe\u001b[38;5;241m.\u001b[39mexpert_values)\n", - "\u001b[0;31mValueError\u001b[0m: too many values to unpack (expected 3)" - ] - } - ], + "outputs": [], "source": [ "class MixtureOfExperts:\n", " def __init__(self, num_experts: int = 4, min_iterations: int = 4, learning_rate: float = 0.1, discount_factor: float = 0.99, exploration_rate: float = 0.2):\n", @@ -495,82 +339,7 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2024-03-25 18:34:46,775 - INFO - Starting to generate expert outputs...\n", - "2024-03-25 18:34:46,776 - INFO - Generating output for Expert 1...\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Generating expert outputs...\n", - "Generating output for Expert 1/3...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2024-03-25 18:34:49,064 - ERROR - Error generating output for Expert 1: field output with value Expert 2: Not necessarily. The presence of a ghost doesn't always mean harm. It's important to understand the context. is not valid\n", - "2024-03-25 18:34:49,065 - INFO - Generating output for Expert 2...\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Generating output for Expert 2/3...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2024-03-25 18:34:51,032 - ERROR - Error generating output for Expert 2: field output with value Not necessarily. It depends on the nature of the ghost and its intentions. Some believe ghosts are simply lost souls seeking help. is not valid\n", - "2024-03-25 18:34:51,033 - INFO - Generating output for Expert 3...\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Generating output for Expert 3/3...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2024-03-25 18:34:55,028 - ERROR - Error generating output for Expert 3: field output with value Not necessarily. The presence of a ghost in a story doesn't always mean harm. It could be a friendly or misunderstood spirit. is not valid\n", - "2024-03-25 18:34:56,417 - ERROR - Error evaluating intrinsic reward: 'dict' object has no attribute 'intrinsic_reward'\n", - "2024-03-25 18:34:56,418 - INFO - All expert outputs have been generated and the most relevant expert has been selected.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Expert output generation complete!\n" - ] - }, - { - "ename": "IndexError", - "evalue": "list index out of range", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mIndexError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[7], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m a \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mawait\u001b[39;00m moe\u001b[38;5;241m.\u001b[39mgenerate_expert_outputs(context, prompt, max_tokens, guidance)\n", - "Cell \u001b[0;32mIn[6], line 156\u001b[0m, in \u001b[0;36mMixtureOfExperts.generate_expert_outputs\u001b[0;34m(self, context, prompt, max_tokens, guidance)\u001b[0m\n\u001b[1;32m 153\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mExpert output generation complete!\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 154\u001b[0m logging\u001b[38;5;241m.\u001b[39minfo(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mAll expert outputs have been generated and the most relevant expert has been selected.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m--> 156\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexpert_outputs\u001b[49m\u001b[43m[\u001b[49m\u001b[43mselected_expert_index\u001b[49m\u001b[43m]\u001b[49m, selected_expert_index, intrinsic_reward\n", - "\u001b[0;31mIndexError\u001b[0m: list index out of range" - ] - } - ], + "outputs": [], "source": [ "a = await moe.generate_expert_outputs(context, prompt, max_tokens, guidance)" ] From ef88c96e4027e2540fb1d9f2580a49e18608c3f3 Mon Sep 17 00:00:00 2001 From: OceanLi <122793010+ohdearquant@users.noreply.github.com> Date: Tue, 26 Mar 2024 12:48:56 -0400 Subject: [PATCH 2/2] updated notebooks --- lionagi/core/direct/__init__.py | 2 + lionagi/core/direct/select.py | 6 +- lionagi/core/direct/vote.py | 3 +- notebooks/lion_direct.ipynb | 281 ++++++++++++++++++++++---------- 4 files changed, 202 insertions(+), 90 deletions(-) diff --git a/lionagi/core/direct/__init__.py b/lionagi/core/direct/__init__.py index 555f2ea87..28565896d 100644 --- a/lionagi/core/direct/__init__.py +++ b/lionagi/core/direct/__init__.py @@ -1,9 +1,11 @@ from .predict import predict from .select import select from .score import score +from .vote import vote __all__ = [ "predict", "select", "score", + "vote" ] diff --git a/lionagi/core/direct/select.py b/lionagi/core/direct/select.py index 894c50098..6468ad328 100644 --- a/lionagi/core/direct/select.py +++ b/lionagi/core/direct/select.py @@ -17,7 +17,6 @@ def __init__( self, sentence=None, choices=None, - num_choices=1, instruction=None, reason=False, confidence_score=False, @@ -27,7 +26,7 @@ def __init__( self.sentence = sentence self.choices = choices - self.task = f"select {num_choices} item(s), from provided choices {choices}." + self.task = f"select 1 item, from provided choices {choices}." if instruction: self.task += f"objetive {instruction}." @@ -41,7 +40,6 @@ def __init__( async def select( sentence, choices=None, - num_choices=1, instruction=None, confidence_score=False, reason=False, @@ -79,7 +77,6 @@ async def select( _template = SelectTemplate( sentence=sentence, choices=choices, - num_choices=num_choices, instruction=instruction, confidence_score=confidence_score, reason=reason, @@ -97,6 +94,7 @@ async def select( ) ans = _template.answer + if ans not in _template.choices: _template.answer = StringMatch.choose_most_similar(ans, _template.choices) diff --git a/lionagi/core/direct/vote.py b/lionagi/core/direct/vote.py index 638af362b..0dd2f7f3f 100644 --- a/lionagi/core/direct/vote.py +++ b/lionagi/core/direct/vote.py @@ -1,10 +1,11 @@ from lionagi.libs import func_call import numpy as np +from .predict import predict from .score import score # for example, directive=predict -async def vote(sentence, directive, num_generations=5, num_output=1, num_scorer=5, score_range=(0,100), num_digit=2, scorer_instruction=None, **kwargs): +async def vote(sentence, directive=predict, num_generations=5, num_output=1, num_scorer=5, score_range=(0,100), num_digit=2, scorer_instruction=None, **kwargs): async def _inner(i): out_ = await directive(sentence, **kwargs) diff --git a/notebooks/lion_direct.ipynb b/notebooks/lion_direct.ipynb index f9a57f0d9..13a48c322 100644 --- a/notebooks/lion_direct.ipynb +++ b/notebooks/lion_direct.ipynb @@ -6,6 +6,7 @@ "metadata": {}, "outputs": [], "source": [ + "from IPython.display import Markdown\n", "from lionagi import direct" ] }, @@ -32,8 +33,11 @@ "outputs": [ { "data": { + "text/markdown": [ + "The shark tries to escape into deeper waters." + ], "text/plain": [ - "'The big white shark tries to swim faster to escape.'" + "" ] }, "execution_count": 3, @@ -42,7 +46,8 @@ } ], "source": [ - "await direct.predict(sentence)" + "out_ = await direct.predict(sentence)\n", + "Markdown(out_.answer)" ] }, { @@ -52,10 +57,11 @@ "outputs": [ { "data": { + "text/markdown": [ + "The shark tries to swim away as fast as it can. The blue whale continues the pursuit, showcasing its surprising agility." + ], "text/plain": [ - "{'answer': 'The big white shark tries to swim faster to escape, but the blue whale is persistent in its pursuit.',\n", - " 'reason': 'Predatory or competitive interactions in the ocean often involve chase sequences, where the predator pursues its target persistently, and the prey tries to escape.',\n", - " 'confidence_score': 0.75}" + "" ] }, "execution_count": 4, @@ -64,20 +70,14 @@ } ], "source": [ - "await direct.predict(\n", + "out_ = await direct.predict(\n", " sentence,\n", " num_sentences=2,\n", " reason=True,\n", - " confidence_score=True,\n", " retry_kwargs={\"timeout\": 15},\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 2. select" + ")\n", + "\n", + "Markdown(out_.answer)" ] }, { @@ -87,8 +87,11 @@ "outputs": [ { "data": { + "text/markdown": [ + "Based on the given sentence, it's logical to predict that the shark would attempt to escape the threat posed by the larger blue whale. The continuation of the chase by the blue whale emphasizes the size and speed capabilities of blue whales, despite their massive size." + ], "text/plain": [ - "'deep ocean'" + "" ] }, "execution_count": 5, @@ -96,40 +99,66 @@ "output_type": "execute_result" } ], + "source": [ + "Markdown(out_.reason)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2. select" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "deep ocean\n" + ] + } + ], "source": [ "choices = [\"funny\", \"catch\", \"apple\", \"deep ocean\"]\n", "\n", - "await direct.select(sentence, choices=choices)" + "out_ = await direct.select(sentence, choices=choices)\n", + "print(out_.answer)" ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, "metadata": {}, "outputs": [ { "data": { + "text/markdown": [ + "deep ocean" + ], "text/plain": [ - "{'answer': 'deep ocean',\n", - " 'reason': \"The context involves a blue whale and a big white shark, which are both deep-sea creatures. The 'deep ocean' choice is directly related to their natural habitat, making it an intriguing and fitting selection for the setup.\",\n", - " 'confidence_score': '0.95'}" + "" ] }, - "execution_count": 6, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "await direct.select(\n", + "out_ = await direct.select(\n", " sentence,\n", " choices=choices,\n", " num_choices=2,\n", " objective=\"find most weird for the setup\",\n", - " reason=True,\n", - " confidence_score=True,\n", " temperature=0.45,\n", - ")" + ")\n", + "\n", + "Markdown(out_.answer)" ] }, { @@ -141,118 +170,200 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 8, "metadata": {}, "outputs": [ { - "data": { - "text/plain": [ - "1" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" + "name": "stdout", + "output_type": "stream", + "text": [ + "5.0\n" + ] } ], "source": [ - "await direct.score(sentence)" + "out_ = await direct.score(sentence)\n", + "print(out_.answer)" ] }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 9, "metadata": {}, "outputs": [ { - "data": { - "text/plain": [ - "{'score': 85.0,\n", - " 'reason': \"It's unusual for a blue whale, which primarily feeds on tiny krill, to chase a big white shark, as this behavior deviates significantly from their known peaceful and solitary nature.\",\n", - " 'confidence_score': 0.9}" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" + "name": "stdout", + "output_type": "stream", + "text": [ + "75.0\n", + "The scenario of a blue whale chasing a big white shark is quite unusual and not typical behavior observed in nature, as blue whales primarily feed on tiny krill and are not known to chase sharks. This makes the situation relatively high on the weirdness scale.\n" + ] } ], "source": [ - "await direct.score(\n", + "out_ = await direct.score(\n", " sentence,\n", " instruction=\"rate weirdness\",\n", " reason=True,\n", " score_range=(1, 100),\n", " num_digit=1,\n", - " confidence_score=True,\n", - ")" + ")\n", + "\n", + "print(out_.answer)\n", + "print(out_.reason)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "### 4. sentiment" + "### 4. vote" ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 10, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'neutral'" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "await direct.sentiment(sentence)" + "from lionagi.core.direct import vote, predict\n", + "\n", + "sentence = \"\"\"\n", + "Why did the blue whale and the big shark start dancing at the bar? Because they heard the bartender was serving \"sea-riously\" good fin-tunes! And then they stopped dancing... because the octopus DJ said it was time for a squid break!\n", + "\"\"\"" ] }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 11, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0.5" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "await direct.sentiment(sentence, to_type=\"num\")" + "# this code made 10 generations of \"predict\" - 10 LLM calls\n", + "# each answer got scored by 10 scorers, - 100 LLM calls\n", + "# let's check the top 5 generations\n", + "\n", + "out_ = await vote(\n", + " sentence, \n", + " directive=predict, \n", + " num_sentences=3, \n", + " num_generations=10, \n", + " num_output=5, \n", + " num_scorer=10, \n", + " score_range=(0,10), \n", + " num_digits=2, \n", + " scorer_instruction=\"rate humor highly critically\",\n", + " temperature=0.4\n", + ")" ] }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 12, "metadata": {}, "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Score 7.5\n" + ] + }, { "data": { + "text/markdown": [ + "So, the blue whale and the big shark decided to take a break and order some krill cocktails. Meanwhile, the octopus DJ started playing some clam jams, getting the whole bar in a bubbly mood. After their break, the whale and the shark were ready to dive back into the dance floor, showing off their smooth jellyfish jive." + ], "text/plain": [ - "'awesome 😎'" + "" ] }, - "execution_count": 11, "metadata": {}, - "output_type": "execute_result" + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Score 7.5\n" + ] + }, + { + "data": { + "text/markdown": [ + "But the party didn't end there. Once the squid break was over, the blue whale, the big shark, and all their sea friends got back on the dance floor, ready to dive into more oceanic beats. The night was young, and the sea creatures were determined to make a splash with their dance moves!" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Score 7.65\n" + ] + }, + { + "data": { + "text/markdown": [ + "So, the blue whale and the big shark took a break, sipping on their ocean-inspired cocktails, chatting about the coral reefs and the latest in aquatic fashion. Meanwhile, the octopus DJ was busy setting up his next set, promising to bring even more waves to the dance floor. Everyone at the bar was eagerly waiting, knowing that when the music started again, it would be a splash hit." + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Score 7.9\n" + ] + }, + { + "data": { + "text/markdown": [ + "So, the whale and the shark took a seat at the bar, sipping on their ocean-inspired cocktails, chatting about the current events in the deep blue. They were soon joined by a group of jellyfish, who were glowing with excitement to join the underwater party. The night was filled with laughter, bubbles, and the best sea-themed music, making it an unforgettable evening under the sea." + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Score 8.25\n" + ] + }, + { + "data": { + "text/markdown": [ + "So, the blue whale and the big shark grabbed a drink and watched the jellyfish do their jelly wobble on the dance floor. It was a sight to sea, making everyone at the bar laugh and cheer. Eventually, the octopus DJ returned, and the party continued with even more sea-riously good fin-tunes." + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" } ], "source": [ - "await direct.sentiment(sentence, choices=[\"awesome 😎\", \"hmmm 🤔\", \"mehhh 😭\"])" + "from IPython.display import Markdown\n", + "\n", + "for i in range(5):\n", + " print(\"Score\", out_[i].score)\n", + " display(Markdown(out_[i].answer))" ] } ], @@ -272,7 +383,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.6" + "version": "3.12.1" } }, "nbformat": 4,