Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add openssa.l2.resource.abstract.AbstractResource.present_full_answer(...) #189

Merged
merged 3 commits into from
Apr 27, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 2 additions & 7 deletions openssa/l2/reasoning/base/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,13 +23,8 @@ def reason(self, task: ATask, n_words: int = 1000) -> str:
task.result: str = ((self.lm.get_response(
prompt=RESOURCE_QA_CONSO_PROMPT_TEMPLATE.format(
question=task.ask, n_words=n_words,
resources_and_answers='\n\n'.join(
(f'INFORMATIONAL RESOURCE #{i + 1} (name: "{r.name}"):\n'
'\n'
f'INFORMATIONAL RESOURCE #{i + 1} OVERVIEW:\n{r.overview}\n'
'\n'
f'ANSWER/SOLUTION #{i + 1}:\n{r.answer(question=task.ask, n_words=n_words)}\n')
for i, r in enumerate(task.resources))))
resources_and_answers='\n\n'.join(r.present_full_answer(question=task.ask, n_words=n_words) # noqa: E501
for r in task.resources)))

if len(task.resources) > 1

Expand Down
22 changes: 7 additions & 15 deletions openssa/l2/reasoning/ooda/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,10 @@
from openssa.l2.task.abstract import ATask
from openssa.l2.task.status import TaskStatus

from ._prompts import OBSERVE_PROMPT_TEMPLATE
from ._prompts import ORIENT_PROMPT_TEMPLATE


type Observation = tuple[str, str, str]
type Observation = str


class OrientResult(TypedDict):
Expand All @@ -33,7 +33,7 @@ def reason(self, task: ATask, n_words: int = 1000) -> str:
- Orient & Decide whether such results are adequate for confident answer/solution
- Act to update Task's status and result
"""
observations: list[Observation] = self.observe(task=task, n_words=n_words)
observations: set[Observation] = self.observe(task=task, n_words=n_words)

# note: Orient & Decide steps are practically combined to economize LM calls
orient_result: OrientResult = self.orient(task=task, observations=observations, n_words=n_words)
Expand All @@ -43,21 +43,13 @@ def reason(self, task: ATask, n_words: int = 1000) -> str:

return task.result

def observe(self, task: ATask, n_words: int = 1000) -> list[Observation]:
def observe(self, task: ATask, n_words: int = 1000) -> set[Observation]:
"""Observe results from available Informational Resources."""
return [(r.name, r.overview, r.answer(question=task.ask, n_words=n_words))
for r in task.resources]
return {r.present_full_answer(question=task.ask, n_words=n_words) for r in task.resources}

def orient(self, task: ATask, observations: list[Observation], n_words: int = 1000) -> OrientResult:
def orient(self, task: ATask, observations: set[Observation], n_words: int = 1000) -> OrientResult:
"""Orient whether observed results are adequate for directly resolving Task."""
prompt: str = OBSERVE_PROMPT_TEMPLATE.format(
question=task.ask, n_words=n_words,
resources_and_answers='\n\n'.join((f'INFORMATIONAL RESOURCE #{i + 1} (name: "{name}"):\n'
'\n'
f'INFORMATIONAL RESOURCE #{i + 1} OVERVIEW:\n{overview}\n'
'\n'
f'ANSWER/SOLUTION #{i + 1}:\n{answer}\n')
for i, (name, overview, answer) in enumerate(observations)))
prompt: str = ORIENT_PROMPT_TEMPLATE.format(question=task.ask, n_words=n_words, observations='\n\n'.join(observations)) # noqa: E501
logger.debug(prompt)

def is_valid(orient_result_dict: OrientResult) -> bool:
Expand Down
6 changes: 3 additions & 3 deletions openssa/l2/reasoning/ooda/_prompts.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
OBSERVE_PROMPT_TEMPLATE: str = \
ORIENT_PROMPT_TEMPLATE: str = \
"""Assuming that the following question/problem/task is posed

```
{question}
```

and you have received various answers/solutions from different informational resources as detailed below,
and you have observed various answers/solutions from different informational resources as detailed below,
please evaluate whether you can answer/solve the posed question/problem/task confidently with concrete results.
If the question/problem/task mentions any RIGOROUS BASES/CRITERIA/DEFINITIONS for judgement,
the concrete results MUST RESPOND TO SUCH BASES/CRITERIA/DEFINITIONS for the answer/solution to be considered confident.
Expand All @@ -24,6 +24,6 @@


```
{resources_and_answers}
{observations}
```
""" # noqa: E122
15 changes: 15 additions & 0 deletions openssa/l2/resource/abstract.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,11 @@ def unique_name(self) -> str:
def name(self) -> str:
"""Return potentially non-unique, but informationally helpful name of Informational Resource."""

@cached_property
def full_name(self):
"""Return full name for presenting Informational Resource clearly, especially in large prompts."""
return f'INFORMATIONAL RESOURCE NAMED "{self.name}" (UNIQUELY NAMED "{self.unique_name}")'

@abstractmethod
def answer(self, question: str, n_words: int = 1000) -> str:
"""Answer question from Informational Resource."""
Expand All @@ -30,5 +35,15 @@ def overview(self) -> str:
"""Return overview of Informational Resource."""
return self.answer(question=RESOURCE_OVERVIEW_PROMPT_TEMPLATE.format(name=self.name))

def present_full_answer(self, question: str, n_words: int = 1000) -> str:
"""Present answer to posed question together with full name & overview of Informational Resource."""
return (f'{self.full_name}\n'
'has the following overview:\n'
f'```\n{self.overview}\n```\n'
'\n'
f'{self.full_name}\n'
'returns the following answer/solution:\n'
f'```\n{self.answer(question=question, n_words=n_words)}\n```\n')


AResource: TypeVar = TypeVar('AResource', bound=AbstractResource, covariant=False, contravariant=False)