Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Verify multimodal fix #1054

Closed
wants to merge 2 commits into from
Closed

Verify multimodal fix #1054

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a3f2186
Fix compatibility with custom dicts for multimodal message content
Oct 8, 2024
f34d108
Fixed up type hints
ivanleomk Oct 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 12 additions & 3 deletions instructor/multimodal.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,19 +75,23 @@ def to_openai(self) -> dict[str, Any]:


def convert_contents(
contents: Union[list[Union[str, Image]], str, Image], # noqa: UP007
contents: Union[ # noqa: UP007
list[Union[str, dict[str, Any], Image]], str, dict[str, Any], Image # noqa: UP007
],
mode: Mode,
) -> Union[str, list[dict[str, Any]]]: # noqa: UP007
"""Convert content items to the appropriate format based on the specified mode."""
if isinstance(contents, str):
return contents
if isinstance(contents, Image):
if isinstance(contents, Image) or isinstance(contents, dict):
contents = [contents]

converted_contents: list[dict[str, Union[str, Image]]] = [] # noqa: UP007
for content in contents:
if isinstance(content, str):
converted_contents.append({"type": "text", "text": content})
elif isinstance(content, dict):
converted_contents.append(content)
elif isinstance(content, Image):
if mode in {Mode.ANTHROPIC_JSON, Mode.ANTHROPIC_TOOLS}:
converted_contents.append(content.to_anthropic())
Expand All @@ -101,7 +105,12 @@ def convert_contents(


def convert_messages(
messages: list[dict[str, Union[str, list[Union[str, Image]]]]], # noqa: UP007
messages: list[
dict[
str,
Union[list[Union[str, dict[str, Any], Image]], str, dict[str, Any], Image], # noqa: UP007
]
], # noqa: UP007
mode: Mode,
) -> list[dict[str, Any]]:
"""Convert messages to the appropriate format based on the specified mode."""
Expand Down
10 changes: 10 additions & 0 deletions tests/test_multimodal.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -180,3 +180,13 @@ def test_convert_contents_anthropic_mode():
assert converted[1]["type"] == "image"
assert converted[1]["source"]["type"] == "base64"
assert converted[1]["source"]["media_type"] == "image/png"


def test_convert_contents_custom_dict():
contents = {
"type": "image_url",
"image_url": {"url": f"data:image/png;base64,base64_img"},
}
converted = list(convert_contents(contents, Mode.TOOLS))
assert len(converted) == 1
assert converted == [contents]
Loading