Python: Enhance BinaryContent for OpenAI Responses API file handling

- Add can_read property to indicate data availability
- Add from_file() class method to create BinaryContent from file paths
- Set data_format='base64' in from_file() to prevent Unicode decode errors
- Update responses agent to handle BinaryContent with proper OpenAI file format
- Use correct OpenAI API structure: filename and file_data with data URI
- Keep BinaryContent completely provider-agnostic using generic properties
- Generate UUID-based filenames with appropriate mime-type extensions
- Add comprehensive tests for new BinaryContent functionality

This implementation enhances BinaryContent instead of introducing FileContent,
maintaining clean separation between generic content handling and
provider-specific logic. Fixes file upload support for OpenAI Responses API.

Author: ltwlf

python/semantic_kernel/agents/open_ai/responses_agent_thread_actions.py

@@ -2,6 +2,7 @@
 
 import asyncio
 import logging
+import uuid
 from collections.abc import AsyncIterable, Sequence
 from functools import reduce
 from typing import TYPE_CHECKING, Any, ClassVar, Literal, TypeVar, cast
@@ -31,6 +32,7 @@
 from semantic_kernel.connectors.ai.function_choice_behavior import FunctionChoiceBehavior
 from semantic_kernel.connectors.ai.open_ai.exceptions.content_filter_ai_exception import ContentFilterAIException
 from semantic_kernel.contents.annotation_content import AnnotationContent
+from semantic_kernel.contents.binary_content import BinaryContent
 from semantic_kernel.contents.chat_history import ChatHistory
 from semantic_kernel.contents.chat_message_content import CMC_ITEM_TYPES, ChatMessageContent
 from semantic_kernel.contents.function_call_content import FunctionCallContent
@@ -719,6 +721,25 @@ def _prepare_chat_history_for_request(
                             "call_id": content.call_id,
                         }
                         response_inputs.append(rfrc_dict)
+                    case BinaryContent() if content.can_read:
+                        # Generate filename with appropriate extension based on mime type
+                        extension = ""
+                        if content.mime_type == "application/pdf":
+                            extension = ".pdf"
+                        elif content.mime_type.startswith("image/"):
+                            extension = f".{content.mime_type.split('/')[-1]}"
+                        elif content.mime_type.startswith("text/"):
+                            extension = ".txt"
+                        filename = f"{uuid.uuid4()}{extension}"
+                        
+                        # Format according to OpenAI Responses API specification
+                        file_data_uri = f"data:{content.mime_type};base64,{content.data_string}"
+                        contents.append({
+                            "type": "input_file",
+                            "filename": filename,
+                            "file_data": file_data_uri,
+                        })
+                        response_inputs.append({"role": original_role, "content": contents})
 
         return response_inputs
 
@@ -975,4 +996,5 @@ def _get_tools(
 
         return tools
 
+
     # endregion

python/semantic_kernel/contents/binary_content.py

@@ -165,6 +165,48 @@ def mime_type(self, value: str):
         if self._data_uri:
             self._data_uri.mime_type = value
 
+    @property
+    def can_read(self) -> bool:
+        """Get whether the content can be read.
+        
+        Returns True if the content has data available for reading.
+        """
+        return self._data_uri is not None
+
+    @classmethod
+    def from_file(
+        cls: type[_T],
+        file_path: str | Path,
+        mime_type: str | None = None,
+    ) -> _T:
+        """Create a BinaryContent from a file.
+        
+        Args:
+            file_path: Path to the file to read
+            mime_type: The mime type of the file content
+            
+        Returns:
+            A BinaryContent instance with the file data
+            
+        Raises:
+            FileNotFoundError: If the file does not exist
+            ContentInitializationError: If the file cannot be read
+        """
+        path = Path(file_path)
+        if not path.exists():
+            raise FileNotFoundError(f"File not found: {file_path}")
+        if not path.is_file():
+            raise ContentInitializationError(f"Path is not a file: {file_path}")
+            
+        try:
+            with open(path, "rb") as file:
+                data = file.read()
+        except Exception as e:
+            raise ContentInitializationError(f"Failed to read file: {e}") from e
+            
+        return cls(data=data, mime_type=mime_type, data_format="base64", uri=str(path))
+
+
     def __str__(self) -> str:
         """Return the string representation of the content."""
         return self.data_uri if self._data_uri else str(self.uri)

python/tests/unit/contents/test_binary_content.py

@@ -1,10 +1,14 @@
 # Copyright (c) Microsoft. All rights reserved.
 
 
+import tempfile
+from pathlib import Path
+
 import pytest
 from numpy import array
 
 from semantic_kernel.contents.binary_content import BinaryContent
+from semantic_kernel.exceptions.content_exceptions import ContentInitializationError
 
 test_cases = [
     pytest.param(BinaryContent(uri="http://test_uri"), id="uri"),
@@ -111,3 +115,92 @@ def test_element_roundtrip(binary):
 @pytest.mark.parametrize("binary", test_cases)
 def test_to_dict(binary):
     assert binary.to_dict() == {"type": "binary", "binary": {"uri": str(binary)}}
+
+
+def test_can_read_with_data():
+    """Test can_read property returns True when data is available."""
+    binary = BinaryContent(data=b"test_data", mime_type="application/pdf")
+    assert binary.can_read is True
+
+
+def test_can_read_without_data():
+    """Test can_read property returns False when no data is available."""
+    binary = BinaryContent(uri="http://example.com/file.pdf")
+    assert binary.can_read is False
+
+
+def test_can_read_empty():
+    """Test can_read property returns False for empty BinaryContent."""
+    binary = BinaryContent()
+    assert binary.can_read is False
+
+
+def test_from_file_success():
+    """Test from_file class method successfully creates BinaryContent from a file."""
+    test_data = b"This is test file content"
+    with tempfile.NamedTemporaryFile(delete=False) as temp_file:
+        temp_file.write(test_data)
+        temp_file_path = temp_file.name
+
+    try:
+        binary = BinaryContent.from_file(temp_file_path, mime_type="application/pdf")
+        assert binary.data == test_data
+        assert binary.mime_type == "application/pdf"
+        assert binary.uri == temp_file_path
+        assert binary.can_read is True
+        # Verify data_string works (should be base64 encoded)
+        assert binary.data_string == "VGhpcyBpcyB0ZXN0IGZpbGUgY29udGVudA=="
+    finally:
+        Path(temp_file_path).unlink()
+
+
+def test_from_file_with_path_object():
+    """Test from_file class method works with Path objects."""
+    test_data = b"Path object test content"
+    with tempfile.NamedTemporaryFile(delete=False) as temp_file:
+        temp_file.write(test_data)
+        temp_path = Path(temp_file.name)
+
+    try:
+        binary = BinaryContent.from_file(temp_path, mime_type="text/plain")
+        assert binary.data == test_data
+        assert binary.mime_type == "text/plain"
+        assert binary.uri == str(temp_path)
+        # Verify data_string works (should be base64 encoded)
+        assert binary.data_string == "UGF0aCBvYmplY3QgdGVzdCBjb250ZW50"
+    finally:
+        temp_path.unlink()
+
+
+def test_from_file_binary_data():
+    """Test from_file handles binary data correctly without encoding errors."""
+    # Test with actual binary PDF-like data
+    test_data = b'%PDF-1.4\n%\xf6\xe4\xfc\xdf\n1 0 obj\n<<\n/Type /Catalog'
+    with tempfile.NamedTemporaryFile(delete=False, suffix='.pdf') as temp_file:
+        temp_file.write(test_data)
+        temp_file_path = temp_file.name
+
+    try:
+        binary = BinaryContent.from_file(temp_file_path, mime_type="application/pdf")
+        assert binary.data == test_data
+        assert binary.mime_type == "application/pdf"
+        assert binary.can_read is True
+        # Should not raise Unicode decode error
+        data_string = binary.data_string
+        assert isinstance(data_string, str)
+        assert len(data_string) > 0
+    finally:
+        Path(temp_file_path).unlink()
+
+
+def test_from_file_nonexistent():
+    """Test from_file raises FileNotFoundError for nonexistent files."""
+    with pytest.raises(FileNotFoundError, match="File not found"):
+        BinaryContent.from_file("/nonexistent/file.pdf")
+
+
+def test_from_file_directory():
+    """Test from_file raises ContentInitializationError for directories."""
+    with tempfile.TemporaryDirectory() as temp_dir:
+        with pytest.raises(ContentInitializationError, match="Path is not a file"):
+            BinaryContent.from_file(temp_dir)