Skip to content

Commit 002d623

Browse files
ccurmeccurme
authored
feat: (core, standard-tests) support PDF inputs in ToolMessages (#33183)
1 parent 34f8031 commit 002d623

File tree

5 files changed

+222
-1
lines changed

5 files changed

+222
-1
lines changed

‎libs/core/langchain_core/tools/base.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,7 @@
8282
"search_result",
8383
"custom_tool_call_output",
8484
"document",
85+
"file",
8586
)
8687

8788

‎libs/partners/anthropic/tests/integration_tests/test_standard.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,10 @@ def supports_pdf_inputs(self) -> bool:
4141
def supports_image_tool_message(self) -> bool:
4242
return True
4343

44+
@property
45+
def supports_pdf_tool_message(self) -> bool:
46+
return True
47+
4448
@property
4549
def supports_anthropic_inputs(self) -> bool:
4650
return True

‎libs/partners/openai/tests/integration_tests/chat_models/test_responses_standard.py‎

Lines changed: 52 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,13 @@
11
"""Standard LangChain interface tests for Responses API"""
22

3+
import base64
34
from pathlib import Path
45
from typing import cast
56

7+
import httpx
68
import pytest
79
from langchain_core.language_models import BaseChatModel
8-
from langchain_core.messages import AIMessage
10+
from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
911

1012
from langchain_openai import ChatOpenAI
1113
from tests.integration_tests.chat_models.test_base_standard import TestOpenAIStandard
@@ -52,6 +54,55 @@ def invoke_with_reasoning_output(self, *, stream: bool = False) -> AIMessage:
5254
input_ = "What was the 3rd highest building in 2000?"
5355
return _invoke(llm, input_, stream)
5456

57+
@property
58+
def supports_pdf_tool_message(self) -> bool:
59+
# OpenAI requires a filename for PDF inputs
60+
# For now, we test with filename in OpenAI-specific tests
61+
return False
62+
63+
def test_openai_pdf_tool_messages(self, model: BaseChatModel) -> None:
64+
"""Test that the model can process PDF inputs in ToolMessages."""
65+
url = "https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf"
66+
pdf_data = base64.b64encode(httpx.get(url).content).decode("utf-8")
67+
68+
tool_message = ToolMessage(
69+
content=[
70+
{
71+
"type": "file",
72+
"source_type": "base64",
73+
"data": pdf_data,
74+
"mime_type": "application/pdf",
75+
"filename": "my-pdf", # specify filename
76+
},
77+
],
78+
tool_call_id="1",
79+
name="random_pdf",
80+
)
81+
82+
messages = [
83+
HumanMessage(
84+
"Get a random PDF using the tool and relay the title verbatim."
85+
),
86+
AIMessage(
87+
[],
88+
tool_calls=[
89+
{
90+
"type": "tool_call",
91+
"id": "1",
92+
"name": "random_pdf",
93+
"args": {},
94+
}
95+
],
96+
),
97+
tool_message,
98+
]
99+
100+
def random_pdf() -> str:
101+
"""Return a random PDF."""
102+
return ""
103+
104+
_ = model.bind_tools([random_pdf]).invoke(messages)
105+
55106

56107
def _invoke(llm: ChatOpenAI, input_: str, stream: bool) -> AIMessage:
57108
if stream:

‎libs/standard-tests/langchain_tests/integration_tests/chat_models.py‎

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -521,6 +521,39 @@ def supports_anthropic_inputs(self) -> bool:
521521
def supports_image_tool_message(self) -> bool:
522522
return False
523523
524+
.. dropdown:: supports_pdf_tool_message
525+
526+
Boolean property indicating whether the chat model supports ToolMessages
527+
that include PDF content, i.e.,
528+
529+
.. code-block:: python
530+
531+
ToolMessage(
532+
content=[
533+
{
534+
"type": "file",
535+
"source_type": "base64",
536+
"data": pdf_data,
537+
"mime_type": "application/pdf",
538+
},
539+
],
540+
tool_call_id="1",
541+
name="random_pdf",
542+
)
543+
544+
(standard format).
545+
546+
If set to ``True``, the chat model will be tested with message sequences that
547+
include ToolMessages of this form.
548+
549+
Example:
550+
551+
.. code-block:: python
552+
553+
@property
554+
def supports_pdf_tool_message(self) -> bool:
555+
return False
556+
524557
.. dropdown:: supported_usage_metadata_details
525558
526559
Property controlling what usage metadata details are emitted in both invoke
@@ -2707,6 +2740,95 @@ def random_image() -> str:
27072740

27082741
_ = model.bind_tools([random_image]).invoke(messages)
27092742

2743+
def test_pdf_tool_message(self, model: BaseChatModel) -> None:
2744+
"""Test that the model can process ToolMessages with PDF inputs.
2745+
2746+
This test should be skipped if the model does not support messages of the
2747+
form:
2748+
2749+
.. code-block:: python
2750+
2751+
ToolMessage(
2752+
content=[
2753+
{
2754+
"type": "file",
2755+
"source_type": "base64",
2756+
"data": pdf_data,
2757+
"mime_type": "application/pdf",
2758+
},
2759+
],
2760+
tool_call_id="1",
2761+
name="random_pdf",
2762+
)
2763+
2764+
containing PDF content blocks in standard format.
2765+
2766+
This test can be skipped by setting the ``supports_pdf_tool_message`` property
2767+
to False (see Configuration below).
2768+
2769+
.. dropdown:: Configuration
2770+
2771+
To disable this test, set ``supports_pdf_tool_message`` to False in your
2772+
test class:
2773+
2774+
.. code-block:: python
2775+
2776+
class TestMyChatModelIntegration(ChatModelIntegrationTests):
2777+
@property
2778+
def supports_pdf_tool_message(self) -> bool:
2779+
return False
2780+
2781+
.. dropdown:: Troubleshooting
2782+
2783+
If this test fails, check that the model can correctly handle messages
2784+
with PDF content blocks in ToolMessages, specifically base64-encoded
2785+
PDFs. Otherwise, set the ``supports_pdf_tool_message`` property to
2786+
False.
2787+
2788+
"""
2789+
if not self.supports_pdf_tool_message:
2790+
pytest.skip("Model does not support PDF tool message.")
2791+
2792+
url = "https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf"
2793+
pdf_data = base64.b64encode(httpx.get(url).content).decode("utf-8")
2794+
2795+
tool_message = ToolMessage(
2796+
content=[
2797+
{
2798+
"type": "file",
2799+
"source_type": "base64",
2800+
"data": pdf_data,
2801+
"mime_type": "application/pdf",
2802+
},
2803+
],
2804+
tool_call_id="1",
2805+
name="random_pdf",
2806+
)
2807+
2808+
messages = [
2809+
HumanMessage(
2810+
"Get a random PDF using the tool and relay the title verbatim."
2811+
),
2812+
AIMessage(
2813+
[],
2814+
tool_calls=[
2815+
{
2816+
"type": "tool_call",
2817+
"id": "1",
2818+
"name": "random_pdf",
2819+
"args": {},
2820+
}
2821+
],
2822+
),
2823+
tool_message,
2824+
]
2825+
2826+
def random_pdf() -> str:
2827+
"""Return a random PDF."""
2828+
return ""
2829+
2830+
_ = model.bind_tools([random_pdf]).invoke(messages)
2831+
27102832
def test_anthropic_inputs(self, model: BaseChatModel) -> None:
27112833
"""Test that model can process Anthropic-style message histories.
27122834

‎libs/standard-tests/langchain_tests/unit_tests/chat_models.py‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -231,6 +231,16 @@ def supports_image_tool_message(self) -> bool:
231231
"""
232232
return False
233233

234+
@property
235+
def supports_pdf_tool_message(self) -> bool:
236+
"""Supports PDF ToolMessages.
237+
238+
(bool) whether the chat model supports ToolMessages that include PDF
239+
content.
240+
241+
"""
242+
return False
243+
234244
@property
235245
def enable_vcr_tests(self) -> bool:
236246
"""(bool) whether to enable VCR tests for the chat model.
@@ -645,6 +655,39 @@ def supports_anthropic_inputs(self) -> bool:
645655
def supports_image_tool_message(self) -> bool:
646656
return False
647657
658+
.. dropdown:: supports_pdf_tool_message
659+
660+
Boolean property indicating whether the chat model supports ToolMessages
661+
that include PDF content, i.e.,
662+
663+
.. code-block:: python
664+
665+
ToolMessage(
666+
content=[
667+
{
668+
"type": "file",
669+
"source_type": "base64",
670+
"data": pdf_data,
671+
"mime_type": "application/pdf",
672+
},
673+
],
674+
tool_call_id="1",
675+
name="random_pdf",
676+
)
677+
678+
(standard format).
679+
680+
If set to ``True``, the chat model will be tested with message sequences that
681+
include ToolMessages of this form.
682+
683+
Example:
684+
685+
.. code-block:: python
686+
687+
@property
688+
def supports_pdf_tool_message(self) -> bool:
689+
return False
690+
648691
.. dropdown:: supported_usage_metadata_details
649692
650693
Property controlling what usage metadata details are emitted in both ``invoke``

0 commit comments

Comments
 (0)