diff --git a/cookbook/openai_v1_cookbook.ipynb b/cookbook/openai_v1_cookbook.ipynb index 56e67ab0f06b4..3fc98346f2a3f 100644 --- a/cookbook/openai_v1_cookbook.ipynb +++ b/cookbook/openai_v1_cookbook.ipynb @@ -20,27 +20,74 @@ "!pip install \"openai>=1\"" ] }, + { + "cell_type": "code", + "execution_count": 1, + "id": "c3e067ce-7a43-47a7-bc89-41f1de4cf136", + "metadata": {}, + "outputs": [], + "source": [ + "from langchain.chat_models import ChatOpenAI\n", + "from langchain.schema.messages import HumanMessage, SystemMessage" + ] + }, { "cell_type": "markdown", - "id": "71c34763-d1e7-4b9a-a9d7-3e4cc0dfc2c4", + "id": "fa7e7e95-90a1-4f73-98fe-10c4b4e0951b", "metadata": {}, "source": [ - "## [JSON mode](https://platform.openai.com/docs/guides/text-generation/json-mode)\n", - "\n", - "Constrain the model to only generate valid JSON. Note that you must include a system message with instructions to use JSON for this mode to work.\n", + "## [Vision](https://platform.openai.com/docs/guides/vision)\n", "\n", - "Only works with certain models. " + "OpenAI released multi-modal models, which can take a sequence of text and images as input." ] }, { "cell_type": "code", - "execution_count": 1, - "id": "c3e067ce-7a43-47a7-bc89-41f1de4cf136", + "execution_count": 2, + "id": "1c8c3965-d3c9-4186-b5f3-5e67855ef916", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "AIMessage(content='The image appears to be a diagram illustrating the architecture or components of a software system')" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "from langchain.chat_models import ChatOpenAI\n", - "from langchain.schema.messages import HumanMessage, SystemMessage" + "chat = ChatOpenAI(model=\"gpt-4-vision-preview\")\n", + "chat.invoke(\n", + " [\n", + " HumanMessage(\n", + " content=[\n", + " {\"type\": \"text\", \"text\": \"What is this image showing\"},\n", + " {\n", + " \"type\": \"image_url\",\n", + " \"image_url\": {\n", + " \"url\": \"https://python.langchain.com/assets/images/langchain_stack-da369071b058555da3d491a695651f15.jpg\",\n", + " \"detail\": \"auto\",\n", + " },\n", + " },\n", + " ]\n", + " )\n", + " ]\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "71c34763-d1e7-4b9a-a9d7-3e4cc0dfc2c4", + "metadata": {}, + "source": [ + "## [JSON mode](https://platform.openai.com/docs/guides/text-generation/json-mode)\n", + "\n", + "Constrain the model to only generate valid JSON. Note that you must include a system message with instructions to use JSON for this mode to work.\n", + "\n", + "Only works with certain models. " ] }, { diff --git a/libs/langchain/langchain/agents/openai_functions_multi_agent/base.py b/libs/langchain/langchain/agents/openai_functions_multi_agent/base.py index fae917805ebdd..0ecfe89a284f8 100644 --- a/libs/langchain/langchain/agents/openai_functions_multi_agent/base.py +++ b/libs/langchain/langchain/agents/openai_functions_multi_agent/base.py @@ -87,7 +87,9 @@ def _parse_ai_message(message: BaseMessage) -> Union[List[AgentAction], AgentFin final_tools.append(_tool) return final_tools - return AgentFinish(return_values={"output": message.content}, log=message.content) + return AgentFinish( + return_values={"output": message.content}, log=str(message.content) + ) class OpenAIMultiFunctionsAgent(BaseMultiActionAgent): diff --git a/libs/langchain/langchain/agents/output_parsers/openai_functions.py b/libs/langchain/langchain/agents/output_parsers/openai_functions.py index 3f82ceff96278..9e6aa315e8094 100644 --- a/libs/langchain/langchain/agents/output_parsers/openai_functions.py +++ b/libs/langchain/langchain/agents/output_parsers/openai_functions.py @@ -72,7 +72,7 @@ def _parse_ai_message(message: BaseMessage) -> Union[AgentAction, AgentFinish]: ) return AgentFinish( - return_values={"output": message.content}, log=message.content + return_values={"output": message.content}, log=str(message.content) ) def parse_result( diff --git a/libs/langchain/langchain/callbacks/infino_callback.py b/libs/langchain/langchain/callbacks/infino_callback.py index 8d54f08037a81..2d35850554d20 100644 --- a/libs/langchain/langchain/callbacks/infino_callback.py +++ b/libs/langchain/langchain/callbacks/infino_callback.py @@ -1,5 +1,5 @@ import time -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, cast from langchain.callbacks.base import BaseCallbackHandler from langchain.schema import AgentAction, AgentFinish, LLMResult @@ -232,7 +232,9 @@ def on_chat_model_start( self.chat_openai_model_name = model_name prompt_tokens = 0 for message_list in messages: - message_string = " ".join(msg.content for msg in message_list) + message_string = " ".join( + cast(str, msg.content) for msg in message_list + ) num_tokens = get_num_tokens( message_string, openai_model_name=self.chat_openai_model_name, @@ -249,7 +251,9 @@ def on_chat_model_start( ) # Send the prompt to infino - prompt = " ".join(msg.content for sublist in messages for msg in sublist) + prompt = " ".join( + cast(str, msg.content) for sublist in messages for msg in sublist + ) self._send_to_infino("prompt", prompt, is_ts=False) # Set the error flag to indicate no error (this will get overridden diff --git a/libs/langchain/langchain/chat_loaders/utils.py b/libs/langchain/langchain/chat_loaders/utils.py index 9a351becf2d27..b6a83c4fe1bf2 100644 --- a/libs/langchain/langchain/chat_loaders/utils.py +++ b/libs/langchain/langchain/chat_loaders/utils.py @@ -21,6 +21,11 @@ def merge_chat_runs_in_session( """ messages: List[BaseMessage] = [] for message in chat_session["messages"]: + if not isinstance(message.content, str): + raise ValueError( + "Chat Loaders only support messages with content type string, " + f"got {message.content}" + ) if not messages: messages.append(deepcopy(message)) elif ( @@ -29,6 +34,11 @@ def merge_chat_runs_in_session( and messages[-1].additional_kwargs["sender"] == message.additional_kwargs.get("sender") ): + if not isinstance(messages[-1].content, str): + raise ValueError( + "Chat Loaders only support messages with content type string, " + f"got {messages[-1].content}" + ) messages[-1].content = ( messages[-1].content + delimiter + message.content ).strip() diff --git a/libs/langchain/langchain/chat_models/anthropic.py b/libs/langchain/langchain/chat_models/anthropic.py index 24d1d7936c0fe..ad74089dd3a4b 100644 --- a/libs/langchain/langchain/chat_models/anthropic.py +++ b/libs/langchain/langchain/chat_models/anthropic.py @@ -1,4 +1,4 @@ -from typing import Any, AsyncIterator, Dict, Iterator, List, Optional +from typing import Any, AsyncIterator, Dict, Iterator, List, Optional, cast from langchain.callbacks.manager import ( AsyncCallbackManagerForLLMRun, @@ -27,14 +27,15 @@ def _convert_one_message_to_text( human_prompt: str, ai_prompt: str, ) -> str: + content = cast(str, message.content) if isinstance(message, ChatMessage): - message_text = f"\n\n{message.role.capitalize()}: {message.content}" + message_text = f"\n\n{message.role.capitalize()}: {content}" elif isinstance(message, HumanMessage): - message_text = f"{human_prompt} {message.content}" + message_text = f"{human_prompt} {content}" elif isinstance(message, AIMessage): - message_text = f"{ai_prompt} {message.content}" + message_text = f"{ai_prompt} {content}" elif isinstance(message, SystemMessage): - message_text = message.content + message_text = content else: raise ValueError(f"Got unknown type {message}") return message_text diff --git a/libs/langchain/langchain/chat_models/azureml_endpoint.py b/libs/langchain/langchain/chat_models/azureml_endpoint.py index 4f40f8acf6db5..53bdc849252b3 100644 --- a/libs/langchain/langchain/chat_models/azureml_endpoint.py +++ b/libs/langchain/langchain/chat_models/azureml_endpoint.py @@ -1,5 +1,5 @@ import json -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, cast from langchain.callbacks.manager import CallbackManagerForLLMRun from langchain.chat_models.base import SimpleChatModel @@ -23,26 +23,21 @@ class LlamaContentFormatter(ContentFormatterBase): @staticmethod def _convert_message_to_dict(message: BaseMessage) -> Dict: """Converts message to a dict according to role""" + content = cast(str, message.content) if isinstance(message, HumanMessage): return { "role": "user", - "content": ContentFormatterBase.escape_special_characters( - message.content - ), + "content": ContentFormatterBase.escape_special_characters(content), } elif isinstance(message, AIMessage): return { "role": "assistant", - "content": ContentFormatterBase.escape_special_characters( - message.content - ), + "content": ContentFormatterBase.escape_special_characters(content), } elif isinstance(message, SystemMessage): return { "role": "system", - "content": ContentFormatterBase.escape_special_characters( - message.content - ), + "content": ContentFormatterBase.escape_special_characters(content), } elif ( isinstance(message, ChatMessage) @@ -50,9 +45,7 @@ def _convert_message_to_dict(message: BaseMessage) -> Dict: ): return { "role": message.role, - "content": ContentFormatterBase.escape_special_characters( - message.content - ), + "content": ContentFormatterBase.escape_special_characters(content), } else: supported = ",".join( diff --git a/libs/langchain/langchain/chat_models/baidu_qianfan_endpoint.py b/libs/langchain/langchain/chat_models/baidu_qianfan_endpoint.py index 58602086826ae..fa04588a0532d 100644 --- a/libs/langchain/langchain/chat_models/baidu_qianfan_endpoint.py +++ b/libs/langchain/langchain/chat_models/baidu_qianfan_endpoint.py @@ -1,15 +1,7 @@ from __future__ import annotations import logging -from typing import ( - Any, - AsyncIterator, - Dict, - Iterator, - List, - Mapping, - Optional, -) +from typing import Any, AsyncIterator, Dict, Iterator, List, Mapping, Optional, cast from langchain.callbacks.manager import ( AsyncCallbackManagerForLLMRun, @@ -211,7 +203,7 @@ def _convert_prompt_msg_params( for i in [i for i, m in enumerate(messages) if isinstance(m, SystemMessage)]: if "system" not in messages_dict: messages_dict["system"] = "" - messages_dict["system"] += messages[i].content + "\n" + messages_dict["system"] += cast(str, messages[i].content) + "\n" return { **messages_dict, diff --git a/libs/langchain/langchain/chat_models/base.py b/libs/langchain/langchain/chat_models/base.py index f69c74f0407e8..a92d02d6a1015 100644 --- a/libs/langchain/langchain/chat_models/base.py +++ b/libs/langchain/langchain/chat_models/base.py @@ -634,7 +634,10 @@ def predict( else: _stop = list(stop) result = self([HumanMessage(content=text)], stop=_stop, **kwargs) - return result.content + if isinstance(result.content, str): + return result.content + else: + raise ValueError("Cannot use predict when output is not a string.") def predict_messages( self, @@ -659,7 +662,10 @@ async def apredict( result = await self._call_async( [HumanMessage(content=text)], stop=_stop, **kwargs ) - return result.content + if isinstance(result.content, str): + return result.content + else: + raise ValueError("Cannot use predict when output is not a string.") async def apredict_messages( self, diff --git a/libs/langchain/langchain/chat_models/google_palm.py b/libs/langchain/langchain/chat_models/google_palm.py index 5b8a62edcd532..3f68d4e45b84c 100644 --- a/libs/langchain/langchain/chat_models/google_palm.py +++ b/libs/langchain/langchain/chat_models/google_palm.py @@ -2,7 +2,7 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, cast from tenacity import ( before_sleep_log, @@ -114,7 +114,7 @@ def _messages_to_prompt_dict( if isinstance(input_message, SystemMessage): if index != 0: raise ChatGooglePalmError("System message must be first input message.") - context = input_message.content + context = cast(str, input_message.content) elif isinstance(input_message, HumanMessage) and input_message.example: if messages: raise ChatGooglePalmError( diff --git a/libs/langchain/langchain/chat_models/human.py b/libs/langchain/langchain/chat_models/human.py index 6c964c9243c5e..f085cb2515bae 100644 --- a/libs/langchain/langchain/chat_models/human.py +++ b/libs/langchain/langchain/chat_models/human.py @@ -58,7 +58,10 @@ def _collect_yaml_input( if message is None: return HumanMessage(content="") if stop: - message.content = enforce_stop_tokens(message.content, stop) + if isinstance(message.content, str): + message.content = enforce_stop_tokens(message.content, stop) + else: + raise ValueError("Cannot use when output is not a string.") return message except yaml.YAMLError: raise ValueError("Invalid YAML string entered.") diff --git a/libs/langchain/langchain/chat_models/minimax.py b/libs/langchain/langchain/chat_models/minimax.py index 93b0b9a1585b2..0fad5e24b106f 100644 --- a/libs/langchain/langchain/chat_models/minimax.py +++ b/libs/langchain/langchain/chat_models/minimax.py @@ -1,6 +1,6 @@ """Wrapper around Minimax chat models.""" import logging -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, cast from langchain.callbacks.manager import ( AsyncCallbackManagerForLLMRun, @@ -27,10 +27,11 @@ def _parse_chat_history(history: List[BaseMessage]) -> List: """Parse a sequence of messages into history.""" chat_history = [] for message in history: + content = cast(str, message.content) if isinstance(message, HumanMessage): - chat_history.append(_parse_message("USER", message.content)) + chat_history.append(_parse_message("USER", content)) if isinstance(message, AIMessage): - chat_history.append(_parse_message("BOT", message.content)) + chat_history.append(_parse_message("BOT", content)) return chat_history diff --git a/libs/langchain/langchain/chat_models/openai.py b/libs/langchain/langchain/chat_models/openai.py index c177758cd4620..1c8a0a92b9bb0 100644 --- a/libs/langchain/langchain/chat_models/openai.py +++ b/libs/langchain/langchain/chat_models/openai.py @@ -316,14 +316,16 @@ def validate_environment(cls, values: Dict) -> Dict: @property def _default_params(self) -> Dict[str, Any]: """Get the default parameters for calling OpenAI API.""" - return { + params = { "model": self.model_name, - "max_tokens": self.max_tokens, "stream": self.streaming, "n": self.n, "temperature": self.temperature, **self.model_kwargs, } + if "vision" not in self.model_name: + params["max_tokens"] = self.max_tokens + return params def completion_with_retry( self, run_manager: Optional[CallbackManagerForLLMRun] = None, **kwargs: Any diff --git a/libs/langchain/langchain/chat_models/pai_eas_endpoint.py b/libs/langchain/langchain/chat_models/pai_eas_endpoint.py index d6435b32172d1..cc029cffd2c4d 100644 --- a/libs/langchain/langchain/chat_models/pai_eas_endpoint.py +++ b/libs/langchain/langchain/chat_models/pai_eas_endpoint.py @@ -2,7 +2,7 @@ import json import logging from functools import partial -from typing import Any, AsyncIterator, Dict, List, Optional +from typing import Any, AsyncIterator, Dict, List, Optional, cast import requests @@ -133,23 +133,24 @@ def format_request_payload( for message in messages: """Converts message to a dict according to role""" + content = cast(str, message.content) if isinstance(message, HumanMessage): - user_content = user_content + [message.content] + user_content = user_content + [content] elif isinstance(message, AIMessage): - assistant_content = assistant_content + [message.content] + assistant_content = assistant_content + [content] elif isinstance(message, SystemMessage): - prompt["system_prompt"] = message.content + prompt["system_prompt"] = content elif isinstance(message, ChatMessage) and message.role in [ "user", "assistant", "system", ]: if message.role == "system": - prompt["system_prompt"] = message.content + prompt["system_prompt"] = content elif message.role == "user": - user_content = user_content + [message.content] + user_content = user_content + [content] elif message.role == "assistant": - assistant_content = assistant_content + [message.content] + assistant_content = assistant_content + [content] else: supported = ",".join([role for role in ["user", "assistant", "system"]]) raise ValueError( @@ -294,7 +295,7 @@ async def _astream( # yield text, if any if text: if run_manager: - await run_manager.on_llm_new_token(content.content) + await run_manager.on_llm_new_token(cast(str, content.content)) yield ChatGenerationChunk(message=content) # break if stop sequence found diff --git a/libs/langchain/langchain/chat_models/vertexai.py b/libs/langchain/langchain/chat_models/vertexai.py index ef23b88e71992..32495be5b8125 100644 --- a/libs/langchain/langchain/chat_models/vertexai.py +++ b/libs/langchain/langchain/chat_models/vertexai.py @@ -3,7 +3,7 @@ import logging from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Union +from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Union, cast from langchain.callbacks.manager import ( AsyncCallbackManagerForLLMRun, @@ -57,8 +57,9 @@ def _parse_chat_history(history: List[BaseMessage]) -> _ChatHistory: vertex_messages, context = [], None for i, message in enumerate(history): + content = cast(str, message.content) if i == 0 and isinstance(message, SystemMessage): - context = message.content + context = content elif isinstance(message, AIMessage): vertex_message = ChatMessage(content=message.content, author="bot") vertex_messages.append(vertex_message) diff --git a/libs/langchain/langchain/chat_models/yandex.py b/libs/langchain/langchain/chat_models/yandex.py index d517703df51cd..0847028aee4f9 100644 --- a/libs/langchain/langchain/chat_models/yandex.py +++ b/libs/langchain/langchain/chat_models/yandex.py @@ -1,6 +1,6 @@ """Wrapper around YandexGPT chat models.""" import logging -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict, List, Optional, Tuple, cast from langchain.callbacks.manager import ( AsyncCallbackManagerForLLMRun, @@ -34,12 +34,13 @@ def _parse_chat_history(history: List[BaseMessage]) -> Tuple[List[Dict[str, str] chat_history = [] instruction = "" for message in history: + content = cast(str, message.content) if isinstance(message, HumanMessage): - chat_history.append(_parse_message("user", message.content)) + chat_history.append(_parse_message("user", content)) if isinstance(message, AIMessage): - chat_history.append(_parse_message("assistant", message.content)) + chat_history.append(_parse_message("assistant", content)) if isinstance(message, SystemMessage): - instruction = message.content + instruction = content return chat_history, instruction diff --git a/libs/langchain/langchain/schema/messages.py b/libs/langchain/langchain/schema/messages.py index 51f9585d63683..f2eca17f86dd7 100644 --- a/libs/langchain/langchain/schema/messages.py +++ b/libs/langchain/langchain/schema/messages.py @@ -64,7 +64,7 @@ class BaseMessage(Serializable): Messages are the inputs and outputs of ChatModels. """ - content: str + content: Union[str, List[Union[str, Dict]]] """The string contents of the message.""" additional_kwargs: dict = Field(default_factory=dict) @@ -87,6 +87,33 @@ def __add__(self, other: Any) -> ChatPromptTemplate: return prompt + other +def merge_content( + first_content: Union[str, List[Union[str, Dict]]], + second_content: Union[str, List[Union[str, Dict]]], +) -> Union[str, List[Union[str, Dict]]]: + # If first chunk is a string + if isinstance(first_content, str): + # If the second chunk is also a string, then merge them naively + if isinstance(second_content, str): + return first_content + second_content + # If the second chunk is a list, add the first chunk to the start of the list + else: + return_list: List[Union[str, Dict]] = [first_content] + return return_list + second_content + # If both are lists, merge them naively + elif isinstance(second_content, List): + return first_content + second_content + # If the first content is a list, and the second content is a string + else: + # If the last element of the first content is a string + # Add the second content to the last element + if isinstance(first_content[-1], str): + return first_content[:-1] + [first_content[-1] + second_content] + else: + # Otherwise, add the second content as a new element of the list + return first_content + [second_content] + + class BaseMessageChunk(BaseMessage): """A Message chunk, which can be concatenated with other Message chunks.""" @@ -121,13 +148,13 @@ def __add__(self, other: Any) -> BaseMessageChunk: # type: ignore if isinstance(self, ChatMessageChunk): return self.__class__( role=self.role, - content=self.content + other.content, + content=merge_content(self.content, other.content), additional_kwargs=self._merge_kwargs_dict( self.additional_kwargs, other.additional_kwargs ), ) return self.__class__( - content=self.content + other.content, + content=merge_content(self.content, other.content), additional_kwargs=self._merge_kwargs_dict( self.additional_kwargs, other.additional_kwargs ), @@ -194,7 +221,7 @@ def __add__(self, other: Any) -> BaseMessageChunk: # type: ignore return self.__class__( example=self.example, - content=self.content + other.content, + content=merge_content(self.content, other.content), additional_kwargs=self._merge_kwargs_dict( self.additional_kwargs, other.additional_kwargs ), @@ -252,7 +279,7 @@ def __add__(self, other: Any) -> BaseMessageChunk: # type: ignore return self.__class__( name=self.name, - content=self.content + other.content, + content=merge_content(self.content, other.content), additional_kwargs=self._merge_kwargs_dict( self.additional_kwargs, other.additional_kwargs ), @@ -290,7 +317,7 @@ def __add__(self, other: Any) -> BaseMessageChunk: # type: ignore return self.__class__( role=self.role, - content=self.content + other.content, + content=merge_content(self.content, other.content), additional_kwargs=self._merge_kwargs_dict( self.additional_kwargs, other.additional_kwargs ), diff --git a/libs/langchain/tests/integration_tests/chat_models/test_fireworks.py b/libs/langchain/tests/integration_tests/chat_models/test_fireworks.py index 73d99484cbff6..43657cdae3c35 100644 --- a/libs/langchain/tests/integration_tests/chat_models/test_fireworks.py +++ b/libs/langchain/tests/integration_tests/chat_models/test_fireworks.py @@ -1,5 +1,6 @@ """Test ChatFireworks wrapper.""" import sys +from typing import cast import pytest @@ -152,7 +153,7 @@ def test_fireworks_streaming_stop_words(chat: ChatFireworks) -> None: last_token = "" for token in chat.stream("I'm Pickle Rick", stop=[","]): - last_token = token.content + last_token = cast(str, token.content) assert isinstance(token.content, str) assert last_token[-1] == "," @@ -183,6 +184,6 @@ async def test_fireworks_astream(chat: ChatFireworks) -> None: async for token in chat.astream( "Who's the best quarterback in the NFL?", stop=[","] ): - last_token = token.content + last_token = cast(str, token.content) assert isinstance(token.content, str) assert last_token[-1] == "," diff --git a/libs/langchain/tests/integration_tests/smith/evaluation/test_runner_utils.py b/libs/langchain/tests/integration_tests/smith/evaluation/test_runner_utils.py index f37cb20a0af97..6010b674c49f4 100644 --- a/libs/langchain/tests/integration_tests/smith/evaluation/test_runner_utils.py +++ b/libs/langchain/tests/integration_tests/smith/evaluation/test_runner_utils.py @@ -490,7 +490,13 @@ async def test_arb_func_on_kv_singleio_dataset( ) def my_func(x: dict) -> str: - return runnable.invoke(x).content + content = runnable.invoke(x).content + if isinstance(content, str): + return content + else: + raise ValueError( + f"Expected message with content type string, got {content}" + ) eval_config = RunEvalConfig(evaluators=[EvaluatorType.QA, EvaluatorType.CRITERIA]) await arun_on_dataset( diff --git a/libs/langchain/tests/unit_tests/schema/runnable/__snapshots__/test_runnable.ambr b/libs/langchain/tests/unit_tests/schema/runnable/__snapshots__/test_runnable.ambr index 2cfe9d5cfef46..2840aa0de3f83 100644 --- a/libs/langchain/tests/unit_tests/schema/runnable/__snapshots__/test_runnable.ambr +++ b/libs/langchain/tests/unit_tests/schema/runnable/__snapshots__/test_runnable.ambr @@ -1685,8 +1685,25 @@ 'type': 'object', }), 'content': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'items': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'type': 'object', + }), + ]), + }), + 'type': 'array', + }), + ]), 'title': 'Content', - 'type': 'string', }), 'example': dict({ 'default': False, @@ -1716,8 +1733,25 @@ 'type': 'object', }), 'content': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'items': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'type': 'object', + }), + ]), + }), + 'type': 'array', + }), + ]), 'title': 'Content', - 'type': 'string', }), 'role': dict({ 'title': 'Role', @@ -1791,8 +1825,25 @@ 'type': 'object', }), 'content': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'items': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'type': 'object', + }), + ]), + }), + 'type': 'array', + }), + ]), 'title': 'Content', - 'type': 'string', }), 'name': dict({ 'title': 'Name', @@ -1822,8 +1873,25 @@ 'type': 'object', }), 'content': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'items': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'type': 'object', + }), + ]), + }), + 'type': 'array', + }), + ]), 'title': 'Content', - 'type': 'string', }), 'example': dict({ 'default': False, @@ -1878,8 +1946,25 @@ 'type': 'object', }), 'content': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'items': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'type': 'object', + }), + ]), + }), + 'type': 'array', + }), + ]), 'title': 'Content', - 'type': 'string', }), 'type': dict({ 'default': 'system', @@ -1944,8 +2029,25 @@ 'type': 'object', }), 'content': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'items': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'type': 'object', + }), + ]), + }), + 'type': 'array', + }), + ]), 'title': 'Content', - 'type': 'string', }), 'example': dict({ 'default': False, @@ -1975,8 +2077,25 @@ 'type': 'object', }), 'content': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'items': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'type': 'object', + }), + ]), + }), + 'type': 'array', + }), + ]), 'title': 'Content', - 'type': 'string', }), 'role': dict({ 'title': 'Role', @@ -2050,8 +2169,25 @@ 'type': 'object', }), 'content': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'items': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'type': 'object', + }), + ]), + }), + 'type': 'array', + }), + ]), 'title': 'Content', - 'type': 'string', }), 'name': dict({ 'title': 'Name', @@ -2081,8 +2217,25 @@ 'type': 'object', }), 'content': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'items': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'type': 'object', + }), + ]), + }), + 'type': 'array', + }), + ]), 'title': 'Content', - 'type': 'string', }), 'example': dict({ 'default': False, @@ -2137,8 +2290,25 @@ 'type': 'object', }), 'content': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'items': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'type': 'object', + }), + ]), + }), + 'type': 'array', + }), + ]), 'title': 'Content', - 'type': 'string', }), 'type': dict({ 'default': 'system', @@ -2187,8 +2357,25 @@ 'type': 'object', }), 'content': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'items': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'type': 'object', + }), + ]), + }), + 'type': 'array', + }), + ]), 'title': 'Content', - 'type': 'string', }), 'example': dict({ 'default': False, @@ -2218,8 +2405,25 @@ 'type': 'object', }), 'content': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'items': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'type': 'object', + }), + ]), + }), + 'type': 'array', + }), + ]), 'title': 'Content', - 'type': 'string', }), 'role': dict({ 'title': 'Role', @@ -2249,8 +2453,25 @@ 'type': 'object', }), 'content': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'items': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'type': 'object', + }), + ]), + }), + 'type': 'array', + }), + ]), 'title': 'Content', - 'type': 'string', }), 'name': dict({ 'title': 'Name', @@ -2280,8 +2501,25 @@ 'type': 'object', }), 'content': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'items': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'type': 'object', + }), + ]), + }), + 'type': 'array', + }), + ]), 'title': 'Content', - 'type': 'string', }), 'example': dict({ 'default': False, @@ -2314,8 +2552,25 @@ 'type': 'object', }), 'content': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'items': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'type': 'object', + }), + ]), + }), + 'type': 'array', + }), + ]), 'title': 'Content', - 'type': 'string', }), 'type': dict({ 'default': 'system', @@ -2355,8 +2610,25 @@ 'type': 'object', }), 'content': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'items': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'type': 'object', + }), + ]), + }), + 'type': 'array', + }), + ]), 'title': 'Content', - 'type': 'string', }), 'example': dict({ 'default': False, @@ -2386,8 +2658,25 @@ 'type': 'object', }), 'content': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'items': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'type': 'object', + }), + ]), + }), + 'type': 'array', + }), + ]), 'title': 'Content', - 'type': 'string', }), 'role': dict({ 'title': 'Role', @@ -2461,8 +2750,25 @@ 'type': 'object', }), 'content': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'items': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'type': 'object', + }), + ]), + }), + 'type': 'array', + }), + ]), 'title': 'Content', - 'type': 'string', }), 'name': dict({ 'title': 'Name', @@ -2492,8 +2798,25 @@ 'type': 'object', }), 'content': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'items': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'type': 'object', + }), + ]), + }), + 'type': 'array', + }), + ]), 'title': 'Content', - 'type': 'string', }), 'example': dict({ 'default': False, @@ -2548,8 +2871,25 @@ 'type': 'object', }), 'content': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'items': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'type': 'object', + }), + ]), + }), + 'type': 'array', + }), + ]), 'title': 'Content', - 'type': 'string', }), 'type': dict({ 'default': 'system', @@ -2589,8 +2929,25 @@ 'type': 'object', }), 'content': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'items': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'type': 'object', + }), + ]), + }), + 'type': 'array', + }), + ]), 'title': 'Content', - 'type': 'string', }), 'example': dict({ 'default': False, @@ -2620,8 +2977,25 @@ 'type': 'object', }), 'content': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'items': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'type': 'object', + }), + ]), + }), + 'type': 'array', + }), + ]), 'title': 'Content', - 'type': 'string', }), 'role': dict({ 'title': 'Role', @@ -2695,8 +3069,25 @@ 'type': 'object', }), 'content': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'items': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'type': 'object', + }), + ]), + }), + 'type': 'array', + }), + ]), 'title': 'Content', - 'type': 'string', }), 'name': dict({ 'title': 'Name', @@ -2726,8 +3117,25 @@ 'type': 'object', }), 'content': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'items': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'type': 'object', + }), + ]), + }), + 'type': 'array', + }), + ]), 'title': 'Content', - 'type': 'string', }), 'example': dict({ 'default': False, @@ -2782,8 +3190,25 @@ 'type': 'object', }), 'content': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'items': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'type': 'object', + }), + ]), + }), + 'type': 'array', + }), + ]), 'title': 'Content', - 'type': 'string', }), 'type': dict({ 'default': 'system', @@ -2815,8 +3240,25 @@ 'type': 'object', }), 'content': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'items': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'type': 'object', + }), + ]), + }), + 'type': 'array', + }), + ]), 'title': 'Content', - 'type': 'string', }), 'example': dict({ 'default': False, @@ -2846,8 +3288,25 @@ 'type': 'object', }), 'content': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'items': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'type': 'object', + }), + ]), + }), + 'type': 'array', + }), + ]), 'title': 'Content', - 'type': 'string', }), 'role': dict({ 'title': 'Role', @@ -2921,8 +3380,25 @@ 'type': 'object', }), 'content': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'items': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'type': 'object', + }), + ]), + }), + 'type': 'array', + }), + ]), 'title': 'Content', - 'type': 'string', }), 'name': dict({ 'title': 'Name', @@ -2952,8 +3428,25 @@ 'type': 'object', }), 'content': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'items': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'type': 'object', + }), + ]), + }), + 'type': 'array', + }), + ]), 'title': 'Content', - 'type': 'string', }), 'example': dict({ 'default': False, @@ -3019,8 +3512,25 @@ 'type': 'object', }), 'content': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'items': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'type': 'object', + }), + ]), + }), + 'type': 'array', + }), + ]), 'title': 'Content', - 'type': 'string', }), 'type': dict({ 'default': 'system', @@ -3076,8 +3586,25 @@ 'type': 'object', }), 'content': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'items': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'type': 'object', + }), + ]), + }), + 'type': 'array', + }), + ]), 'title': 'Content', - 'type': 'string', }), 'example': dict({ 'default': False, @@ -3107,8 +3634,25 @@ 'type': 'object', }), 'content': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'items': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'type': 'object', + }), + ]), + }), + 'type': 'array', + }), + ]), 'title': 'Content', - 'type': 'string', }), 'role': dict({ 'title': 'Role', @@ -3138,8 +3682,25 @@ 'type': 'object', }), 'content': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'items': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'type': 'object', + }), + ]), + }), + 'type': 'array', + }), + ]), 'title': 'Content', - 'type': 'string', }), 'name': dict({ 'title': 'Name', @@ -3169,8 +3730,25 @@ 'type': 'object', }), 'content': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'items': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'type': 'object', + }), + ]), + }), + 'type': 'array', + }), + ]), 'title': 'Content', - 'type': 'string', }), 'example': dict({ 'default': False, @@ -3203,8 +3781,25 @@ 'type': 'object', }), 'content': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'items': dict({ + 'anyOf': list([ + dict({ + 'type': 'string', + }), + dict({ + 'type': 'object', + }), + ]), + }), + 'type': 'array', + }), + ]), 'title': 'Content', - 'type': 'string', }), 'type': dict({ 'default': 'system', diff --git a/libs/langchain/tests/unit_tests/schema/runnable/test_runnable.py b/libs/langchain/tests/unit_tests/schema/runnable/test_runnable.py index ec54c527c067f..1ca2d0c81b318 100644 --- a/libs/langchain/tests/unit_tests/schema/runnable/test_runnable.py +++ b/libs/langchain/tests/unit_tests/schema/runnable/test_runnable.py @@ -295,7 +295,18 @@ async def typed_async_lambda_impl(x: str) -> int: "description": "A Message from an AI.", "type": "object", "properties": { - "content": {"title": "Content", "type": "string"}, + "content": { + "title": "Content", + "anyOf": [ + {"type": "string"}, + { + "type": "array", + "items": { + "anyOf": [{"type": "string"}, {"type": "object"}] + }, + }, + ], + }, "additional_kwargs": { "title": "Additional Kwargs", "type": "object", @@ -319,7 +330,18 @@ async def typed_async_lambda_impl(x: str) -> int: "description": "A Message from a human.", "type": "object", "properties": { - "content": {"title": "Content", "type": "string"}, + "content": { + "title": "Content", + "anyOf": [ + {"type": "string"}, + { + "type": "array", + "items": { + "anyOf": [{"type": "string"}, {"type": "object"}] + }, + }, + ], + }, "additional_kwargs": { "title": "Additional Kwargs", "type": "object", @@ -343,7 +365,18 @@ async def typed_async_lambda_impl(x: str) -> int: "description": "A Message that can be assigned an arbitrary speaker (i.e. role).", # noqa: E501 "type": "object", "properties": { - "content": {"title": "Content", "type": "string"}, + "content": { + "title": "Content", + "anyOf": [ + {"type": "string"}, + { + "type": "array", + "items": { + "anyOf": [{"type": "string"}, {"type": "object"}] + }, + }, + ], + }, "additional_kwargs": { "title": "Additional Kwargs", "type": "object", @@ -363,7 +396,18 @@ async def typed_async_lambda_impl(x: str) -> int: "description": "A Message for priming AI behavior, usually passed in as the first of a sequence\nof input messages.", # noqa: E501 "type": "object", "properties": { - "content": {"title": "Content", "type": "string"}, + "content": { + "title": "Content", + "anyOf": [ + {"type": "string"}, + { + "type": "array", + "items": { + "anyOf": [{"type": "string"}, {"type": "object"}] + }, + }, + ], + }, "additional_kwargs": { "title": "Additional Kwargs", "type": "object", @@ -382,7 +426,18 @@ async def typed_async_lambda_impl(x: str) -> int: "description": "A Message for passing the result of executing a function back to a model.", # noqa: E501 "type": "object", "properties": { - "content": {"title": "Content", "type": "string"}, + "content": { + "title": "Content", + "anyOf": [ + {"type": "string"}, + { + "type": "array", + "items": { + "anyOf": [{"type": "string"}, {"type": "object"}] + }, + }, + ], + }, "additional_kwargs": { "title": "Additional Kwargs", "type": "object",