Fix tools execution #724

rjambrecic · 2025-01-30T14:54:29Z

Why are these changes needed?

We have two issues:

Function which returns list of some pydantic BaseModel -> e.g. list[Message]

from autogen.agentchat import AssistantAgent, UserProxyAgent
import os
from pydantic import BaseModel

llm_config={"config_list": [{"model": "gpt-4", "api_key": os.getenv("OPENAI_API_KEY")}]}

user_proxy_agent = UserProxyAgent(
    name="UserProxyAgent",
    human_input_mode="NEVER",
)
assistant = AssistantAgent(
    name="AssistantAgent",
    llm_config=llm_config,
)

class Message(BaseModel):
    text: str

@user_proxy_agent.register_for_execution()
@assistant.register_for_llm(description="Generate a response to a message")
def get_list_of_messages() -> list[Message]:
    return [Message(text="Hello"), Message(text="How are you?")]


user_proxy_agent.initiate_chat(assistant, message="Get list of messages", max_turns=2)

UserProxyAgent (to AssistantAgent):

Get list of messages

--------------------------------------------------------------------------------
AssistantAgent (to UserProxyAgent):

***** Suggested tool call (call_eLcTJAgCQHhg4wEwfT6Dr26E): get_list_of_messages *****
Arguments: 
{}
*************************************************************************************

--------------------------------------------------------------------------------

>>>>>>>> EXECUTING FUNCTION get_list_of_messages...
Call ID: call_eLcTJAgCQHhg4wEwfT6Dr26E
Input arguments: {}
UserProxyAgent (to AssistantAgent):

***** Response from calling tool (call_eLcTJAgCQHhg4wEwfT6Dr26E) *****
Error: Object of type Message is not JSON serializable
**********************************************************************

Issue #643

from autogen.agentchat.contrib.rag.parser_utils import docling_parse_docs
from autogen.tools.tool import Tool
from autogen.agentchat import AssistantAgent, UserProxyAgent
import os
from pydantic import BaseModel

llm_config={"config_list": [{"model": "gpt-4", "api_key": os.getenv("OPENAI_API_KEY")}]}

user_proxy_agent = UserProxyAgent(
    name="UserProxyAgent",
    human_input_mode="NEVER",
)
assistant = AssistantAgent(
    name="AssistantAgent",
    llm_config=llm_config,
)

tool = Tool(
    name="docling_parse_docs",
    description="Parse a list of documents",
    func_or_tool=docling_parse_docs,
)
tool.register_for_execution(user_proxy_agent)
tool.register_for_llm(assistant)

# create tmp folder with one .md file
import tempfile
with tempfile.TemporaryDirectory() as tmpdirname:
    md_file = os.path.join(tmpdirname, "example.md")
    with open(md_file, "w") as f:
        f.write("# Hello\nThis is a test")
    
    user_proxy_agent.function_map["docling_parse_docs"](input_file_path=str(md_file), output_dir_path=str(tmpdirname))
    # message = f"Use docling_parse_docs -> input_file_path: {md_file}, output_dir_path: {tmpdirname}"
    # user_proxy_agent.initiate_chat(assistant, message=message, max_turns=2)

Saved markdown output to: [/var/folders/79/kmq3y0pj0rbgrrnf4xlq8q0c0000gn/T/tmpzpyqp18u](https://file+.vscode-resource.vscode-cdn.net/var/folders/79/kmq3y0pj0rbgrrnf4xlq8q0c0000gn/T/tmpzpyqp18u)
---------------------------------------------------------------------------
PydanticUserError                         Traceback (most recent call last)
Cell In[4], [line 33](vscode-notebook-cell:?execution_count=4&line=33)
     [30](vscode-notebook-cell:?execution_count=4&line=30) with open(md_file, "w") as f:
     [31](vscode-notebook-cell:?execution_count=4&line=31)     f.write("# Hello\nThis is a test")
---> [33](vscode-notebook-cell:?execution_count=4&line=33) user_proxy_agent.function_map["docling_parse_docs"](input_file_path=str(md_file), output_dir_path=str(tmpdirname))
     [34](vscode-notebook-cell:?execution_count=4&line=34) # message = f"Use docling_parse_docs -> input_file_path: {md_file}, output_dir_path: {tmpdirname}"
     [35](vscode-notebook-cell:?execution_count=4&line=35) # user_proxy_agent.initiate_chat(assistant, message=message, max_turns=2)

File ~/projects/ag2/autogen/tools/function_utils.py:348, in load_basemodels_if_needed.<locals>._load_parameters_if_needed(*args, **kwargs)
    [345](https://file+.vscode-resource.vscode-cdn.net/Users/robert/projects/ag2/~/projects/ag2/autogen/tools/function_utils.py:345)     kwargs[k] = f(kwargs[k], param_annotations[k])
    [347](https://file+.vscode-resource.vscode-cdn.net/Users/robert/projects/ag2/~/projects/ag2/autogen/tools/function_utils.py:347) # call the original function
--> [348](https://file+.vscode-resource.vscode-cdn.net/Users/robert/projects/ag2/~/projects/ag2/autogen/tools/function_utils.py:348) return func(*args, **kwargs)

File ~/projects/ag2/autogen/agentchat/conversable_agent.py:2651, in ConversableAgent._wrap_function.<locals>._wrapped_func(*args, **kwargs)
   [2648](https://file+.vscode-resource.vscode-cdn.net/Users/robert/projects/ag2/~/projects/ag2/autogen/agentchat/conversable_agent.py:2648) @load_basemodels_if_needed
   [2649](https://file+.vscode-resource.vscode-cdn.net/Users/robert/projects/ag2/~/projects/ag2/autogen/agentchat/conversable_agent.py:2649) @functools.wraps(func)
   [2650](https://file+.vscode-resource.vscode-cdn.net/Users/robert/projects/ag2/~/projects/ag2/autogen/agentchat/conversable_agent.py:2650) def _wrapped_func(*args, **kwargs):
-> [2651](https://file+.vscode-resource.vscode-cdn.net/Users/robert/projects/ag2/~/projects/ag2/autogen/agentchat/conversable_agent.py:2651)     retval = func(*args, **kwargs, **inject_params)
   [2652](https://file+.vscode-resource.vscode-cdn.net/Users/robert/projects/ag2/~/projects/ag2/autogen/agentchat/conversable_agent.py:2652)     if logging_enabled():
   [2653](https://file+.vscode-resource.vscode-cdn.net/Users/robert/projects/ag2/~/projects/ag2/autogen/agentchat/conversable_agent.py:2653)         log_function_use(self, func, kwargs, retval)

File ~/projects/ag2/.venv/lib/python3.10/site-packages/fast_depends/use.py:169, in _wrap_inject.<locals>.func_wrapper.<locals>.injected_wrapper(*args, **kwargs)
    [166](https://file+.vscode-resource.vscode-cdn.net/Users/robert/projects/ag2/~/projects/ag2/.venv/lib/python3.10/site-packages/fast_depends/use.py:166) @wraps(func)
    [167](https://file+.vscode-resource.vscode-cdn.net/Users/robert/projects/ag2/~/projects/ag2/.venv/lib/python3.10/site-packages/fast_depends/use.py:167) def injected_wrapper(*args: P.args, **kwargs: P.kwargs) -> T:
    [168](https://file+.vscode-resource.vscode-cdn.net/Users/robert/projects/ag2/~/projects/ag2/.venv/lib/python3.10/site-packages/fast_depends/use.py:168)     with ExitStack() as stack:
--> [169](https://file+.vscode-resource.vscode-cdn.net/Users/robert/projects/ag2/~/projects/ag2/.venv/lib/python3.10/site-packages/fast_depends/use.py:169)         r = real_model.solve(
    [170](https://file+.vscode-resource.vscode-cdn.net/Users/robert/projects/ag2/~/projects/ag2/.venv/lib/python3.10/site-packages/fast_depends/use.py:170)             *args,
    [171](https://file+.vscode-resource.vscode-cdn.net/Users/robert/projects/ag2/~/projects/ag2/.venv/lib/python3.10/site-packages/fast_depends/use.py:171)             stack=stack,
    [172](https://file+.vscode-resource.vscode-cdn.net/Users/robert/projects/ag2/~/projects/ag2/.venv/lib/python3.10/site-packages/fast_depends/use.py:172)             dependency_overrides=overrides,
    [173](https://file+.vscode-resource.vscode-cdn.net/Users/robert/projects/ag2/~/projects/ag2/.venv/lib/python3.10/site-packages/fast_depends/use.py:173)             cache_dependencies={},
    [174](https://file+.vscode-resource.vscode-cdn.net/Users/robert/projects/ag2/~/projects/ag2/.venv/lib/python3.10/site-packages/fast_depends/use.py:174)             nested=False,
    [175](https://file+.vscode-resource.vscode-cdn.net/Users/robert/projects/ag2/~/projects/ag2/.venv/lib/python3.10/site-packages/fast_depends/use.py:175)             **kwargs,
    [176](https://file+.vscode-resource.vscode-cdn.net/Users/robert/projects/ag2/~/projects/ag2/.venv/lib/python3.10/site-packages/fast_depends/use.py:176)         )
    [177](https://file+.vscode-resource.vscode-cdn.net/Users/robert/projects/ag2/~/projects/ag2/.venv/lib/python3.10/site-packages/fast_depends/use.py:177)         return r
    [179](https://file+.vscode-resource.vscode-cdn.net/Users/robert/projects/ag2/~/projects/ag2/.venv/lib/python3.10/site-packages/fast_depends/use.py:179)     raise AssertionError("unreachable")

File ~/projects/ag2/.venv/lib/python3.10/site-packages/fast_depends/core/model.py:419, in CallModel.solve(self, stack, cache_dependencies, dependency_overrides, nested, *args, **kwargs)
    [416](https://file+.vscode-resource.vscode-cdn.net/Users/robert/projects/ag2/~/projects/ag2/.venv/lib/python3.10/site-packages/fast_depends/core/model.py:416)     response = call(*final_args, **final_kwargs)
    [418](https://file+.vscode-resource.vscode-cdn.net/Users/robert/projects/ag2/~/projects/ag2/.venv/lib/python3.10/site-packages/fast_depends/core/model.py:418) try:
--> [419](https://file+.vscode-resource.vscode-cdn.net/Users/robert/projects/ag2/~/projects/ag2/.venv/lib/python3.10/site-packages/fast_depends/core/model.py:419)     cast_gen.send(response)
    [420](https://file+.vscode-resource.vscode-cdn.net/Users/robert/projects/ag2/~/projects/ag2/.venv/lib/python3.10/site-packages/fast_depends/core/model.py:420) except StopIteration as e:
    [421](https://file+.vscode-resource.vscode-cdn.net/Users/robert/projects/ag2/~/projects/ag2/.venv/lib/python3.10/site-packages/fast_depends/core/model.py:421)     value: T = e.value

File ~/projects/ag2/.venv/lib/python3.10/site-packages/fast_depends/core/model.py:316, in CallModel._solve(self, cache_dependencies, dependency_overrides, *args, **kwargs)
    [313](https://file+.vscode-resource.vscode-cdn.net/Users/robert/projects/ag2/~/projects/ag2/.venv/lib/python3.10/site-packages/fast_depends/core/model.py:313) response = yield args_, kwargs_, call
    [315](https://file+.vscode-resource.vscode-cdn.net/Users/robert/projects/ag2/~/projects/ag2/.venv/lib/python3.10/site-packages/fast_depends/core/model.py:315) if self.cast and not self.is_generator:
--> [316](https://file+.vscode-resource.vscode-cdn.net/Users/robert/projects/ag2/~/projects/ag2/.venv/lib/python3.10/site-packages/fast_depends/core/model.py:316)     response = self._cast_response(response)
    [318](https://file+.vscode-resource.vscode-cdn.net/Users/robert/projects/ag2/~/projects/ag2/.venv/lib/python3.10/site-packages/fast_depends/core/model.py:318) if self.use_cache:  # pragma: no branch
    [319](https://file+.vscode-resource.vscode-cdn.net/Users/robert/projects/ag2/~/projects/ag2/.venv/lib/python3.10/site-packages/fast_depends/core/model.py:319)     cache_dependencies[call] = response

File ~/projects/ag2/.venv/lib/python3.10/site-packages/fast_depends/core/model.py:325, in CallModel._cast_response(self, value)
    [323](https://file+.vscode-resource.vscode-cdn.net/Users/robert/projects/ag2/~/projects/ag2/.venv/lib/python3.10/site-packages/fast_depends/core/model.py:323) def _cast_response(self, /, value: Any) -> Any:
    [324](https://file+.vscode-resource.vscode-cdn.net/Users/robert/projects/ag2/~/projects/ag2/.venv/lib/python3.10/site-packages/fast_depends/core/model.py:324)     if self.response_model is not None:
--> [325](https://file+.vscode-resource.vscode-cdn.net/Users/robert/projects/ag2/~/projects/ag2/.venv/lib/python3.10/site-packages/fast_depends/core/model.py:325)         return self.response_model(response=value).response
    [326](https://file+.vscode-resource.vscode-cdn.net/Users/robert/projects/ag2/~/projects/ag2/.venv/lib/python3.10/site-packages/fast_depends/core/model.py:326)     else:
    [327](https://file+.vscode-resource.vscode-cdn.net/Users/robert/projects/ag2/~/projects/ag2/.venv/lib/python3.10/site-packages/fast_depends/core/model.py:327)         return value

    [... skipping hidden 1 frame]

File ~/projects/ag2/.venv/lib/python3.10/site-packages/pydantic/_internal/_mock_val_ser.py:100, in MockValSer.__getattr__(self, item)
     [98](https://file+.vscode-resource.vscode-cdn.net/Users/robert/projects/ag2/~/projects/ag2/.venv/lib/python3.10/site-packages/pydantic/_internal/_mock_val_ser.py:98) # raise an AttributeError if `item` doesn't exist
     [99](https://file+.vscode-resource.vscode-cdn.net/Users/robert/projects/ag2/~/projects/ag2/.venv/lib/python3.10/site-packages/pydantic/_internal/_mock_val_ser.py:99) getattr(self._val_or_ser, item)
--> [100](https://file+.vscode-resource.vscode-cdn.net/Users/robert/projects/ag2/~/projects/ag2/.venv/lib/python3.10/site-packages/pydantic/_internal/_mock_val_ser.py:100) raise PydanticUserError(self._error_message, code=self._code)

PydanticUserError: `ResponseModel` is not fully defined; you should define `ConversionResult`, then call `ResponseModel.model_rebuild()`.

For further information visit https://errors.pydantic.dev/2.10/u/class-not-fully-defined

Related issue number

Closes #643

Checks

I've included any doc changes needed for https://docs.ag2.ai/. See https://docs.ag2.ai/docs/contributor-guide/documentation to build and test documentation locally.
I've added tests (if relevant) corresponding to the changes introduced in this PR.
I've made sure all auto checks have passed.

rjambrecic added 6 commits January 30, 2025 11:51

WIP

b809a0c

WIP

ed3f82d

WIP

f7ef4ff

Modigy _change_return_annotation to work with both sync and async tools

4ac4327

Add test_set_return_annotation_to_any

6e4484e

Add test_register_docling_parse_docs_as_a_tool

5206f9b

rjambrecic had a problem deploying to openai1 January 30, 2025 14:54 — with GitHub Actions Failure

rjambrecic temporarily deployed to openai1 January 30, 2025 14:54 — with GitHub Actions Inactive

rjambrecic had a problem deploying to openai1 January 30, 2025 14:54 — with GitHub Actions Failure

rjambrecic temporarily deployed to openai1 January 30, 2025 14:54 — with GitHub Actions Inactive

rjambrecic had a problem deploying to openai1 January 30, 2025 14:54 — with GitHub Actions Failure

rjambrecic temporarily deployed to openai1 January 30, 2025 14:54 — with GitHub Actions Inactive

rjambrecic assigned davorrunje Jan 30, 2025

Merge remote-tracking branch 'origin/main' into fix-tool-calling-issue

9818ec3

rjambrecic had a problem deploying to openai1 January 30, 2025 16:08 — with GitHub Actions Failure

rjambrecic temporarily deployed to openai1 January 30, 2025 16:08 — with GitHub Actions Inactive

rjambrecic had a problem deploying to openai1 January 30, 2025 16:08 — with GitHub Actions Failure

rjambrecic temporarily deployed to openai1 January 30, 2025 16:08 — with GitHub Actions Inactive

polishing

7de61ae

davorrunje had a problem deploying to openai1 January 31, 2025 08:43 — with GitHub Actions Failure

davorrunje had a problem deploying to openai1 January 31, 2025 08:43 — with GitHub Actions Error

davorrunje had a problem deploying to openai1 January 31, 2025 08:43 — with GitHub Actions Failure

davorrunje had a problem deploying to openai1 January 31, 2025 08:43 — with GitHub Actions Error

davorrunje approved these changes Jan 31, 2025

View reviewed changes

Merge remote-tracking branch 'origin/main' into fix-tool-calling-issue

d18bdc7

davorrunje temporarily deployed to openai1 January 31, 2025 08:45 — with GitHub Actions Inactive

davorrunje had a problem deploying to openai1 January 31, 2025 08:45 — with GitHub Actions Failure

davorrunje temporarily deployed to openai1 January 31, 2025 08:45 — with GitHub Actions Inactive

davorrunje had a problem deploying to openai1 January 31, 2025 08:45 — with GitHub Actions Failure

davorrunje temporarily deployed to openai1 January 31, 2025 08:45 — with GitHub Actions Inactive

rjambrecic added this pull request to the merge queue Jan 31, 2025

Merged via the queue into main with commit 9ac515d Jan 31, 2025
283 of 301 checks passed

rjambrecic deleted the fix-tool-calling-issue branch January 31, 2025 09:49

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

Fix tools execution #724

Fix tools execution #724

Uh oh!

rjambrecic commented Jan 30, 2025 •

edited

Loading

Uh oh!

Uh oh!

Reviewers

Assignees

Labels

Projects

Milestone

Development

Uh oh!

3 participants

Fix tools execution #724

Fix tools execution #724

Uh oh!

Conversation

rjambrecic commented Jan 30, 2025 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Why are these changes needed?

Related issue number

Checks

Uh oh!

Uh oh!

Reviewers

Assignees

Labels

Projects

Milestone

Development

Uh oh!

3 participants

rjambrecic commented Jan 30, 2025 •

edited

Loading