feature #478 [Agent] Allow exposing tool exceptions to AI (valtzu)

This PR was merged into the main branch.

Discussion
----------

[Agent] Allow exposing tool exceptions to AI

| Q             | A
| ------------- | ---
| Bug fix?      | no
| New feature?  | yes
| Docs?         | yes
| Issues        | Fix #255
| License       | MIT

Make it possible to throw custom exceptions that expose the error to the LLM when using `FaultTolerantToolbox`.

Example use case: an event listener for `ToolCallArgumentsResolved` that validates the tool call arguments and throws if any violations found. Then the LLM can auto-fix the input.

Commits
-------

a06e551 [Agent] Allow exposing exceptions to AI

Author: OskarStark

fixtures/Tool/ToolCustomException.php

@@ -0,0 +1,33 @@
+<?php
+
+/*
+ * This file is part of the Symfony package.
+ *
+ * (c) Fabien Potencier <fabien@symfony.com>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace Symfony\AI\Fixtures\Tool;
+
+use Symfony\AI\Agent\Toolbox\Attribute\AsTool;
+use Symfony\AI\Agent\Toolbox\Exception\ToolExecutionExceptionInterface;
+
+#[AsTool('tool_custom_exception', description: 'This tool is broken and it exposes the error', method: 'bar')]
+final class ToolCustomException
+{
+    public function bar(): string
+    {
+        throw new class('Custom error.') extends \RuntimeException implements ToolExecutionExceptionInterface {
+            public function getToolCallResult(): array
+            {
+                return [
+                    'error' => true,
+                    'error_code' => 'ERR42',
+                    'error_description' => 'Temporary error, try again later.',
+                ];
+            }
+        };
+    }
+}

src/agent/doc/index.rst

@@ -268,6 +268,37 @@ to the LLM::
 
     $agent = new Agent($platform, $model, inputProcessor: [$toolProcessor], outputProcessor: [$toolProcessor]);
 
+If you want to expose the underlying error to the LLM, you can throw a custom exception that implements `ToolExecutionExceptionInterface`::
+
+    use Symfony\AI\Agent\Toolbox\Exception\ToolExecutionExceptionInterface;
+
+    class EntityNotFoundException extends \RuntimeException implements ToolExecutionExceptionInterface
+    {
+        public function __construct(private string $entityName, private int $id)
+        {
+        }
+
+        public function getToolCallResult(): mixed
+        {
+            return \sprintf('No %s found with id %d', $this->entityName, $this->id);
+        }
+    }
+
+    #[AsTool('get_user_age', 'Get age by user id')]
+    class GetUserAge
+    {
+        public function __construct(private UserRepository $userRepository)
+        {
+        }
+
+        public function __invoke(int $id): int
+        {
+            $user = $this->userRepository->find($id) ?? throw new EntityNotFoundException('user', $id);
+
+            return $user->getAge();
+        }
+    }
+
 **Tool Filtering**
 
 To limit the tools provided to the LLM in a specific agent call to a subset of the configured tools, you can use the

src/agent/src/Toolbox/Exception/ToolExecutionException.php

@@ -16,7 +16,7 @@
 /**
  * @author Christopher Hertel <mail@christopher-hertel.de>
  */
-final class ToolExecutionException extends \RuntimeException implements ExceptionInterface
+final class ToolExecutionException extends \RuntimeException implements ToolExecutionExceptionInterface
 {
     public ?ToolCall $toolCall = null;
 
@@ -27,4 +27,9 @@ public static function executionFailed(ToolCall $toolCall, \Throwable $previous)
 
         return $exception;
     }
+
+    public function getToolCallResult(): string
+    {
+        return \sprintf('An error occurred while executing tool "%s".', $this->toolCall->name);
+    }
 }

src/agent/src/Toolbox/Exception/ToolExecutionExceptionInterface.php

@@ -0,0 +1,20 @@
+<?php
+
+/*
+ * This file is part of the Symfony package.
+ *
+ * (c) Fabien Potencier <fabien@symfony.com>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace Symfony\AI\Agent\Toolbox\Exception;
+
+/**
+ * @author Valtteri R <valtzu@gmail.com>
+ */
+interface ToolExecutionExceptionInterface extends ExceptionInterface
+{
+    public function getToolCallResult(): mixed;
+}

src/agent/src/Toolbox/FaultTolerantToolbox.php

@@ -11,7 +11,7 @@
 
 namespace Symfony\AI\Agent\Toolbox;
 
-use Symfony\AI\Agent\Toolbox\Exception\ToolExecutionException;
+use Symfony\AI\Agent\Toolbox\Exception\ToolExecutionExceptionInterface;
 use Symfony\AI\Agent\Toolbox\Exception\ToolNotFoundException;
 use Symfony\AI\Platform\Result\ToolCall;
 use Symfony\AI\Platform\Tool\Tool;
@@ -37,8 +37,8 @@ public function execute(ToolCall $toolCall): mixed
     {
         try {
             return $this->innerToolbox->execute($toolCall);
-        } catch (ToolExecutionException $e) {
-            return \sprintf('An error occurred while executing tool "%s".', $e->toolCall->name);
+        } catch (ToolExecutionExceptionInterface $e) {
+            return $e->getToolCallResult();
         } catch (ToolNotFoundException) {
             $names = array_map(fn (Tool $metadata) => $metadata->name, $this->getTools());
 

src/agent/src/Toolbox/Toolbox.php

@@ -15,6 +15,7 @@
 use Psr\Log\NullLogger;
 use Symfony\AI\Agent\Toolbox\Event\ToolCallArgumentsResolved;
 use Symfony\AI\Agent\Toolbox\Exception\ToolExecutionException;
+use Symfony\AI\Agent\Toolbox\Exception\ToolExecutionExceptionInterface;
 use Symfony\AI\Agent\Toolbox\Exception\ToolNotFoundException;
 use Symfony\AI\Agent\Toolbox\ToolFactory\ReflectionToolFactory;
 use Symfony\AI\Platform\Result\ToolCall;
@@ -81,6 +82,8 @@ public function execute(ToolCall $toolCall): mixed
             $this->eventDispatcher?->dispatch(new ToolCallArgumentsResolved($tool, $metadata, $arguments));
 
             $result = $tool->{$metadata->reference->method}(...$arguments);
+        } catch (ToolExecutionExceptionInterface $e) {
+            throw $e;
         } catch (\Throwable $e) {
             $this->logger->warning(\sprintf('Failed to execute tool "%s".', $toolCall->name), ['exception' => $e]);
             throw ToolExecutionException::executionFailed($toolCall, $e);

src/agent/src/Toolbox/ToolboxInterface.php

@@ -11,7 +11,7 @@
 
 namespace Symfony\AI\Agent\Toolbox;
 
-use Symfony\AI\Agent\Toolbox\Exception\ToolExecutionException;
+use Symfony\AI\Agent\Toolbox\Exception\ToolExecutionExceptionInterface;
 use Symfony\AI\Agent\Toolbox\Exception\ToolNotFoundException;
 use Symfony\AI\Platform\Result\ToolCall;
 use Symfony\AI\Platform\Tool\Tool;
@@ -27,8 +27,8 @@ interface ToolboxInterface
     public function getTools(): array;
 
     /**
-     * @throws ToolExecutionException if the tool execution fails
-     * @throws ToolNotFoundException  if the tool is not found
+     * @throws ToolExecutionExceptionInterface if the tool execution fails
+     * @throws ToolNotFoundException           if the tool is not found
      */
     public function execute(ToolCall $toolCall): mixed;
 }

src/agent/tests/Toolbox/FaultTolerantToolboxTest.php

@@ -15,6 +15,7 @@
 use PHPUnit\Framework\Attributes\UsesClass;
 use PHPUnit\Framework\TestCase;
 use Symfony\AI\Agent\Toolbox\Exception\ToolExecutionException;
+use Symfony\AI\Agent\Toolbox\Exception\ToolExecutionExceptionInterface;
 use Symfony\AI\Agent\Toolbox\Exception\ToolNotFoundException;
 use Symfony\AI\Agent\Toolbox\FaultTolerantToolbox;
 use Symfony\AI\Agent\Toolbox\ToolboxInterface;
@@ -62,6 +63,26 @@ public function testFaultyToolCall()
         $this->assertSame($expected, $actual);
     }
 
+    public function testCustomToolExecutionException()
+    {
+        $faultyToolbox = $this->createFaultyToolbox(
+            static fn () => new class extends \RuntimeException implements ToolExecutionExceptionInterface {
+                public function getToolCallResult(): array
+                {
+                    return ['error' => 'custom'];
+                }
+            },
+        );
+
+        $faultTolerantToolbox = new FaultTolerantToolbox($faultyToolbox);
+        $expected = ['error' => 'custom'];
+
+        $toolCall = new ToolCall('123456789', 'tool_xyz');
+        $actual = $faultTolerantToolbox->execute($toolCall);
+
+        $this->assertSame($expected, $actual);
+    }
+
     private function createFaultyToolbox(\Closure $exceptionFactory): ToolboxInterface
     {
         return new class($exceptionFactory) implements ToolboxInterface {

src/agent/tests/Toolbox/ToolboxTest.php

@@ -18,11 +18,13 @@
 use Symfony\AI\Agent\Toolbox\Attribute\AsTool;
 use Symfony\AI\Agent\Toolbox\Exception\ToolConfigurationException;
 use Symfony\AI\Agent\Toolbox\Exception\ToolExecutionException;
+use Symfony\AI\Agent\Toolbox\Exception\ToolExecutionExceptionInterface;
 use Symfony\AI\Agent\Toolbox\Exception\ToolNotFoundException;
 use Symfony\AI\Agent\Toolbox\Toolbox;
 use Symfony\AI\Agent\Toolbox\ToolFactory\ChainFactory;
 use Symfony\AI\Agent\Toolbox\ToolFactory\MemoryToolFactory;
 use Symfony\AI\Agent\Toolbox\ToolFactory\ReflectionToolFactory;
+use Symfony\AI\Fixtures\Tool\ToolCustomException;
 use Symfony\AI\Fixtures\Tool\ToolDate;
 use Symfony\AI\Fixtures\Tool\ToolException;
 use Symfony\AI\Fixtures\Tool\ToolMisconfigured;
@@ -60,6 +62,7 @@ protected function setUp(): void
             new ToolOptionalParam(),
             new ToolNoParams(),
             new ToolException(),
+            new ToolCustomException(),
             new ToolDate(),
         ], new ReflectionToolFactory());
     }
@@ -122,6 +125,12 @@ public function testGetTools()
             'This tool is broken',
         );
 
+        $toolCustomException = new Tool(
+            new ExecutionReference(ToolCustomException::class, 'bar'),
+            'tool_custom_exception',
+            'This tool is broken and it exposes the error',
+        );
+
         $toolDate = new Tool(
             new ExecutionReference(ToolDate::class, '__invoke'),
             'tool_date',
@@ -145,6 +154,7 @@ public function testGetTools()
             $toolOptionalParam,
             $toolNoParams,
             $toolException,
+            $toolCustomException,
             $toolDate,
         ];
 
@@ -177,6 +187,14 @@ public function testExecuteWithException()
         $this->toolbox->execute(new ToolCall('call_1234', 'tool_exception'));
     }
 
+    public function testExecuteWithCustomException()
+    {
+        $this->expectException(ToolExecutionExceptionInterface::class);
+        $this->expectExceptionMessage('Custom error.');
+
+        $this->toolbox->execute(new ToolCall('call_1234', 'tool_custom_exception'));
+    }
+
     #[DataProvider('executeProvider')]
     public function testExecute(string $expected, string $toolName, array $toolPayload = [])
     {